pythonselenium-webdriverwebdriver

Selenium doesn't open website with provided url (url is valid)


I try to open whois EURID website with no luck. Selenium opens browser (tried with Chrome and FF), but when I try to open particular URL (http://whois.eurid.eu/):

nothing opens, I got blank page only.

I have driver set by this function:

    def set_driver() -> WebDriver:
        service = Service()
        if cfg["browser"]["browser_name"].strip().lower() == "firefox":
            options = webdriver.FirefoxOptions()
            driver = webdriver.Firefox(service=service, options=options)
        elif cfg["browser"]["browser_name"].strip().lower() == "edge":
            options = webdriver.EdgeOptions()
            driver = webdriver.Edge(service=service, options=options)
        else:
            options = webdriver.ChromeOptions()
            driver = webdriver.Chrome(service=service, options=options)
        return driver

Using driver (set above) I try to open webiste and scrape some data (information about domain):

    domain_name = "ddddddd.eu"  # any name ending with '.eu'

    URL = f"https://whois.eurid.eu/en/search/?domain={domain_name}"

    drv = set_driver()
    drv.get(URL) 
    time.sleep(5)

I think this website is currently somehow protected, as I have had no problems with opening it before.


Solution

  • Cloudflare Protection or User Agent Detection or JavaScript might cause the issue.

    Please find below an enhanced solution for your consideration:

    from selenium import webdriver
    from selenium.webdriver.chrome.service import Service
    from selenium.webdriver.chrome.options import Options
    from selenium.webdriver.common.by import By
    from selenium.webdriver.support.ui import WebDriverWait
    from selenium.webdriver.support import expected_conditions as EC
    import time
    
    def get_eurid_whois(domain):
        chrome_options = Options()
        chrome_options.add_argument("--start-maximized")
        chrome_options.add_argument("--disable-blink-features=AutomationControlled")
        chrome_options.add_argument("--disable-infobars")
        chrome_options.add_argument("--disable-extensions")
        chrome_options.add_argument("--disable-popup-blocking")
        chrome_options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                                    "AppleWebKit/537.36 (KHTML, like Gecko) "
                                    "Chrome/122.0.0.0 Safari/537.36")
    
        driver = webdriver.Chrome(service=Service(), options=chrome_options)
    
        try:
            url = f"https://whois.eurid.eu/en/search/?domain={domain}"
            driver.get(url)
    
            time.sleep(3)
    
            try:
                WebDriverWait(driver, 15).until(
                    EC.presence_of_element_located((By.CSS_SELECTOR, ".btn"))
                )
            except:
                print("Page didn't load properly. Check if blocked by CAPTCHA.")
                return None
    
            try:
                register_button = driver.find_element(By.XPATH, "//a[contains(text(), 'Register now')]")
                print("Domain is available. 'Register now' button found.")
                return "Available"
            except:
                print("Domain is not available. 'Register now' button not found.")
    
            whois_data = driver.find_element(By.CSS_SELECTOR, ".btn").text
            return whois_data
    
        finally:
            driver.quit()
    
    print(get_eurid_whois("ddddddd.eu"))