pythonseleniumwebdriverwaitexpected-condition

selenium: Can not find element even with full xpath


I am trying to extract data using selenium from [website][1]. But I can not access the button I want to click. I have tried many methods by no use. I don't know why this is happening.
Here is the code.

from selenium import webdriver
from selenium.webdriver.chrome import service
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager

option = webdriver.ChromeOptions()
option.add_argument("start-maximized")
option.add_experimental_option("excludeSwitches", ["enable-automation"])
option.add_experimental_option('useAutomationExtension', False)
option.add_argument("--disable-blink-features")
option.add_argument("--disable-gpu")
#option.add_argument("--headless")
option.add_argument('--disable-blink-features=AutomationControlled')
wd = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=option)


url = "https://greyhoundbet.racingpost.com/#result-meeting/track_id=4&r_date=2021-01-01&r_time=13:24"
wd.execute_cdp_cmd('Network.setUserAgentOverride', {"userAgent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36'})
wd.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
wd.get(url)

button = wd.find_elements(By.XPATH, "/html/body/div[3]/div[2]/div[2]/div[2]/div[1]/div/span")
#button = wd.find_elements(By.CLASSNAME, 'btnLeft btnBlueText')
#button = wd.find_element_by_id('resultsDateBtn')
#used above methods along with relative xpath

print(button)
for i in button:
    if i.get_attribute('id') == "resultsDateBtn":
        i.click()

wd.quit()

I always get empty list or could not find element error [1]: https://greyhoundbet.racingpost.com/#results-list/r_date=2021-01-01


Solution

  • You are missing a wait / delay.
    You should let the page loaded before accessing the web element.
    The best way to do that is to use expected conditions, as following:

    from selenium import webdriver
    from selenium.webdriver.chrome import service
    from selenium.webdriver.common.by import By
    from selenium.webdriver.support.ui import WebDriverWait
    from selenium.webdriver.support import expected_conditions as EC
    from selenium.webdriver.chrome.service import Service
    from webdriver_manager.chrome import ChromeDriverManager
    
    option = webdriver.ChromeOptions()
    option.add_argument("start-maximized")
    option.add_experimental_option("excludeSwitches", ["enable-automation"])
    option.add_experimental_option('useAutomationExtension', False)
    option.add_argument("--disable-blink-features")
    option.add_argument("--disable-gpu")
    #option.add_argument("--headless")
    option.add_argument('--disable-blink-features=AutomationControlled')
    wd = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=option)
    url = "https://greyhoundbet.racingpost.com/#results-list/r_date=2021-01-01"
    wd.execute_cdp_cmd('Network.setUserAgentOverride', {"userAgent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36'})
    wd.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
    wd.get(url)
    wait = WebDriverWait(wd, 20)
    button = wait.until(EC.visibility_of_element_located((By.ID, "resultsDateBtn")))
    #now you can click this button etc
    button.click()