pythonselenium-webdriverselenium-chromedriver

Error message when scraping website via selenium


The Python code:

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.support.ui import Select
import pandas as pd
import time

# define the website to scrape and path where the chromedriver is located
website = 'https://www.adamchoi.co.uk/overs/detailed'
path = 'C:/users/Administrator/Downloads/chromedriver-win64/chromedriver.exe'  # write your path here
service = Service(executable_path=path)  # selenium 4
driver = webdriver.Chrome(service=service)  # define 'driver' variable
# open Google Chrome with chromedriver
driver.get(website)

# locate and click on a button
all_matches_button = driver.find_element(by='xpath', value='//label[@analytics-event="All matches"]')
all_matches_button.click()

# select elements in the table
matches = driver.find_elements(by='xpath', value='//tr')

# storage data in lists
date = []
home_team = []
score = []
away_team = []

# looping through the matches list
for match in matches:
    date.append(match.find_element(by='xpath', value='./td[1]').text)
    home = match.find_element(by='xpath', value='./td[2]').text
    home_team.append(home)
    print(home)
    score.append(match.find_element(by='xpath', value='./td[3]').text)
    away_team.append(match.find_element(by='xpath', value='./td[4]').text)
# quit drive we opened at the beginning
driver.quit()

# Create Dataframe in Pandas and export to CSV (Excel)
df = pd.DataFrame({'date': date, 'home_team': home_team, 'score': score, 'away_team': away_team})
df.to_csv('football_data.csv', index=False)
print(df)

How can I solve the following error message by modifying the above Python code?

Traceback (most recent call last):
  File "C:\Users\Administrator\PycharmProjects\WebScraping\1.selenium4-adamchoi.py", line 31, in <module>
    home = match.find_element(by='xpath', value='./td[2]').text
           ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\Administrator\PycharmProjects\WebScraping\.venv\Lib\site-packages\selenium\webdriver\remote\webelement.py", line 601, in find_element
    return self._execute(Command.FIND_CHILD_ELEMENT, {"using": by, "value": value})["value"]
           ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\Administrator\PycharmProjects\WebScraping\.venv\Lib\site-packages\selenium\webdriver\remote\webelement.py", line 572, in _execute
    return self._parent.execute(command, params)
           ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
  File "C:\Users\Administrator\PycharmProjects\WebScraping\.venv\Lib\site-packages\selenium\webdriver\remote\webdriver.py", line 458, in execute
    self.error_handler.check_response(response)
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^
  File "C:\Users\Administrator\PycharmProjects\WebScraping\.venv\Lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 233, in check_response
    raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"xpath","selector":"./td[2]"}
  (Session info: chrome=142.0.7444.60); For documentation on this error, please visit: https://www.selenium.dev/documentation/webdriver/troubleshooting/errors#nosuchelementexception
Stacktrace:
Symbols not available. Dumping unresolved backtrace:
    0x7ff6a9617a35
    0x7ff6a9617a90
    0x7ff6a93916ad
    0x7ff6a93ea13e
    0x7ff6a93ea44c
    0x7ff6a93dcc9c
    0x7ff6a93dcb56
    0x7ff6a943b8fb
    0x7ff6a93db068
    0x7ff6a93dbe93
    0x7ff6a98d29d0
    0x7ff6a98cce50
    0x7ff6a98ecc45
    0x7ff6a96330ce
    0x7ff6a963adbf
    0x7ff6a9620c14
    0x7ff6a9620dcf
    0x7ff6a9606828
    0x7ffb0b847bd4
    0x7ffb0c6eced1


Process finished with exit code 1

Solution

  • There are rows with text "Next match: ..." and "Next match O/U 2.5 odds vs ..." which have only one td in row - this is your problem.

    enter image description here

    You may use try/except to catch error and skip rows.

    Or you may get all td in row and check if there are 4 elements or not:
    (frankly, there are 6 <td> in row - maybe some of them are for some extra info or icons)

    for match in matches:
        all_tds = match.find_elements(by="xpath", value="./td")
    
        if len(all_tds) < 6: 
            print("not enough <td> in row")
        else:
            date.append(all_tds[0].text)
            # all_tds[1] - empty column
            home_team.append(all_tds[2].text)
            score.append(all_tds[3].text)
            away_team.append(all_tds[4].text)
            # all_tds[5] - empty column
    
            print(date[-1], home_team[-1], score[-1], away_team[-1])
    

    Result:

    26-10-2025 Arsenal 1 - 0 Crystal Palace
    18-10-2025 Fulham 0 - 1 Arsenal
    04-10-2025 Arsenal 2 - 0 West Ham
    28-09-2025 Newcastle 1 - 2 Arsenal
    21-09-2025 Arsenal 1 - 1 Man City
    13-09-2025 Arsenal 3 - 0 Nott'm Forest
    31-08-2025 Liverpool 1 - 0 Arsenal
    23-08-2025 Arsenal 5 - 0 Leeds
    17-08-2025 Man United 0 - 1 Arsenal
    not enough <td> in row
    not enough <td> in row
    26-10-2025 Aston Villa 1 - 0 Man City
    19-10-2025 Tottenham 1 - 2 Aston Villa
    05-10-2025 Aston Villa 2 - 1 Burnley
    28-09-2025 Aston Villa 3 - 1 Fulham
    21-09-2025 Sunderland 1 - 1 Aston Villa
    13-09-2025 Everton 0 - 0 Aston Villa
    31-08-2025 Aston Villa 0 - 3 Crystal Palace
    23-08-2025 Brentford 1 - 0 Aston Villa
    16-08-2025 Aston Villa 0 - 0 Newcastle
    not enough <td> in row
    not enough <td> in row
    

    Full code used for tests (with extra code to closing Cookies Message)

    from selenium import webdriver
    import pandas as pd
    
    website = "https://www.adamchoi.co.uk/overs/detailed"
    driver = webdriver.Chrome()  # Selenium 4 can automatically download driver
    driver.get(website)
    
    # TODO: close cookies message
    driver.find_element(by="xpath", value='//button[@aria-label="Consent"]').click()
    
    all_matches_button = driver.find_element(
        by="xpath", value='//label[@analytics-event="All matches"]'
    )
    all_matches_button.click()
    
    matches = driver.find_elements(by="xpath", value="//tr")
    
    date = []
    home_team = []
    score = []
    away_team = []
    
    for match in matches:
        all_tds = match.find_elements(by="xpath", value="./td")
    
        if len(all_tds) < 6:
            print("not enough <td> in row")
        else:
            date.append(all_tds[0].text)
            # all_tds[1] - empty column
            home_team.append(all_tds[2].text)
            score.append(all_tds[3].text)
            away_team.append(all_tds[4].text)
            # all_tds[5] - empty column
            print(date[-1], home_team[-1], score[-1], away_team[-1])
    
    driver.quit()
    
    df = pd.DataFrame(
        {"date": date, "home_team": home_team, "score": score, "away_team": away_team}
    )
    df.to_csv("football_data.csv", index=False)
    print(df)