pythonselenium-webdriverweb-scraping

Why doesn't Selenium find elements?


I want to scrape the address of the houses from the website, but the code returns no output, can someone tell me where the problem is from?

here is my code:

from selenium import webdriver
from selenium.webdriver.common.by import By

url= "https://www.zoopla.co.uk/house-prices/england/?new_homes=include&q=england+&orig_q=united+kingdom&view_type=list&pn=1"
driver=webdriver.Chrome()
driver.get(url)
# Find elements using By.CLASS_NAME
houses = driver.find_elements(By.CLASS_NAME, "w6xkpv1 w6xkpv4")

for house in houses:
    # Find address within each house element
    address = house.find_element(By.XPATH, './/*[@id="main-content"]/div[2]/div/div/section/div[1]/div[2]/div[1]/div/div/div/div[1]/a/h2').text
    print(address)

Solution

  • You could take a completely different approach using CSS selectors as follows:

    from selenium.webdriver import Chrome
    from selenium.webdriver.support.ui import WebDriverWait
    from selenium.webdriver.support import expected_conditions as EC
    from selenium.webdriver.common.by import By
    
    url = "https://www.zoopla.co.uk/house-prices/england/?new_homes=include&q=england+&orig_q=united+kingdom&view_type=list&pn=1"
    
    # handle elements that may not be currently visible
    def etext(e):
        if e:
            if t := e.text.strip():
                return t
            if (p := e.get_property("textContent")) and isinstance(p, str):
                return p.strip()
        return ""
    
    with Chrome() as driver:
        driver.get(url)
        wait = WebDriverWait(driver, 10)
        ec = EC.presence_of_all_elements_located
        sel = By.CSS_SELECTOR, "div[data-testid=result-item] h2"
        for h2 in wait.until(ec(sel)):
            print(etext(h2))