pythonselenium-webdriver

Clicking on expand button using Selenium not possible?

i try to click the "Expand All" Button button

using the following code:

import time
import os, sys
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

WAIT = 1
path = os.path.abspath(os.path.dirname(sys.argv[0])) 

print(f"Checking Browser driver...")
options = Options()
# options.add_argument('--headless=new')  
options.add_argument("start-maximized")
options.add_argument('--log-level=3')  
options.add_experimental_option("prefs", {"profile.default_content_setting_values.notifications": 1})    
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('excludeSwitches', ['enable-logging'])
options.add_experimental_option('useAutomationExtension', False)
options.add_argument('--disable-blink-features=AutomationControlled') 
srv=Service()
driver = webdriver.Chrome (service=srv, options=options)    
# driver.minimize_window()
waitWD = WebDriverWait (driver, 10)         

baseLink = "https://tmsearch.uspto.gov/search/search-information"
print(f"Working for {baseLink}")  
driver.get (baseLink)     
waitWD.until(EC.presence_of_element_located((By.XPATH,'//input[@id="searchbar"]'))).send_keys("SpaceX")
waitWD.until(EC.element_to_be_clickable((By.XPATH, '//button[@class="btn btn-primary md-icon ng-star-inserted"]'))).click()   
waitWD.until(EC.presence_of_element_located((By.XPATH,'//input[@id="goodsAndServices"]'))).send_keys("shirt")
waitWD.until(EC.element_to_be_clickable((By.XPATH, '//button[@class="btn btn-primary md-icon ng-star-inserted"]'))).click()   
time.sleep(WAIT)
soup = BeautifulSoup (driver.page_source, 'lxml')   

driver.execute_script("arguments[0].click();", waitWD.until(EC.element_to_be_clickable((By.XPATH, "(//span[text()='  wordmark '])[1]"))))    
time.sleep(WAIT)    
driver.execute_script("arguments[0].click();", waitWD.until(EC.presence_of_element_located((By.XPATH, '//div[@class="expand_all expanded"]'))))

but i only get this error

Working for https://tmsearch.uspto.gov/search/search-information
Traceback (most recent call last):
  File "F:\DEV\Fiverr2024\TRY\cliff_ckshorts\temp.py", line 41, in <module>
    driver.execute_script("arguments[0].click();", waitWD.until(EC.presence_of_element_located((By.XPATH, '//div[@class="expand_all expanded"]'))))
                                                   ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "F:\DEV\venv\selenium\Lib\site-packages\selenium\webdriver\support\wait.py", line 105, in until
    raise TimeoutException(message, screen, stacktrace)
selenium.common.exceptions.TimeoutException: Message:

How can i press the button on this website using selenium?

Solution

  • There are a few issues with your code:

    1. Many times you are waiting for presence of an element. Presence means that the element exists in the DOM, not that it's interactable. If you try to click or get text from an element that is only present, you will get an ElementNotInteractableException. Instead use EC.element_to_be_clickable if you want to click the element, otherwise use EC.visibility_of_element_located if you want to get text, send keys, etc.

    2. In general, time.sleep() is a bad practice because it's a "dumb" wait. It waits for exactly X seconds even if the element was ready after 1s, etc. Instead use WebDriverWait.

    3. When a new browser tab/window opens up due to a Selenium action, you have to switch to that new window to interact with it. Otherwise Selenium will continue to interact with the original page. In your case, this happens after clicking each Wordmark.

      To switch windows, use driver.switch_to.window(). Once you are done with the new window, you can close it with driver.close() but then you need to switch context back to the original window using driver.switch_to.window() again. There are lots of resources on the web for how to use this correctly.

    There are a bunch of issues with that site currently. It regularly doesn't return search results and throws and error which means you have to search again when this happens. That's why there's some extra code that's not usually necessary.

    I rewrote and simplified the code and it's working, even with all the site issues going on right now.

    from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC

url = 'https://tmsearch.uspto.gov/search/search-information'
driver = webdriver.Chrome()
driver.maximize_window()
driver.get(url)

search_general = "SpaceX\n"
search_goods_services = "shirt\n"
wait = WebDriverWait(driver, 5)
wait_short = WebDriverWait(driver, 1)

def error_found():
    try:
        wait_short.until(EC.visibility_of_element_located((By.XPATH, "//p[text()='The system was unable to perform your search.']")))
        return True
    except TimeoutException:
        return False

wait.until(EC.element_to_be_clickable((By.ID, "searchbar"))).send_keys(search_general)
driver.find_element(By.ID, "goodsAndServices").send_keys(search_goods_services)
wordmarks = wait.until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, "div.card-body span.clickable")))
for wordmark in wordmarks:
    driver.execute_script("arguments[0].click();", wordmark)
    driver.switch_to.window(driver.window_handles[1])

    expand = driver.find_elements(By.CSS_SELECTOR, "div.expand_all.expanded")
    if expand:
        expand[0].click()
    else:
        # "#errorMsg", "The system was unable to perform your search.", "//p[text()='The system was unable to perform your search.']"
        # when the site fails to perform the search, click the "Status" button
        while error_found():
            driver.find_element(By.ID, "statusSearch").click()
            wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "div.docSearchLoadingGif")))
            wait.until(EC.invisibility_of_element_located((By.CSS_SELECTOR, "div.docSearchLoadingGif")))

    # do something on this page
    print(wait.until(EC.visibility_of_element_located((By.XPATH, "//div[.//div[text()='Mark:']]/div[contains(@class, 'markText')]"))).text)
    driver.close()
    driver.switch_to.window(driver.window_handles[0])