pythonhtmlcssselenium-webdriver

With Selenium, how can I find an element by CSS Selector that has to wait for the parent DIV's style to change from display: none; to display: block?


I'm using Selenium to loop through a patient's folders on a hospital's web portal and download all of the PDF files inside each folder. Here is a screenshot of the folders list:

enter image description here

Using Visit #269977 as an example, when I click on the Options link below each folder, a menu pops up with options:

enter image description here

I want Selenium to click the Print option. I can find the element by its XPATH for a specific folder, however that is not useful when I begin looping through folders, as I'm unsure what the index is of each list item. Ideally I'd use the CSS Selector, but the element doesn't seem to be visible.

enter image description here

Here's what the DOM tree looks like prior to Selenium clicking on Options:

enter image description here

Here's what the DOM tree looks like after Selenium clicks on Options:

enter image description here

How can I ensure the Print option can be located by CSS Selector (or another viable alternative) so that I can click on it?

In the code below I've included multiple things I've tried. Each returns an error saying the element can't be found. Happy to provide additional information. Thanks!

#selenium version == 4.27.1
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
import time

# Initial Login:
chrome_options = Options()
chrome_options.add_experimental_option("detach", True)
browser = webdriver.Chrome(options=chrome_options)
browser.get("https://...")
enterEmail = browser.find_element(By.XPATH, "//*[@id='email']")
enterEmail.send_keys("email address")
enterPwd = browser.find_element(By.XPATH, "//*[@id='password']")
enterPwd.send_keys("password")
time.sleep(2)
signIn = browser.find_element(By.XPATH, "//*[@id='login_btn']")
signIn.click()
time.sleep(2)

# Create Dictionary of patient information to loop through:
patientDict = {
'25-003049': ['John Doe', '267087'],
'25-003050': ['John Doe', '269226'],
'25-003051': ['John Doe', '275687']
}

# Create List of patients that errored during download for manual review:
errorList = []

for key in patientDict:

    try:
        # Declare Variables for patientDict keys and values:
        case_number = key
        patient_name = patientDict[key][0]
        visit_id = patientDict[key][1]

        browser.get("https://...")
        searchMenuOption = browser.find_element(By.CLASS_NAME, "search")
        searchMenuOption.click()
        searchName = browser.find_element(By.XPATH, "/html/body/div[1]/div[3]/div[2]/div[1]/div[1]/input")
        searchName.send_keys(patient_name)
        clickMagGlass = browser.find_element(By.XPATH, "/html/body/div[1]/div[3]/div[2]/div[1]/div[1]/button")
        clickMagGlass.click()
        clickName = browser.find_element(By.XPATH, "/html/body/div[1]/div[3]/div[2]/div[1]/ul/li")
        clickName.click()
        items = browser.find_elements(By.XPATH, "//*['/html/body/div[1]/div[4]/div[3]/div[2]/ul/li' and contains(text()," + visit_id + ")]")

        for item in items:
            x = item.find_elements(By.XPATH, ".//descendant::a")
            for y in x:
                y.click()

                # Can find absolute XPATH with list item and href indices:
                clickPrint = y.find_elements(By.XPATH, "//*[@id='documentlist']/li[5]/div[5]/a[5]")

                # It will find it now by CSS Selector based on Alohci's suggested update in the comments, but shows as not interactable if kept in a loop:
                clickPrint = y.find_elements(By.CSS_SELECTOR, ".tool.print")

                for printDialog in clickPrint:
                    printDialog.click()

                 # I've also tried it without the loop, but it returns an unable to locate element:
                clickPrint = y.find_element(By.CSS_SELECTOR, ".tool.print")
                clickPrint.click()

    except:
        errorList.append(patient_name)

<li class="full ui-selectee ui-droppable" data-ref-id="0" data-parent-id="" data-doc-id="406494" data-type="folder" data-uploaded="1688654514" style="">
   <div class="progress center-block open-progress">Opening<span>in progress</span></div>
   <h1 class="column title">
      <a href="#" class="folder " style="opacity: 0;"></a><span class="filename">Visit Jul. 6th, 2023</span>
   </h1>
   <span class="column options">Visit #269977 <a href="#" class="tool-toggle">Options</a>
   </span>
   <span class="column create-date">Jul 06, 2023 09:41:54AM</span>
   <div class="column content-icon">
      <span class="folder-legend">17</span>&nbsp;<span class="type">Folders</span>
   </div>
   <div class="column favorite-icon">
   </div>
   <div class="input with-btn big-border rename hidden">
      <input placeholder="New folder name" value="Visit Jul. 6th, 2023" type="text" name="new_folder_name406494">
      <button class="btn-primary">Ok</button>
   </div>
   <div class="tools open" style="display: block;">
       <a href="#" class="tool rename">Rename</a>
       <a href="#" class="tool comment">Comment</a>
       <a href="#" class="tool delete">Delete</a>
       <a href="#" class="tool export">Export</a>
       <a href="#" class="tool print">Print</a>
       <a href="#" class="no-tool-icon"></a>
   </div>

Solution

  • Given the HTML you posted, I think a couple XPaths are what you need. Both use the visit ID as a reference point to locate 'Options' and then 'Print' and click them.

    //span[contains(text(),'#{visit_id}')]/a[text()='Options']
    
    //li[./span[contains(text(),'#{visit_id}')]]/div/a[text()='Print']
    

    Some feedback...

    1. If you are going to search by ID, then use By.ID instead of By.XPATH, etc. For example,

      By.XPATH, "//*[@id='password']"
      

      turns into

      By.ID, "password"
      

      It's much easier to read and understand quickly.

    2. If you aren't going to use an element more than once, don't assign it to a variable, e.g.

      signIn = browser.find_element(By.XPATH, "//*[@id='login_btn']")
      signIn.click()
      

      becomes

      browser.find_element(By.XPATH, "//*[@id='login_btn']").click()
      

      For me, this reduces code and makes it easier to read.

    3. Using time.sleep() is a bad practice. Sleeps are "dumb"... they always wait the specified time even if the desired element is available sooner. Instead use WebDriverWait() to wait for the desired element to be in a specific state, e.g. clickable would be

      from selenium.webdriver.support import expected_conditions as EC
      from selenium.webdriver.support.wait import WebDriverWait
      
      wait = WebDriverWait(browser, 10)
      wait.until(EC.element_to_be_clickable((By.ID, "login_btn"))).click()
      

      This will wait up to 10 seconds for the element to be clickable and then click it. If it's ready in 1s, it will be clicked at that point and the script moves on to the next line. This increases execution speed while minimizing intermittent failures due to elements not in the right state.

    4. XPaths that start with "/html" are brittle. The smallest page change will likely break that locator resulting in an exception being thrown when running the script. I can help make a better locator if you post the HTML for those three elements.

    Given the suggestions above, I rewrote the code you posted as best I could. Without access to the actual page, I can't be sure that it will run as is. You'll have to give it a try and let me know of any issues.


    The updated code

    #selenium version == 4.27.1
    from selenium import webdriver
    from selenium.webdriver.common.by import By
    from selenium.webdriver.chrome.options import Options
    from selenium.webdriver.support import expected_conditions as EC
    from selenium.webdriver.support.wait import WebDriverWait
    
    # Initial Login:
    chrome_options = Options()
    chrome_options.add_experimental_option("detach", True)
    browser = webdriver.Chrome(options=chrome_options)
    browser.get("https://...")
    
    wait = WebDriverWait(browser, 10)
    
    browser.find_element(By.ID, "email").send_keys("email address")
    browser.find_element(By.ID, "password").send_keys("password")
    browser.find_element(By.ID, "login_btn").click()
    
    # Create Dictionary of patient information to loop through:
    patientDict = {
        '25-003049': ['John Doe', '267087'],
        '25-003050': ['John Doe', '269226'],
        '25-003051': ['John Doe', '275687']
    }
    
    # Create List of patients that errored during download for manual review:
    errorList = []
    
    for case_number, patient_data in patientDict.items():
        try:
            # Declare Variables for patientDict keys and values:
            patient_name = patient_data[0]
            visit_id = patient_data[1]
    
            browser.get("https://...")
            browser.find_element(By.CLASS_NAME, "search").click()
            browser.find_element(By.XPATH, "/html/body/div[1]/div[3]/div[2]/div[1]/div[1]/input").send_keys(patient_name)
            browser.find_element(By.XPATH, "/html/body/div[1]/div[3]/div[2]/div[1]/div[1]/button").click()
            browser.find_element(By.XPATH, "/html/body/div[1]/div[3]/div[2]/div[1]/ul/li").click()
    
            browser.find_element(By.XPATH, f"//span[contains(text(),'#{visit_id}')]/a[text()='Options']").click()
            wait.until(EC.element_to_be_clickable((By.XPATH, f"//li[./span[contains(text(),'#{visit_id}')]]/div/a[text()='Print']"))).click()
    
        except:
            errorList.append(patient_name)