How to ensure the results-section got refreshed after a button click using python and selenium

I am doing web scraping using python selenium with the website

I am entering a value into the 'Search value' textbox and clicking the 'Search now' button within a loop. The input and button click actions are functioning correctly.

Since the search results are displayed on the same page, how can I ensure that the results section has been refreshed before I read the data, to avoid capturing outdated or stale information?

I am doing this search based on Isin code by selecting it from LEI Data Field.

Below is my code snippet.

def read(value):
    search_textbox = WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.XPATH ,"//input[contains(@class,'input') and @type='text']")))
    search_textbox.click()
    time.sleep(1)
    search_textbox.clear()
    time.sleep(1)

    script = """
            var input = arguments[0];
            var value = arguments[1];
            input.value = value;
            var event = new Event('input', { bubbles: true });
            input.dispatchEvent(event);
            """
    driver.execute_script(script, search_textbox, value)
    time.sleep(1)

    #Button
    search_btn = WebDriverWait(driver, 30).until(EC.element_to_be_clickable((By.XPATH,"//button[contains(@class,'search-btn') and normalize-space(text())='Search now']")))
    search_btn.click()
    time.sleep(2)

    WebDriverWait(driver, 30).until(EC.visibility_of_element_located((By.ID, 'results-section')))
    results_section = driver.find_element(By.ID, 'results-section')

return results_section


for value in values:
    data = read(value)
    bla bla bla.....

To me WebDriverWait(driver, 30).until(EC.visibility_of_element_located((By.ID, 'results-section'))) this piece of code seems doesn't really help to identify whether the results-secion got refreshed after button clicking on search button.

How I ensure/wait the results-section got refreshed after the search button click ?

I Am quite new to selenium. Appreciate your help on this.

Example values: US035240AQ30, KYG040111059 & INE437A01024 (some values doesn't have any matching output )

Solution

You'll need to compare your existing results with the new refreshed results on the basis of a value.

If you've specific condition then pls edit the old_results line to refer to something in particular or text as you want to compare .

from selenium.common.exceptions import StaleElementReferenceException

def wait_for_refresh(driver, old_element, timeout=30):
    WebDriverWait(driver, timeout).until(EC.staleness_of(old_element))

def read(value):
    try:
#You can modify this Xpath here to change value to other columns in table or inlcude first 5 values as well currently it checks only for 1st one.
        old_results = driver.find_element(By.XPATH, '//div[class="table-cells"]/div[@class="table-cell legal-name"]/a').text
    except:
        old_results = None  # first time, nothing to compare against

    search_textbox = WebDriverWait(driver, 30).until(
        EC.presence_of_element_located((By.XPATH, "//input[contains(@class,'input') and @type='text']"))
    )
    driver.execute_script("arguments[0].value = '';", search_textbox)
    time.sleep(0.5)

    script = """
        var input = arguments[0];
        var value = arguments[1];
        input.value = value;
        var event = new Event('input', { bubbles: true });
        input.dispatchEvent(event);
    """
    driver.execute_script(script, search_textbox, value)
    time.sleep(0.5)


    search_btn = WebDriverWait(driver, 30).until(
        EC.element_to_be_clickable((By.XPATH, "//button[contains(@class,'search-btn') and normalize-space(text())='Search now']"))
    )
    search_btn.click()


    if old_results:
        wait_for_refresh(driver, old_results)


    new_results = WebDriverWait(driver, 30).until(
        EC.presence_of_element_located((By.ID, "results-section"))
    )
    return new_results