Im trying to scrape excel data by clicking on multiple dropdown and picking dates in the following page. Have reproduced my python code below. The weblink I'm trying to scrape: https://www.niftyindices.com/reports/historical-data
So i first select P/E, P/B & Div.Yield values, then i select "Equity" then "NIFTY 500" in Index, then chose the from and to dates and then submit.
import os
import time
import traceback
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import Select, WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
# ✅ ChromeDriver Path
chrome_driver_path = r"C:\Users\abcd\Documents\chromedriver-win64\chromedriver.exe"
# ✅ Download directory
download_path = r"C:\Users\abcd\Downloads\Indices"
# ✅ Chrome Options
options = Options()
options.add_argument("--window-size=1920x1080")
options.add_argument("--disable-popup-blocking")
options.add_argument("--disable-blink-features=AutomationControlled")
# ✅ Initialize WebDriver
service = Service(chrome_driver_path)
service.start()
driver = webdriver.Remote(service.service_url, options=options)
driver.set_page_load_timeout(300)
wait = WebDriverWait(driver, 60)
try:
# ✅ Open the webpage
url = "https://www.niftyindices.com/reports/historical-data"
print("🔹 Opening webpage...")
driver.get(url)
# ✅ Wait for the page to fully load
time.sleep(5)
# ✅ Check if there's an iframe and switch to it
iframes = driver.find_elements(By.TAG_NAME, "iframe")
if iframes:
print("🔹 Switching to iframe...")
driver.switch_to.frame(iframes[0]) # Switch to the first iframe
# ✅ Select "P/E, P/B & Div. Yield values"
print("🔹 Clicking Historical Menu...")
menu_button = wait.until(EC.element_to_be_clickable((By.ID, "HistoricalMenu")))
menu_button.click()
time.sleep(2)
print("🔹 Selecting 'P/E, P/B & Div. Yield values'...")
div_yield_option = wait.until(
EC.element_to_be_clickable((By.XPATH, "//li[contains(text(), 'P/E, P/B & Div.Yield values')]"))
)
div_yield_option.click()
time.sleep(2)
# ✅ Select "Equity" from the dropdown
print("🔹 Selecting 'Equity' dropdown...")
equity_dropdown = wait.until(EC.element_to_be_clickable((By.ID, "ddlHistoricaldivtypee")))
Select(equity_dropdown).select_by_value("Equity")
time.sleep(2)
# ✅ Select "NIFTY 500" from the dropdown
print("🔹 Selecting 'NIFTY 500' dropdown...")
index_dropdown = wait.until(EC.element_to_be_clickable((By.ID, "ddlHistoricaldivtypeeindex")))
Select(index_dropdown).select_by_value("NIFTY 500")
time.sleep(2)
# ✅ Function to Select Date
def select_date(date_field_id, target_day, target_month, target_year):
print(f"🔹 Selecting date: {target_day}-{target_month}-{target_year}")
try:
# ✅ Check if date picker is inside an iframe
iframes = driver.find_elements(By.TAG_NAME, "iframe")
if iframes:
print(f"🔹 {len(iframes)} iframe(s) found. Switching to first iframe...")
driver.switch_to.frame(iframes[0])
time.sleep(2)
# ✅ Wait for the date picker field to be clickable
date_field = wait.until(EC.element_to_be_clickable((By.ID, date_field_id)))
# ✅ Scroll into view if it's off-screen
driver.execute_script("arguments[0].scrollIntoView(true);", date_field)
time.sleep(1)
# ✅ Remove 'readonly' attribute (if applicable) to allow interaction
driver.execute_script("arguments[0].removeAttribute('readonly');", date_field)
# ✅ Click the date field to open the calendar
date_field.click()
time.sleep(2) # Wait for the calendar to appear
# ✅ Select the year
year_dropdown = wait.until(EC.element_to_be_clickable((By.CLASS_NAME, "ui-datepicker-year")))
Select(year_dropdown).select_by_value(str(target_year))
time.sleep(1)
# ✅ Select the month
month_dropdown = wait.until(EC.element_to_be_clickable((By.CLASS_NAME, "ui-datepicker-month")))
Select(month_dropdown).select_by_value(str(target_month - 1)) # Month values are 0-indexed
time.sleep(1)
# ✅ Select the day
day_xpath = f"//td[@data-handler='selectDay'][@data-month='{target_month - 1}'][@data-year='{target_year}']/a[text()='{target_day}']"
day_element = wait.until(EC.element_to_be_clickable((By.XPATH, day_xpath)))
day_element.click()
time.sleep(2)
print(f"✅ Successfully selected {target_day}-{target_month}-{target_year}")
# ✅ Switch back to default content after selection
driver.switch_to.default_content()
except Exception as e:
print(f"❌ Error selecting date: {target_day}-{target_month}-{target_year}")
print(traceback.format_exc())
# ✅ Select the FROM date (01-Jan-2024)
select_date("datepickerFromDivYield", target_day=1, target_month=1, target_year=2024)
# ✅ Select the TO date (31-Jan-2025)
select_date("datepickerToDivYield", target_day=31, target_month=1, target_year=2025)
# ✅ Click Submit button
print("🔹 Clicking Submit button...")
submit_button = wait.until(EC.element_to_be_clickable((By.ID, "submit_buttonDivdata")))
submit_button.click()
# ✅ Wait for CSV download link
print("🔹 Waiting for CSV download link...")
csv_link = wait.until(EC.element_to_be_clickable((By.ID, "exporthistoricaldiv")))
csv_link.click()
# ✅ Wait for file download
print("🔹 Waiting for file download...")
time.sleep(15)
# ✅ Verify if file is downloaded
print("🔹 Checking downloaded files...")
files = os.listdir(download_path)
csv_files = [f for f in files if f.endswith(".csv")]
if csv_files:
print(f"✅ CSV file downloaded successfully: {csv_files[-1]}")
else:
print("❌ No CSV file downloaded.")
except Exception as e:
print(f"⚠️ An error occurred: {e}")
print(traceback.format_exc())
finally:
driver.quit()
The error i get:
❌ Error selecting date: 1-1-2024
Traceback (most recent call last):
File "c:\users\am364971\documents\python scripts\download_nse_indices.py", line 84, in select_date
date_field = wait.until(EC.element_to_be_clickable((By.ID, date_field_id)))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\AM364971\AppData\Local\anaconda3\Lib\site-packages\selenium\webdriver\support\wait.py", line 146, in until
raise TimeoutException(message, screen, stacktrace)
selenium.common.exceptions.TimeoutException: Message:
🔹 Selecting date: 31-1-2025
❌ Error selecting date: 31-1-2025
Traceback (most recent call last):
File "c:\users\am364971\documents\python scripts\download_nse_indices.py", line 84, in select_date
date_field = wait.until(EC.element_to_be_clickable((By.ID, date_field_id)))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\AM364971\AppData\Local\anaconda3\Lib\site-packages\selenium\webdriver\support\wait.py", line 146, in until
raise TimeoutException(message, screen, stacktrace)
selenium.common.exceptions.TimeoutException: Message:
🔹 Clicking Submit button...
🔹 Waiting for CSV download link...
⚠️ An error occurred: Message:
Traceback (most recent call last):
File "c:\users\am364971\documents\python scripts\download_nse_indices.py", line 136, in <module>
csv_link = wait.until(EC.element_to_be_clickable((By.ID, "exporthistoricaldiv")))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\AM364971\AppData\Local\anaconda3\Lib\site-packages\selenium\webdriver\support\wait.py", line 146, in until
raise TimeoutException(message, screen, stacktrace)
selenium.common.exceptions.TimeoutException: Message:
Kindly help in debugging.
You are complicating the process too much. The code gets a timeout exception when it tries to select the dates.
The problem is When you are using to scroll to the element script and this causes the element to be hidden under the header menu.
Also, you should click on the parent div instead of the read-only input element.
I have disabled the script lines and updated the selector for the date selection button.
The following code downloads the CSV files.
import os
import time
import traceback
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import Select, WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
# ✅ ChromeDriver Path
chrome_driver_path = r"C:\\Users\\abcd\\Documents\\chromedriver-win64\\chromedriver.exe"
# ✅ Download directory
download_path = r"C:\\Users\\abcd\\Downloads"
# ✅ Chrome Options
options = Options()
options.add_argument("--window-size=1920x1080")
options.add_argument("--disable-popup-blocking")
options.add_argument("--disable-blink-features=AutomationControlled")
# ✅ Initialize WebDriver
service = Service(chrome_driver_path)
service.start()
driver = webdriver.Remote(service.service_url, options=options)
driver.set_page_load_timeout(300)
wait = WebDriverWait(driver, 60)
try:
# ✅ Open the webpage
url = "https://www.niftyindices.com/reports/historical-data"
print("🔹 Opening webpage...")
driver.get(url)
# ✅ Wait for the page to fully load
time.sleep(5)
# ✅ Check if there's an iframe and switch to it
iframes = driver.find_elements(By.TAG_NAME, "iframe")
if iframes:
print("🔹 Switching to iframe...")
driver.switch_to.frame(iframes[0]) # Switch to the first iframe
# ✅ Select "P/E, P/B & Div. Yield values"
print("🔹 Clicking Historical Menu...")
menu_button = wait.until(EC.element_to_be_clickable((By.ID, "HistoricalMenu")))
menu_button.click()
time.sleep(2)
print("🔹 Selecting 'P/E, P/B & Div. Yield values'...")
div_yield_option = wait.until(
EC.element_to_be_clickable((By.XPATH, "//li[contains(text(), 'P/E, P/B & Div.Yield values')]"))
)
div_yield_option.click()
time.sleep(2)
# ✅ Select "Equity" from the dropdown
print("🔹 Selecting 'Equity' dropdown...")
equity_dropdown = wait.until(EC.element_to_be_clickable((By.ID, "ddlHistoricaldivtypee")))
Select(equity_dropdown).select_by_value("Equity")
time.sleep(2)
# ✅ Select "NIFTY 500" from the dropdown
print("🔹 Selecting 'NIFTY 500' dropdown...")
index_dropdown = wait.until(EC.element_to_be_clickable((By.ID, "ddlHistoricaldivtypeeindex")))
Select(index_dropdown).select_by_value("NIFTY 500")
time.sleep(2)
# ✅ Function to Select Date
def select_date(date_field_id, target_day, target_month, target_year):
print(f"🔹 Selecting date: {target_day}-{target_month}-{target_year}")
try:
# ✅ Check if date picker is inside an iframe
iframes = driver.find_elements(By.TAG_NAME, "iframe")
if iframes:
print(f"🔹 {len(iframes)} iframe(s) found. Switching to first iframe...")
driver.switch_to.frame(iframes[0])
time.sleep(2)
# ✅ Wait for the date picker field to be clickable
date_field = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, f"div.dateHolder:has(#{date_field_id})")))
# ✅ Click the date field to open the calendar
date_field.click()
wait.until(EC.visibility_of_element_located((By.ID,"ui-datepicker-div"))) # Wait for the calendar to appear
# ✅ Select the year
year_dropdown = wait.until(EC.element_to_be_clickable((By.CLASS_NAME, "ui-datepicker-year")))
Select(year_dropdown).select_by_value(str(target_year))
time.sleep(1)
# ✅ Select the month
month_dropdown = wait.until(EC.element_to_be_clickable((By.CLASS_NAME, "ui-datepicker-month")))
Select(month_dropdown).select_by_value(str(target_month - 1)) # Month values are 0-indexed
time.sleep(1)
# ✅ Select the day
day_xpath = f"//td[@data-handler='selectDay'][@data-month='{target_month - 1}'][@data-year='{target_year}']/a[text()='{target_day}']"
day_element = wait.until(EC.element_to_be_clickable((By.XPATH, day_xpath)))
day_element.click()
time.sleep(2)
print(f"✅ Successfully selected {target_day}-{target_month}-{target_year}")
# ✅ Switch back to default content after selection
driver.switch_to.default_content()
except Exception as e:
print(f"❌ Error selecting date: {target_day}-{target_month}-{target_year}")
print(traceback.format_exc())
# ✅ Select the FROM date (01-Jan-2024)
select_date("datepickerFromDivYield", target_day=1, target_month=1, target_year=2024)
# ✅ Select the TO date (31-Jan-2025)
select_date("datepickerToDivYield", target_day=31, target_month=1, target_year=2025)
# ✅ Click Submit button
print("🔹 Clicking Submit button...")
submit_button = wait.until(EC.element_to_be_clickable((By.ID, "submit_buttonDivdata")))
submit_button.click()
# ✅ Wait for CSV download link
print("🔹 Waiting for CSV download link...")
csv_link = wait.until(EC.element_to_be_clickable((By.ID, "exporthistoricaldiv")))
csv_link.click()
# ✅ Wait for file download
print("🔹 Waiting for file download...")
time.sleep(15)
# ✅ Verify if file is downloaded
print("🔹 Checking downloaded files...")
files = os.listdir(download_path)
csv_files = [f for f in files if f.endswith(".csv")]
if csv_files:
print(f"✅ CSV file downloaded successfully: {csv_files[-1]}")
else:
print("❌ No CSV file downloaded.")
except Exception as e:
print(f"⚠️ An error occurred: {e}")
print(traceback.format_exc())
finally:
driver.quit()