I was wondering if someone could help me out with a web scraping problem.. I am new to both python and web scraping..
I am trying to use the below python program to scrape data from the following webpage,
"https://www.cocorahs.org/ViewData/ListDailyPrecipReports.aspx"
but I keep getting a Runtime error "table#ucReportList_ReportGrid not found" on the raise RuntimeError line (3rd last line).
Now, this program works perfectly fine for a webpage without checkboxes.. However, the webpage I am trying to scrape the data from has some checkboxes and I think that's exactly the part I'm messing up with..
I did a view page source and looked up the html tag names but I feel I am missing something very conceptual.. I am wondering what it might be !!
The two lines after the FL-BV-163 line are the ones I am worried about..
Below is the code. What changes should I make/add to it ? Should I be using some more libraries ? Thank You !!
import requests
from bs4 import BeautifulSoup
from requests_html import HTMLSession
import pandas as pd
from io import StringIO
from datetime import datetime
session = requests.Session()
response = session.get('https://www.cocorahs.org/ViewData/ListDailyPrecipReports.aspx')
soup = BeautifulSoup(response.content, "html.parser")
view_state = soup.find("input", {"name": "__VIEWSTATE", "value": True})["value"]
view_state_generator = soup.find("input", {"name": "__VIEWSTATEGENERATOR", "value": True})["value"]
event_validation = soup.find("input", {"name": "__EVENTVALIDATION", "value": True})["value"]
response = session.post('https://www.cocorahs.org/ViewData/ListDailyPrecipReports.aspx', data={
"__EVENTTARGET": "",
"__EVENTARGUMENT": "",
"__LASTFOCUS": "",
"VAM_Group": "",
"__VIEWSTATE": view_state,
"VAM_JSE": "1",
"__VIEWSTATEGENERATOR": view_state_generator,
"__EVENTVALIDATION": event_validation,
"obsSwitcher:ddlObsUnits": "usunits",
"frmPrecipReportSearch:ucStationTextFieldsFilter:tbTextFieldValue": "FL-BV-163",
"frmPrecipReportSearch:ucStationTextFieldsFilter:cblTextFieldsToSearch:0": "checked",
"frmPrecipReportSearch:ucStationTextFieldsFilter:cblTextFieldsToSearch:1": "",
"frmPrecipReportSearch:ucStateCountyFilter:ddlCountry": "allcountries",
"frmPrecipReportSearch:ucDateRangeFilter:dcStartDate:di": "6/13/2025",
"frmPrecipReportSearch:ucDateRangeFilter:dcStartDate:hfDate": "2025-06-13",
"frmPrecipReportSearch:ucDateRangeFilter:dcEndDate:di": "6/16/2025",
"frmPrecipReportSearch:ucDateRangeFilter:dcEndDate:hfDate": "2025-06-16",
"frmPrecipReportSearch:ddlPrecipField": "GaugeCatch",
"frmPrecipReportSearch:ucPrecipValueFilter:ddlOperator": "LessEqual",
"frmPrecipReportSearch:ucPrecipValueFilter:tbPrecipValue:tbPrecip": "0.15",
"frmPrecipReportSearch:btnSearch": "Search",
})
table = BeautifulSoup(response.content, "html.parser").find("table", id="ucReportList_ReportGrid")
if table is None:
raise RuntimeError("table#ucReportList_ReportGrid not found")
df = pd.read_html(StringIO(str(table)))[0]
print(df)
There's errors in your form data. Correct values for the parameters in question are:
"frmPrecipReportSearch:ucStationTextFieldsFilter:cblTextFieldsToSearch:0": "on",
"frmPrecipReportSearch:ucStateCountyFilter:ddlCountry": "0",
"frmPrecipReportSearch:ddlPrecipField": "TotalPrecipAmt",
also, then remove:
"frmPrecipReportSearch:ucStationTextFieldsFilter:cblTextFieldsToSearch:1": "",
Full code:
import pandas as pd
from io import StringIO
from datetime import datetime
session = requests.Session()
response = session.get('https://www.cocorahs.org/ViewData/ListDailyPrecipReports.aspx')
soup = BeautifulSoup(response.content, "html.parser")
view_state = soup.find("input", {"name": "__VIEWSTATE", "value": True})["value"]
view_state_generator = soup.find("input", {"name": "__VIEWSTATEGENERATOR", "value": True})["value"]
event_validation = soup.find("input", {"name": "__EVENTVALIDATION", "value": True})["value"]
headers = {
'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36'}
data={"__EVENTTARGET": "",
"__EVENTARGUMENT": "",
"__LASTFOCUS": "",
"VAM_Group": "",
"__VIEWSTATE": view_state,
"VAM_JSE": "1",
"__VIEWSTATEGENERATOR": view_state_generator,
"__EVENTVALIDATION": event_validation,
"obsSwitcher:ddlObsUnits": "usunits",
"frmPrecipReportSearch:ucStationTextFieldsFilter:tbTextFieldValue": "FL-BV-163",
"frmPrecipReportSearch:ucStationTextFieldsFilter:cblTextFieldsToSearch:0": "on",
"frmPrecipReportSearch:ucStateCountyFilter:ddlCountry": "0",
"frmPrecipReportSearch:ucDateRangeFilter:dcStartDate:di": "6/13/2025",
"frmPrecipReportSearch:ucDateRangeFilter:dcStartDate:hfDate": "2025-06-13",
"frmPrecipReportSearch:ucDateRangeFilter:dcEndDate:di": "6/16/2025",
"frmPrecipReportSearch:ucDateRangeFilter:dcEndDate:hfDate": "2025-06-16",
"frmPrecipReportSearch:ddlPrecipField": "TotalPrecipAmt",
"frmPrecipReportSearch:ucPrecipValueFilter:ddlOperator": "LessEqual",
"frmPrecipReportSearch:ucPrecipValueFilter:tbPrecipValue:tbPrecip": "0.15",
"frmPrecipReportSearch:btnSearch": "Search"
}
response = session.post('https://www.cocorahs.org/ViewData/ListDailyPrecipReports.aspx', headers=headers, data=data)
table = BeautifulSoup(response.content, "html.parser").find("table", id="ucReportList_ReportGrid")
if table is None:
raise RuntimeError("table#ucReportList_ReportGrid not found")
df = pd.read_html(StringIO(str(table)))[0]
Output:
print(df.to_string())
0 1 2 3 4 5 6 7 8 9 10 11
0 Obs Date ▲ Obs Time Station Number Station Name Gauge Catch in. 24hr Snowfall in in SLR Snowpack in in Density Notes State County View Maps
1 6/16/2025 8:00 AM FL-BV-163 Melbourne 4.7 SSE 0.00 0.0 NA NA NA NA NA NaN FL Brevard NaN Active | Static
2 6/14/2025 8:00 AM FL-BV-163 Melbourne 4.7 SSE 0.00 0.0 NA NA NA NA NA NaN FL Brevard NaN Active | Static
3 6/13/2025 8:00 AM FL-BV-163 Melbourne 4.7 SSE 0.00 0.0 NA NA NA NA NA NaN FL Brevard NaN Active | Static