pythonpostget

Post Request Not working for scraping a website


Step 1: Send a GET request to fetch the page and extract hidden form values We'll first send a GET request to the page and extract necessary hidden form values like __VIEWSTATE, __VIEWSTATEGENERATOR, and __EVENTVALIDATION which are needed for subsequent POST requests.

import requests
from bs4 import BeautifulSoup
import re

# Define the URL and headers for the initial GET request
url = "https://jamabandi.punjab.gov.in/default.aspx"
headers = {
    'Accept': '*/*',
    'Accept-Language': 'en-GB,en-US;q=0.9,en;q=0.8',
    'Cache-Control': 'no-cache',
    'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
    'Origin': 'https://jamabandi.punjab.gov.in',
    'Referer': 'https://jamabandi.punjab.gov.in/default.aspx',
    'Sec-Fetch-Dest': 'empty',
    'Sec-Fetch-Mode': 'cors',
    'Sec-Fetch-Site': 'same-origin',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36',
    'X-Requested-With': 'XMLHttpRequest'
}

# Send GET request to fetch the page
response = requests.get(url, headers=headers)

# Extract hidden form fields (VIEWSTATE, VIEWSTATEGENERATOR, EVENTVALIDATION)
viewstate = re.search(r'__VIEWSTATE" value="([^"]+)', response.text).group(1)
viewstategenerator = re.search(r'__VIEWSTATEGENERATOR" value="([^"]+)', response.text).group(1)
eventvalidation = re.search(r'__EVENTVALIDATION" value="([^"]+)', response.text).group(1)

# Print the extracted values to verify
print("VIEWSTATE:", viewstate)
print("VIEWSTATEGENERATOR:", viewstategenerator)
print("EVENTVALIDATION:", eventvalidation)

Step 2: Post request to select the district Now that we have the necessary hidden form values, we can post the district selection by sending a POST request.

 Define the payload for the POST request to select a district
payload = {
    'ctl00$ScriptManager1': 'ctl00$ContentPlaceHolder1$updRegionSelector|ctl00$ContentPlaceHolder1$ddlDistrict',
    'ctl00$SelectRegion$rdPeriod': '1',
    'ctl00$SelectRegion$ddlDistrict': '10',  # Choose district here (example: 10)
    'ctl00$SelectRegion$ddlTehsil': '',
    'ctl00$SelectRegion$ddlVillage': '',
    'ctl00$SelectRegion$ddlYear': '',
    'ctl00$ddlLang': 'en-US',
    'ctl00$ContentPlaceHolder1$rdPeriod': '1',
    'ctl00$ContentPlaceHolder1$ddlDistrict': '10',  # Same district
    '__EVENTTARGET': 'ctl00$ContentPlaceHolder1$ddlDistrict',
    '__EVENTARGUMENT': '',
    '__LASTFOCUS': '',
    '__VIEWSTATE': viewstate,
    '__VIEWSTATEGENERATOR': viewstategenerator,
    '__EVENTVALIDATION': eventvalidation,
    '__ASYNCPOST': 'true'
}

# Send the POST request to select the district
post_url = "https://jamabandi.punjab.gov.in/default.aspx"
response = requests.post(post_url, headers=headers, data=payload)

# Print response to check if the request was successful
print(response.text)

Step 3: Select Tehsil Next, we move to select the Tehsil, following the same approach as the district selection. We need to extract the hidden form values again from the updated page (the one that follows the district selection) and post the Tehsil selection.

    # Extract hidden form fields after the district selection (VIEWSTATE, VIEWSTATEGENERATOR, EVENTVALIDATION)
    viewstate = re.search(r'__VIEWSTATE" value="([^"]+)', response.text).group(1)
    viewstategenerator = re.search(r'__VIEWSTATEGENERATOR" value="([^"]+)', response.text).group(1)
    eventvalidation = re.search(r'__EVENTVALIDATION" value="([^"]+)', response.text).group(1)

# Define the payload for the POST request to select the Tehsil
payload = {
    'ctl00$ScriptManager1': 'ctl00$ContentPlaceHolder1$updRegionSelector|ctl00$ContentPlaceHolder1$ddlTehsil',
    'ctl00$SelectRegion$rdPeriod': '1',
    'ctl00$SelectRegion$ddlDistrict': '10',
    'ctl00$SelectRegion$ddlTehsil': '63',  # Choose Tehsil here (example: 63)
    'ctl00$SelectRegion$ddlVillage': '',
    'ctl00$SelectRegion$ddlYear': '',
    'ctl00$ddlLang': 'en-US',
    'ctl00$ContentPlaceHolder1$rdPeriod': '1',
    'ctl00$ContentPlaceHolder1$ddlDistrict': '10',
    'ctl00$ContentPlaceHolder1$ddlTehsil': '63',
    '__EVENTTARGET': 'ctl00$ContentPlaceHolder1$ddlTehsil',
    '__EVENTARGUMENT': '',
    '__LASTFOCUS': '',
    '__VIEWSTATE': viewstate,
    '__VIEWSTATEGENERATOR': viewstategenerator,
    '__EVENTVALIDATION': eventvalidation,
    '__ASYNCPOST': 'true'
}



  # Send the POST request to select the Tehsil
    response = requests.post(post_url, headers=headers, data=payload)
    
    # Print response to check if the request was successful
    print(response.text)

Step 4: Select Village We follow the same approach as before, but now we need to select the village. The hidden form values should be extracted again after selecting the Tehsil.

# Extract hidden form fields after the Tehsil selection (VIEWSTATE, VIEWSTATEGENERATOR, EVENTVALIDATION)
viewstate = re.search(r'__VIEWSTATE" value="([^"]+)', response.text).group(1)
viewstategenerator = re.search(r'__VIEWSTATEGENERATOR" value="([^"]+)', response.text).group(1)
eventvalidation = re.search(r'__EVENTVALIDATION" value="([^"]+)', response.text).group(1)

# Define the payload for the POST request to select the Village
payload = {
    'ctl00$ScriptManager1': 'ctl00$ContentPlaceHolder1$updRegionSelector|ctl00$ContentPlaceHolder1$ddlVillage',
    'ctl00$SelectRegion$rdPeriod': '1',
    'ctl00$SelectRegion$ddlDistrict': '10',
    'ctl00$SelectRegion$ddlTehsil': '63',
    'ctl00$SelectRegion$ddlVillage': '1E36A812-C218-DD11-8334-000E0CA49FC8',  # Choose Village here (example village ID)
    'ctl00$SelectRegion$ddlYear': '',
    'ctl00$ddlLang': 'en-US',
    'ctl00$ContentPlaceHolder1$rdPeriod': '1',
    'ctl00$ContentPlaceHolder1$ddlDistrict': '10',
    'ctl00$ContentPlaceHolder1$ddlTehsil': '63',
    'ctl00$ContentPlaceHolder1$ddlVillage': '1E36A812-C218-DD11-8334-000E0CA49FC8',
    '__EVENTTARGET': 'ctl00$ContentPlaceHolder1$ddlVillage',
    '__EVENTARGUMENT': '',
    '__LASTFOCUS': '',
    '__VIEWSTATE': viewstate,
    '__VIEWSTATEGENERATOR': viewstategenerator,
    '__EVENTVALIDATION': eventvalidation,
    '__ASYNCPOST': 'true'
}

# Send the POST request to select the Village
response = requests.post(post_url, headers=headers, data=payload)

# Print response to check if the request was successful
print(response.text)

Step 5: Select Year Now, we move to select the year in the same manner.

# Extract hidden form fields after the Village selection (VIEWSTATE, VIEWSTATEGENERATOR, EVENTVALIDATION)
viewstate = re.search(r'__VIEWSTATE" value="([^"]+)', response.text).group(1)
viewstategenerator = re.search(r'__VIEWSTATEGENERATOR" value="([^"]+)', response.text).group(1)
eventvalidation = re.search(r'__EVENTVALIDATION" value="([^"]+)', response.text).group(1)

# Define the payload for the POST request to select the Year
payload = {
    'ctl00$ScriptManager1': 'ctl00$ContentPlaceHolder1$updRegionSelector|ctl00$ContentPlaceHolder1$ddlYear',
    'ctl00$SelectRegion$rdPeriod': '1',
    'ctl00$SelectRegion$ddlDistrict': '10',
    'ctl00$SelectRegion$ddlTehsil': '63',
    'ctl00$SelectRegion$ddlVillage': '1E36A812-C218-DD11-8334-000E0CA49FC8',
    'ctl00$SelectRegion$ddlYear': '3',  # Choose Year here (example: 3)
    'ctl00$ddlLang': 'en-US',
    'ctl00$ContentPlaceHolder1$rdPeriod': '1',
    'ctl00$ContentPlaceHolder1$ddlDistrict': '10',
    'ctl00$ContentPlaceHolder1$ddlTehsil': '63',
    'ctl00$ContentPlaceHolder1$ddlVillage': '1E36A812-C218-DD11-8334-000E0CA49FC8',
    'ctl00$ContentPlaceHolder1$ddlYear': '3',
    '__EVENTTARGET': 'ctl00$ContentPlaceHolder1$ddlYear',
    '__EVENTARGUMENT': '',
    '__LASTFOCUS': '',
    '__VIEWSTATE': viewstate,
    '__VIEWSTATEGENERATOR': viewstategenerator,
    '__EVENTVALIDATION': eventvalidation,
    '__ASYNCPOST': 'true'
}

# Send the POST request to select the Year
response = requests.post(post_url, headers=headers, data=payload)

# Print response to check if the request was successful
print(response.text)

Step 6: Send GET Request to the Mutation Page

The first part of code sends a GET request to the mutation page to retrieve necessary information, such as cookies and hidden fields like __VIEWSTATE, __VIEWSTATEGENERATOR, and __EVENTVALIDATION.

import requests
cookie1 = response.cookies.get_dict()
cookie2 = cookie1['dgrLAndrecordPLRS']

# Define the URL
url = "https://jamabandi.punjab.gov.in/Mutation.aspx?itemPID=4"

# Define the headers
headers = {
    "authority": "jamabandi.punjab.gov.in",
    "method": "GET",
    "scheme": "https",
    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
    "accept-encoding": "gzip, deflate, br, zstd",
    "accept-language": "en-GB,en;q=0.9",
    "referer": "https://jamabandi.punjab.gov.in/default.aspx",
    "sec-ch-ua": '"Not A(Brand";v="8", "Chromium";v="132", "Google Chrome";v="132"',
    "sec-ch-ua-mobile": "?0",
    "sec-ch-ua-platform": '"macOS"',
    "sec-fetch-dest": "document",
    "sec-fetch-mode": "navigate",
    "sec-fetch-site": "same-origin",
    "sec-fetch-user": "?1",
    "upgrade-insecure-requests": "1",
    "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36"
}

# Define the cookies
cookies = {
    "dgrLAndrecordPLRS": cookie2
}

# Make the GET request
response = requests.get(url, headers=headers, cookies=cookies)

# Check the response status
if response.status_code == 200:
    print("Request was successful!")
    print(response.text)  # Print the HTML content
else:
    print(f"Failed to fetch the page. Status Code: {response.status_code}")
Step 2: Extract Hidden Fields from the HTML

After successfully sending the GET request, you use BeautifulSoup to parse the HTML response and extract necessary hidden fields (__VIEWSTATE, __VIEWSTATEGENERATOR, and __EVENTVALIDATION) that will be required in the subsequent POST request.This is where it is not working.

from bs4 import BeautifulSoup

# Initialize a session to persist cookies
session = requests.Session()

# Step 1: Send a GET request to the page to get necessary hidden fields
url = "https://jamabandi.punjab.gov.in/Mutation.aspx?itemPID=4"
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36"
}
headers.update({
    "Referer": url,
    "X-Requested-With": "XMLHttpRequest"
})

# Send GET request to fetch the page content
response = session.get(url, headers=headers)

# Parse HTML to extract __VIEWSTATE, __VIEWSTATEGENERATOR, and __EVENTVALIDATION
soup = BeautifulSoup(response.text, "html.parser")
viewstate = soup.find("input", {"name": "__VIEWSTATE"})["value"]
viewstategenerator = soup.find("input", {"name": "__VIEWSTATEGENERATOR"})["value"]
eventvalidation = soup.find("input", {"name": "__EVENTVALIDATION"})["value"]

# Debug: Print extracted values
print("VIEWSTATE:", viewstate)
print("VIEWSTATEGENERATOR:", viewstategenerator)
print("EVENTVALIDATION:", eventvalidation)
Step 3: Prepare Payload for POST Request

Next, you define the payload for the POST request. This includes the hidden fields you extracted (__VIEWSTATE, __VIEWSTATEGENERATOR, and __EVENTVALIDATION), as well as any form fields required by the mutation form.

# Step 8: Prepare the payload for the POST request
payload = {
    "ctl00$ScriptManager1": "ctl00$ContentPlaceHolder1$updMutationPanel|ctl00$ContentPlaceHolder1$ddlMutationNumber",
    "__EVENTTARGET": "ctl00$ContentPlaceHolder1$ddlMutationNumber",
    "__EVENTARGUMENT": "",
    "__LASTFOCUS": "",
    "__VIEWSTATE": viewstate,
    "__VIEWSTATEGENERATOR": viewstategenerator,
    "__SCROLLPOSITIONX": "0",
    "__SCROLLPOSITIONY": "0",
    "__EVENTVALIDATION": eventvalidation,
    "ctl00$SelectRegion$rdPeriod": "1",
    "ctl00$SelectRegion$ddlDistrict": "",
    "ctl00$SelectRegion$ddlTehsil": "",
    "ctl00$SelectRegion$ddlVillage": "",
    "ctl00$SelectRegion$ddlYear": "",
    "ctl00$ddlLang": "en-US",
    "ctl00$ContentPlaceHolder1$ddlMutationNumber": "5672",
    "__ASYNCPOST": "true"
}

Step 4: Send POST Request to Select Mutation

After preparing the payload, you send the POST request to select the mutation number. This step may trigger CAPTCHA handling if a mutation is selected.

# Step 3: Send the POST request
post_url = "https://jamabandi.punjab.gov.in/Mutation.aspx?itemPID=4"
response = session.post(post_url, headers=headers, data=payload)

# Print response to verify
print(response.text)

Solution

  • I did a little research and I managed to reproduce the whole chain of requests to reach the captcha page. My script completely replicates what you would do in a browser. The point is that you need to use one session for all requests. After you have made the first request - session cookies will be set, which will be used in all subsequent requests. Nothing will work without them!

    Here is a script that you can, run and test. It will load a response with captcha input.

    import re  
      
    import requests  
    from bs4 import BeautifulSoup  
      
      
    def parse_dynamic_form_data_from_get_response(response) -> dict[str, str]:  
        soup = BeautifulSoup(response.text, 'html.parser')  
        input_names = (  
            '__VIEWSTATE',  
            '__VIEWSTATEGENERATOR',  
            '__EVENTVALIDATION',  
        )  
        dynamic_form_data = {}  
        for input_name in input_names:  
            value = soup.find('input', attrs={'name': input_name})['value']  
            dynamic_form_data[input_name] = value  
        return dynamic_form_data  
      
      
    def parse_dynamic_form_data_from_post_response(response) -> dict[str, str]:  
        text = getattr(response, 'text', response)  
        keys = (  
            '__VIEWSTATE',  
            '__VIEWSTATEGENERATOR',  
            '__EVENTVALIDATION',  
        )  
        dynamic_form_data = {}  
        for key in keys:  
            value = re.search(rf'{key}\|(.+?)\|', text).group(1)  
            dynamic_form_data[key] = value  
        return dynamic_form_data  
      
      
    class Scraper:  
        BASE_URL = 'https://jamabandi.punjab.gov.in'  
        BASE_HEADERS = {  
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:137.0) Gecko/20100101 Firefox/137.0',  
        }  
      
        def __init__(self) -> None:  
            self.session = requests.Session()  
      
        def load_init_page(self):  
            url = f'{self.BASE_URL}/default.aspx'  
            return self.session.get(url=url)  
      
        def select_district(self, dynamic_form_data):  
            form_data = {  
                'ctl00$ScriptManager1': 'ctl00$ContentPlaceHolder1$updRegionSelector|ctl00$ContentPlaceHolder1$ddlDistrict',  
                'ctl00$SelectRegion$rdPeriod': '1',  
                'ctl00$SelectRegion$ddlDistrict': '',  
                'ctl00$SelectRegion$ddlTehsil': '',  
                'ctl00$SelectRegion$ddlVillage': '',  
                'ctl00$SelectRegion$ddlYear': '',  
                'ctl00$ddlLang': 'en-US',  
                'ctl00$ContentPlaceHolder1$rdPeriod': '1',  
                'ctl00$ContentPlaceHolder1$ddlDistrict': '10',  
                '__EVENTTARGET': 'ctl00$ContentPlaceHolder1$ddlDistrict',  
                '__EVENTARGUMENT': '',  
                '__LASTFOCUS': '',  
                '__ASYNCPOST': 'true',  
                **dynamic_form_data,  
            }  
            url = f'{self.BASE_URL}/default.aspx'  
            headers = self.BASE_HEADERS | {  
                'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8',  
            }  
            return self.session.post(url=url, data=form_data, headers=headers)  
      
        def select_tehsil(self, dynamic_form_data):  
            form_data = {  
                'ctl00$ScriptManager1': 'ctl00$ContentPlaceHolder1$updRegionSelector|ctl00$ContentPlaceHolder1$ddlTehsil',  
                'ctl00$SelectRegion$rdPeriod': '1',  
                'ctl00$SelectRegion$ddlDistrict': '',  
                'ctl00$SelectRegion$ddlTehsil': '',  
                'ctl00$SelectRegion$ddlVillage': '',  
                'ctl00$SelectRegion$ddlYear': '',  
                'ctl00$ddlLang': 'en-US',  
                'ctl00$ContentPlaceHolder1$rdPeriod': '1',  
                'ctl00$ContentPlaceHolder1$ddlDistrict': '10',  
                'ctl00$ContentPlaceHolder1$ddlTehsil': '62',  
                '__EVENTTARGET': 'ctl00$ContentPlaceHolder1$ddlTehsil',  
                '__EVENTARGUMENT': '',  
                '__LASTFOCUS': '',  
                '__ASYNCPOST': 'true',  
                **dynamic_form_data,  
            }  
            url = f'{self.BASE_URL}/default.aspx'  
            headers = self.BASE_HEADERS | {  
                'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8',  
            }  
            return self.session.post(url=url, data=form_data, headers=headers)  
      
        def select_village(self, dynamic_form_data):  
            form_data = {  
                'ctl00$ScriptManager1': 'ctl00$ContentPlaceHolder1$updRegionSelector|ctl00$ContentPlaceHolder1$ddlVillage',  
                'ctl00$SelectRegion$rdPeriod': '1',  
                'ctl00$SelectRegion$ddlDistrict': '',  
                'ctl00$SelectRegion$ddlTehsil': '',  
                'ctl00$SelectRegion$ddlVillage': '',  
                'ctl00$SelectRegion$ddlYear': '',  
                'ctl00$ddlLang': 'en-US',  
                'ctl00$ContentPlaceHolder1$rdPeriod': '1',  
                'ctl00$ContentPlaceHolder1$ddlDistrict': '10',  
                'ctl00$ContentPlaceHolder1$ddlTehsil': '62',  
                'ctl00$ContentPlaceHolder1$ddlVillage': 'E645A091-F0D4-DF11-8826-001E0B4643B6',  
                '__EVENTTARGET': 'ctl00$ContentPlaceHolder1$ddlVillage',  
                '__EVENTARGUMENT': '',  
                '__LASTFOCUS': '',  
                '__ASYNCPOST': 'true',  
                **dynamic_form_data,  
            }  
            url = f'{self.BASE_URL}/default.aspx'  
            headers = self.BASE_HEADERS | {  
                'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8',  
            }  
            return self.session.post(url=url, data=form_data, headers=headers)  
      
        def select_year(self, dynamic_form_data):  
            form_data ={  
                'ctl00$ScriptManager1': 'ctl00$ContentPlaceHolder1$updRegionSelector|ctl00$ContentPlaceHolder1$ddlYear',  
                'ctl00$SelectRegion$rdPeriod': '1',  
                'ctl00$SelectRegion$ddlDistrict': '',  
                'ctl00$SelectRegion$ddlTehsil': '',  
                'ctl00$SelectRegion$ddlVillage': '',  
                'ctl00$SelectRegion$ddlYear': '',  
                'ctl00$ddlLang': 'en-US',  
                'ctl00$ContentPlaceHolder1$rdPeriod': '1',  
                'ctl00$ContentPlaceHolder1$ddlDistrict': '10',  
                'ctl00$ContentPlaceHolder1$ddlTehsil': '62',  
                'ctl00$ContentPlaceHolder1$ddlVillage': 'E645A091-F0D4-DF11-8826-001E0B4643B6',  
                'ctl00$ContentPlaceHolder1$ddlYear': '4',  
                '__EVENTTARGET': 'ctl00$ContentPlaceHolder1$ddlYear',  
                '__EVENTARGUMENT': '',  
                '__LASTFOCUS': '',  
                '__ASYNCPOST': 'true',  
                **dynamic_form_data,  
            }  
            url = f'{self.BASE_URL}/default.aspx'  
            headers = self.BASE_HEADERS | {  
                'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8',  
            }  
            return self.session.post(url=url, data=form_data, headers=headers)  
      
        def set_region(self, dynamic_form_data):  
            form_data = {  
                'ctl00$ScriptManager1': 'ctl00$ContentPlaceHolder1$updRegionSelector|ctl00$ContentPlaceHolder1$btnSearch',  
                'ctl00$SelectRegion$rdPeriod': '1',  
                'ctl00$SelectRegion$ddlDistrict': '',  
                'ctl00$SelectRegion$ddlTehsil': '',  
                'ctl00$SelectRegion$ddlVillage': '',  
                'ctl00$SelectRegion$ddlYear': '',  
                'ctl00$ddlLang': 'en-US',  
                'ctl00$ContentPlaceHolder1$rdPeriod': '1',  
                'ctl00$ContentPlaceHolder1$ddlDistrict': '10',  
                'ctl00$ContentPlaceHolder1$ddlTehsil': '62',  
                'ctl00$ContentPlaceHolder1$ddlVillage': 'E645A091-F0D4-DF11-8826-001E0B4643B6',  
                'ctl00$ContentPlaceHolder1$ddlYear': '4',  
                '__EVENTTARGET': '',  
                '__EVENTARGUMENT': '',  
                '__LASTFOCUS': '',  
                '__ASYNCPOST': 'true',  
                'ctl00$ContentPlaceHolder1$btnSearch': 'Set Region',  
                **dynamic_form_data,  
            }  
            url = f'{self.BASE_URL}/default.aspx'  
            headers = self.BASE_HEADERS | {  
                'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8',  
            }  
            return self.session.post(url=url, data=form_data, headers=headers)  
      
        def load_mutation_page(self):  
            url = f'{self.BASE_URL}/Mutation.aspx?itemPID=4'  
            return self.session.get(url=url, headers=self.BASE_HEADERS)  
      
        def select_mutation(self, dynamic_form_data):  
            form_data = {  
                'ctl00$ScriptManager1': 'ctl00$ContentPlaceHolder1$updRegionSelector|ctl00$ContentPlaceHolder1$ddlVillage',  
                'ctl00$SelectRegion$rdPeriod': '1',  
                'ctl00$SelectRegion$ddlDistrict': '',  
                'ctl00$SelectRegion$ddlTehsil': '',  
                'ctl00$SelectRegion$ddlVillage': '',  
                'ctl00$SelectRegion$ddlYear': '',  
                'ctl00$ddlLang': 'en-US',  
                'ctl00$ContentPlaceHolder1$ddlMutationNumber': '10429',  
                '__EVENTTARGET': 'ctl00$ContentPlaceHolder1$ddlMutationNumber',  
                '__EVENTARGUMENT': '',  
                '__LASTFOCUS': '',  
                '__ASYNCPOST': 'true',  
                **dynamic_form_data,  
            }  
            url = f'{self.BASE_URL}/Mutation.aspx?itemPID=4'  
            headers = self.BASE_HEADERS | {  
                'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8',  
            }  
            return self.session.post(url=url, data=form_data, headers=headers)  
      
        def load_captcha_response(self):  
            handlers = (  
                self.select_district,  
                self.select_tehsil,  
                self.select_village,  
                self.select_year,  
                self.set_region,  
            )  
            response = self.load_init_page()  
            data = parse_dynamic_form_data_from_get_response(response=response)  
      
            for handler in handlers:  
                response = handler(dynamic_form_data=data)  
                data = parse_dynamic_form_data_from_post_response(response)  
      
            response = self.load_mutation_page()  
            data = parse_dynamic_form_data_from_get_response(response=response)  
            response = self.select_mutation(dynamic_form_data=data)  
            return response  
      
      
    if __name__ == '__main__':  
        scraper = Scraper()  
        result = scraper.load_captcha_response()  
        print(result.text)
    

    As you can see - one session is used for all requests with cookies set. I have tried to name each method so that you can understand what exactly happens there. If you need additional explanations of the code - write about it, I will expand my answer.

    Here is the HTML part of the response, where you can see that the mutation number was successfully selected and the captcha was loaded:

    <div class="m-t-sm">
        <div class="row form-group">
            <div class="col-md-4 text-right m-t-sm device-custom-label">
                <span id="ContentPlaceHolder1_lblMutationNumber" class="label-lg-font-size">Mutation Number<span class="text-danger">*</span>                                                       </span>
            </div>
            <div class="col-md-4">
              
                <select name="ctl00$ContentPlaceHolder1$ddlMutationNumber" onchange="javascript:setTimeout(&#39;__doPostBack(\&#39;ctl00$ContentPlaceHolder1$ddlMutationNumber\&#39;,\&#39;\&#39;)&#39;, 0)" id="ContentPlaceHolder1_ddlMutationNumber" class="form-control wide">
    <option value="">Please Select</option>
    <option value="10245-">10245-</option>
    <option value="10284-">10284-</option>
    <option value="10307-">10307-</option>
    <option value="10307--">10307--</option>
    <option value="10380-">10380-</option>
    <option value="10382-">10382-</option>
    <option value="10383-">10383-</option>
    <option value="10384-">10384-</option>
    <option value="10386-">10386-</option>
    <option value="10387--">10387--</option>
    <option value="10388-">10388-</option>
    <option value="10389-">10389-</option>
    <option value="10390-">10390-</option>
    <option value="10391-">10391-</option>
    <option value="10392---">10392---</option>
    <option value="10392----">10392----</option>
    <option value="10393--">10393--</option>
    <option value="10394-">10394-</option>
    <option value="10395-">10395-</option>
    <option value="10396-">10396-</option>
    <option value="10397">10397</option>
    <option value="10398">10398</option>
    <option value="10399">10399</option>
    <option value="10400">10400</option>
    <option value="10401">10401</option>
    <option value="10402">10402</option>
    <option value="10403">10403</option>
    <option value="10404">10404</option>
    <option value="10405">10405</option>
    <option value="10406">10406</option>
    <option value="10408">10408</option>
    <option value="10409">10409</option>
    <option value="10410">10410</option>
    <option value="10411">10411</option>
    <option value="10412">10412</option>
    <option value="10413">10413</option>
    <option value="10414">10414</option>
    <option value="10415">10415</option>
    <option value="10416">10416</option>
    <option value="10417">10417</option>
    <option value="10418">10418</option>
    <option value="10418-">10418-</option>
    <option value="10419">10419</option>
    <option value="10420">10420</option>
    <option value="10421">10421</option>
    <option value="10422">10422</option>
    <option value="10423">10423</option>
    <option value="10424">10424</option>
    <option value="10425">10425</option>
    <option value="10425-">10425-</option>
    <option value="10426">10426</option>
    <option value="10427">10427</option>
    <option value="10428">10428</option>
    <option selected="selected" value="10429">10429</option>
    <option value="10430">10430</option>
    <option value="10431">10431</option>
    <option value="10432">10432</option>
    <option value="10433">10433</option>
    <option value="10435">10435</option>
    <option value="10436">10436</option>
    <option value="10437">10437</option>
    <option value="10438">10438</option>
    <option value="10440">10440</option>
    <option value="10441">10441</option>
    <option value="10442">10442</option>
    <option value="10442-">10442-</option>
    <option value="10442--">10442--</option>
    <option value="10443">10443</option>
    <option value="10444">10444</option>
    <option value="10445">10445</option>
    <option value="10446">10446</option>
    <option value="10447">10447</option>
    <option value="10447-">10447-</option>
    <option value="10448">10448</option>
    <option value="10449">10449</option>
    <option value="10450">10450</option>
    <option value="10451">10451</option>
    <option value="10452">10452</option>
    <option value="10453">10453</option>
    <option value="10454">10454</option>
    <option value="10455">10455</option>
    <option value="10456">10456</option>
    <option value="10457">10457</option>
    <option value="10457-">10457-</option>
    <option value="10457--">10457--</option>
    <option value="10458">10458</option>
    <option value="10459">10459</option>
    <option value="10460">10460</option>
    <option value="10461">10461</option>
    <option value="10462">10462</option>
    <option value="10463">10463</option>
    <option value="10464">10464</option>
    <option value="10465">10465</option>
    <option value="10466">10466</option>
    <option value="10467">10467</option>
    <option value="10468">10468</option>
    <option value="10469">10469</option>
    <option value="10470">10470</option>
    <option value="10471">10471</option>
    <option value="10472">10472</option>
    <option value="10473">10473</option>
    <option value="10474">10474</option>
    <option value="10475">10475</option>
    <option value="10476">10476</option>
    <option value="10477">10477</option>
    <option value="10478">10478</option>
    <option value="10479">10479</option>
    <option value="10480">10480</option>
    <option value="10481">10481</option>
    <option value="10482">10482</option>
    <option value="10483">10483</option>
    <option value="10484">10484</option>
    <option value="10485">10485</option>
    <option value="10485-">10485-</option>
    <option value="10485--">10485--</option>
    <option value="10486">10486</option>
    <option value="10487">10487</option>
    <option value="10488">10488</option>
    <option value="10489">10489</option>
    <option value="10490">10490</option>
    <option value="10491">10491</option>
    <option value="10492">10492</option>
    <option value="10493">10493</option>
    <option value="10494">10494</option>
    <option value="10495">10495</option>
    <option value="10496">10496</option>
    <option value="10497">10497</option>
    <option value="10498">10498</option>
    <option value="10499">10499</option>
    <option value="10500">10500</option>
    <option value="10501">10501</option>
    <option value="10502">10502</option>
    <option value="10503">10503</option>
    <option value="10504">10504</option>
    <option value="10505">10505</option>
    <option value="10506">10506</option>
    <option value="10507">10507</option>
    <option value="10508">10508</option>
    <option value="10509">10509</option>
    <option value="10510">10510</option>
    <option value="10511">10511</option>
    <option value="10512">10512</option>
    <option value="10513">10513</option>
    <option value="10514">10514</option>
    <option value="10515">10515</option>
    <option value="10516-">10516-</option>
    <option value="10517">10517</option>
    <option value="10518">10518</option>
    <option value="10519">10519</option>
    <option value="10520">10520</option>
    <option value="10521">10521</option>
    <option value="10522">10522</option>
    <option value="10523">10523</option>
    <option value="10524">10524</option>
    <option value="10525">10525</option>
    <option value="10526">10526</option>
    <option value="10527">10527</option>
    <option value="10528">10528</option>
    <option value="10528-">10528-</option>
    <option value="10529">10529</option>
    <option value="10530">10530</option>
    <option value="10531">10531</option>
    <option value="10532">10532</option>
    <option value="10533">10533</option>
    <option value="10533-">10533-</option>
    <option value="10534">10534</option>
    <option value="10535">10535</option>
    <option value="10536">10536</option>
    <option value="10537">10537</option>
    <option value="10538">10538</option>
    <option value="10539">10539</option>
    <option value="10540">10540</option>
    <option value="10541">10541</option>
    <option value="10541-">10541-</option>
    <option value="10542">10542</option>
    <option value="10543">10543</option>
    <option value="10544">10544</option>
    <option value="10545">10545</option>
    <option value="10546">10546</option>
    <option value="10547">10547</option>
    <option value="10548">10548</option>
    <option value="10549">10549</option>
    <option value="10550">10550</option>
    <option value="10551">10551</option>
    <option value="10552">10552</option>
    <option value="10553">10553</option>
    <option value="10554">10554</option>
    <option value="10555">10555</option>
    <option value="10556">10556</option>
    <option value="10557">10557</option>
    <option value="10558">10558</option>
    <option value="10559">10559</option>
    <option value="10560">10560</option>
    <option value="10561">10561</option>
    <option value="10562">10562</option>
    <option value="10563">10563</option>
    <option value="10564">10564</option>
    <option value="10565">10565</option>
    <option value="10566">10566</option>
    <option value="10567">10567</option>
    <option value="10568">10568</option>
    <option value="10569">10569</option>
    <option value="10570">10570</option>
    <option value="10571">10571</option>
    <option value="10572">10572</option>
    <option value="10573">10573</option>
    <option value="10574">10574</option>
    <option value="10574-">10574-</option>
    <option value="10575">10575</option>
    <option value="10576">10576</option>
    <option value="10577">10577</option>
    <option value="10578">10578</option>
    <option value="10579">10579</option>
    <option value="10580">10580</option>
    <option value="10581">10581</option>
    <option value="10582">10582</option>
    <option value="10583">10583</option>
    <option value="10584">10584</option>
    <option value="10585">10585</option>
    <option value="10586">10586</option>
    <option value="10587">10587</option>
    <option value="10588">10588</option>
    <option value="10589">10589</option>
    <option value="10590">10590</option>
    <option value="10591">10591</option>
    <option value="10591-">10591-</option>
    <option value="10592">10592</option>
    <option value="10592-">10592-</option>
    <option value="10592--">10592--</option>
    <option value="10593">10593</option>
    <option value="10594">10594</option>
    <option value="10595">10595</option>
    <option value="10596">10596</option>
    <option value="10597">10597</option>
    <option value="10598">10598</option>
    <option value="10599">10599</option>
    <option value="10600">10600</option>
    <option value="10601">10601</option>
    <option value="10602">10602</option>
    <option value="10603">10603</option>
    <option value="10604">10604</option>
    <option value="10605">10605</option>
    <option value="10606">10606</option>
    <option value="10607">10607</option>
    <option value="10608">10608</option>
    <option value="10609">10609</option>
    <option value="10609-">10609-</option>
    <option value="10610">10610</option>
    <option value="10611">10611</option>
    <option value="10612">10612</option>
    <option value="10613">10613</option>
    <option value="10614">10614</option>
    <option value="10615">10615</option>
    <option value="10616">10616</option>
    <option value="10617">10617</option>
    <option value="10618">10618</option>
    <option value="10619">10619</option>
    <option value="10620-">10620-</option>
    <option value="10621">10621</option>
    <option value="10622">10622</option>
    <option value="10623">10623</option>
    <option value="10624">10624</option>
    <option value="10625">10625</option>
    <option value="10626">10626</option>
    <option value="10627">10627</option>
    <option value="10628">10628</option>
    <option value="10629">10629</option>
    <option value="10630">10630</option>
    <option value="10631">10631</option>
    <option value="10632">10632</option>
    <option value="10633">10633</option>
    <option value="10634">10634</option>
    <option value="10635">10635</option>
    <option value="10636">10636</option>
    <option value="10637">10637</option>
    <option value="10638">10638</option>
    <option value="10639">10639</option>
    <option value="10640">10640</option>
    <option value="10641">10641</option>
    <option value="10642">10642</option>
    <option value="10643">10643</option>
    <option value="10644">10644</option>
    <option value="10645">10645</option>
    <option value="10646">10646</option>
    
    </select>
            </div>
            <div class="col-md-4 text-left m-t-sm">
                <span id="ContentPlaceHolder1_RFVMutationNumber" class="text-danger label-lg-font-size" style="display:none;">Please select any mutation number</span>
            </div>
        </div>
    
    
        <div id="ContentPlaceHolder1_dvbbtnViewReportMutationNumber" class="text-center">
          <div class=" form-group  text-center">
             <div class="col-md-4 text-right m-t-sm">
                 </div>
             <div class="col-md-8 text-left m-t-sm">
                <img id="ContentPlaceHolder1_imgCaptcha" src="GenerateCaptcha.aspx?638773135804886029" />
               <input type="submit" name="ctl00$ContentPlaceHolder1$btnRefresh" value="Refresh" id="ContentPlaceHolder1_btnRefresh" class="btn btn-primary submit-custom-btn" />
                 </div>
        </div>
          <div class=" form-group  text-left">
                <div class="col-md-4 text-right m-t-sm device-custom-label">
                    <label class="control-label" for="captcha">
                        <span id="ContentPlaceHolder1_Label1" class="custom-label">Enter Captcha</span>
                        </label>
                 </div>
                    <div class="col-md-4">
               <input name="ctl00$ContentPlaceHolder1$txtCaptcha" type="text" id="ContentPlaceHolder1_txtCaptcha" class="form-control" />
                         </div>
                    <div class="col-md-4">
                <span id="ContentPlaceHolder1_reqCaptcha" class="text-danger" style="display:none;">Required.</span>
                        </div>
        </div>
            <div class="col-md-12 m-t-sm">
                <input type="submit" name="ctl00$ContentPlaceHolder1$btnViewReportMutationNumber" value="View Report" onclick="javascript:WebForm_DoPostBackWithOptions(new WebForm_PostBackOptions(&quot;ctl00$ContentPlaceHolder1$btnViewReportMutationNumber&quot;, &quot;&quot;, true, &quot;MutationNumber&quot;, &quot;&quot;, false, false))" id="ContentPlaceHolder1_btnViewReportMutationNumber" class="btn btn-primary submit-custom-btn" ErrorMessage="Please select any mutation number" />
            </div>
        </div>
    </div>
    
    </div>
            </fieldset>