I have a page that I want to scrape but it's not always available to be scraped and I want the code to be running 24/7 So I made this
import requests
import response
from bs4 import BeautifulSoup
import re
import time
import io
import ssl
import os
import getpass
import urllib3
from time import sleep
from lxml.etree import tostring
import subprocess
import smtplib
import sys
import lxml
print('Enter the National-ID : ')
NID = input()
burp0_url = "I removed the URL"
burp0_cookies = {"I removed the cookies"}
burp0_headers = {"I removed the headers"}
burp0_data = "I removed the data"
r = requests.post(burp0_url, headers=burp0_headers, cookies=burp0_cookies, data=burp0_data)
soup=BeautifulSoup(r.content,'html.parser')
script = soup.find('script')
address = soup.find("input", {"name": "ctl00$ContentPlaceHolder1$EmpAdrs"})
DOB = soup.find("input", {"name": "ctl00$ContentPlaceHolder1$EmpBD"})
gender = soup.find('option', {'selected':'selected'})
status = soup.find('option', {'value':'206510000'})
for x in soup.findAll('table', {'class':'auto-style1'}):
for no in soup.findAll('input', {'name':'ctl00$ContentPlaceHolder1$EmpNatNo'}):
for name in soup.findAll('input', {'name':'ctl00$ContentPlaceHolder1$EmpName'}):
with io.open('x.txt', 'a', encoding="utf-8") as f:
ary = [name , no, address, DOB]
f.write (ary[0]["value"]+ ",")
f.write (ary[1]["value"]+ ",")
f.write (ary[2]["value"]+ ",")
f.write (ary[3]["value"]+ ",")
f.write (gender.text + ",")
f.write (status.text + "\n")
print (script)
My issue is when the program doesn't find (value) or whatever it ends the program like this
Traceback (most recent call last):
File "C:\Users\cr\Desktop\jms\CORVID\Coding\py\1oen.py", line 35, in <module>
f.write (ary[0]["value"]+ ",")
File "C:\python39\lib\site-packages\bs4\element.py", line 1406, in __getitem__
return self.attrs[key]
KeyError: 'value'
I want it to keep executing it I tried to do the try-except thing But I couldn't and it didn't work And I think Try-except doesn't show Bugs when I have to debug them. So can someone please help me?
Print your fields in a loop so you can put try/except
around each of them.
Use this:
for x in soup.findAll('table', {'class':'auto-style1'}):
for no in soup.findAll('input', {'name':'ctl00$ContentPlaceHolder1$EmpNatNo'}):
for name in soup.findAll('input', {'name':'ctl00$ContentPlaceHolder1$EmpName'}):
with io.open('x.txt', 'a', encoding="utf-8") as f:
ary = [name , no, address, DOB]
for field in ary:
try:
val = field["value"]
except:
val = ""
f.write(val + ",")
f.write (gender.text + ",")
f.write (status.text + "\n")
print(script)
in place of this:
for x in soup.findAll('table', {'class':'auto-style1'}):
for no in soup.findAll('input', {'name':'ctl00$ContentPlaceHolder1$EmpNatNo'}):
for name in soup.findAll('input', {'name':'ctl00$ContentPlaceHolder1$EmpName'}):
with io.open('x.txt', 'a', encoding="utf-8") as f:
ary = [name , no, address, DOB]
f.write (ary[0]["value"]+ ",")
f.write (ary[1]["value"]+ ",")
f.write (ary[2]["value"]+ ",")
f.write (ary[3]["value"]+ ",")
f.write (gender.text + ",")
f.write (status.text + "\n")
print (script)