I am new to Web Scraping and I am trying to automate retrieving parcel information from a car website. I need to get the seller's phone number from each parsed link, but the number is under the Pop-Up
from bs4 import BeautifulSoup
import requests
page = 0
while page <= 500:
url = "https://auto.ria.com/uk/search/?indexName=auto,order_auto,newauto_search&categories.main.id=1&country.import.usa.not=-1&price.USD.gte=1111&price.currency=1&abroad.not=0&custom.not=1&page=" + str(page)
request = requests.get(url)
soup = BeautifulSoup(request.text, "html.parser")
cars = soup.find_all("a",class_ = "m-link-ticket")
if cars != []:
for links in cars:
print(links.get('href'))
page += 1
else:
break
I checked many videos, but i don't understand how to do it
Try:
import requests
import pandas as pd
from bs4 import BeautifulSoup
def get_phone(id_):
url1 = f"https://auto.ria.com/demo/bu/mainPage/rotator/item/{id_}?type=bu&langId=4"
url2 = "https://auto.ria.com/users/phones/{id_}?hash={userSecureHash}"
data = requests.get(url1).json()
userSecureHash = data["userSecure"]["hash"]
try:
phone = requests.get(url2.format(id_=id_, userSecureHash=userSecureHash)).json()
except:
return data, []
out = []
for p in phone["phones"]:
out.append(p["phoneFormatted"])
return data, out
url = "https://auto.ria.com/uk/search/?indexName=auto,order_auto,newauto_search&categories.main.id=1&country.import.usa.not=-1&price.USD.gte=1111&price.currency=1&abroad.not=0&custom.not=1&page=0"
soup = BeautifulSoup(requests.get(url).content, "html.parser")
all_data = []
for a in soup.select("a.address"):
id_ = a["href"].split("_")[-1].split(".")[0]
data, phones = get_phone(id_)
if not data:
continue
data.pop("userSecure")
all_data.append({**data, "phones": phones})
df = pd.DataFrame(all_data)
print(df)
Prints:
isAutoBuy categoryId autoId userId markaId modelId bodyId stateId marka model year yearLetter yearRaceSlash version city cityLocative photoBig photoMedium photoSmall photoRiaSmall photoPath link race currency currencyId priceFromText type typeName USD EUR UAH level levelExpire topType phones
0 False 1 34988974 12860231 24 1945 5 5 Ford Edge 2019 р. / Львові https://cdn4.riastatic.com/photosnew/auto/photo/ford_edge__509055949f.jpg https://cdn4.riastatic.com/photosnew/auto/photo/ford_edge__509055949m.jpg https://cdn4.riastatic.com/photosnew/auto/photo/ford_edge__509055949b.jpg https://cdn4.riastatic.com/photosnew/auto/photo/ford_edge__509055949sx.jpg auto/photo/ford_edge__509055949.jpg /auto_ford_edge_34988974.html 33 тис. км USD 1 bu hide 33 900 31 332 1 261 419 130 2023-08-10T08:42:09.000Z [(098) 439 76 31]
1 False 1 34819008 11247719 84 39690 2 6 Volkswagen Passat 2011 р. / Прилуках https://cdn1.riastatic.com/photosnew/auto/photo/volkswagen_passat__504803346f.jpg https://cdn1.riastatic.com/photosnew/auto/photo/volkswagen_passat__504803346m.jpg https://cdn1.riastatic.com/photosnew/auto/photo/volkswagen_passat__504803346b.jpg https://cdn1.riastatic.com/photosnew/auto/photo/volkswagen_passat__504803346sx.jpg auto/photo/volkswagen_passat__504803346.jpg /auto_volkswagen_passat_34819008.html 300 тис. км USD 1 bu hide 8 300 7 671 308 843 104 2023-08-12T09:18:13.000Z [(050) 012 08 68]
2 False 1 34952066 1911193 48 2622 4 12 Mercedes-Benz B-Class 2018 р. / Одесі https://cdn0.riastatic.com/photosnew/auto/photo/mercedes-benz_b-class__508113635f.jpg https://cdn0.riastatic.com/photosnew/auto/photo/mercedes-benz_b-class__508113635m.jpg https://cdn0.riastatic.com/photosnew/auto/photo/mercedes-benz_b-class__508113635b.jpg https://cdn0.riastatic.com/photosnew/auto/photo/mercedes-benz_b-class__508113635sx.jpg auto/photo/mercedes-benz_b-class__508113635.jpg /auto_mercedes-benz_b-class_34952066.html 7 тис. км USD 1 bu hide 33 900 31 332 1 261 419 82 2023-08-11T05:52:06.000Z [(067) 277 03 03]
3 False 1 34894081 12336674 9 96 5 12 BMW X5 2023 р. / Одесі https://cdn2.riastatic.com/photosnew/auto/photo/bmw_x5__506757257f.jpg https://cdn2.riastatic.com/photosnew/auto/photo/bmw_x5__506757257m.jpg https://cdn2.riastatic.com/photosnew/auto/photo/bmw_x5__506757257b.jpg https://cdn2.riastatic.com/photosnew/auto/photo/bmw_x5__506757257sx.jpg auto/photo/bmw_x5__506757257.jpg /auto_bmw_x5_34894081.html 1 тис. км USD 1 bu hide 112 800 104 255 4 197 288 77 2023-08-11T18:44:43.000Z [(066) 588 88 01]
4 False 1 34748594 12336674 9 96 5 12 BMW X5 2023 р. / Одесі https://cdn4.riastatic.com/photosnew/auto/photo/bmw_x5__504797374f.jpg https://cdn4.riastatic.com/photosnew/auto/photo/bmw_x5__504797374m.jpg https://cdn4.riastatic.com/photosnew/auto/photo/bmw_x5__504797374b.jpg https://cdn4.riastatic.com/photosnew/auto/photo/bmw_x5__504797374sx.jpg auto/photo/bmw_x5__504797374.jpg /auto_bmw_x5_34748594.html 1 тис. км USD 1 bu hide 110 800 102 406 4 122 868 76 2023-08-11T18:47:24.000Z [(066) 588 88 01]
5 False 1 34952912 11530222 9 96 5 9 BMW X5 2012 р. / Рівному https://cdn4.riastatic.com/photosnew/auto/photo/bmw_x5__508132844f.jpg https://cdn4.riastatic.com/photosnew/auto/photo/bmw_x5__508132844m.jpg https://cdn4.riastatic.com/photosnew/auto/photo/bmw_x5__508132844b.jpg https://cdn4.riastatic.com/photosnew/auto/photo/bmw_x5__508132844sx.jpg auto/photo/bmw_x5__508132844.jpg /auto_bmw_x5_34952912.html 267 тис. км USD 1 bu hide 18 750 17 330 697 688 76 2023-08-11T13:52:07.000Z [(068) 293 90 40]
6 False 1 34990951 5031482 9 96 5 25 BMW X5 2015 р. / Чернівцях https://cdn0.riastatic.com/photosnew/auto/photo/bmw_x5__509102045f.jpg https://cdn0.riastatic.com/photosnew/auto/photo/bmw_x5__509102045m.jpg https://cdn0.riastatic.com/photosnew/auto/photo/bmw_x5__509102045b.jpg https://cdn0.riastatic.com/photosnew/auto/photo/bmw_x5__509102045sx.jpg auto/photo/bmw_x5__509102045.jpg /auto_bmw_x5_34990951.html 135 тис. км USD 1 bu hide 40 900 37 802 1 521 889 75 2023-08-12T11:44:07.000Z [(095) 484 63 42]
7 False 1 34932999 10728379 2233 47858 3 11 Tesla Model 3 2020 р. / Дніпрі https://cdn0.riastatic.com/photosnew/auto/photo/tesla_model-3__507614740f.jpg https://cdn0.riastatic.com/photosnew/auto/photo/tesla_model-3__507614740m.jpg https://cdn0.riastatic.com/photosnew/auto/photo/tesla_model-3__507614740b.jpg https://cdn0.riastatic.com/photosnew/auto/photo/tesla_model-3__507614740sx.jpg auto/photo/tesla_model-3__507614740.jpg /auto_tesla_model-3_34932999.html 57 тис. км USD 1 bu hide 29 900 27 635 1 112 579 75 2023-08-10T06:58:07.000Z [(067) 561 21 61]
8 False 1 34809199 5201628 9 96 5 10 BMW X5 2022 р. / Києві https://cdn4.riastatic.com/photosnew/auto/photo/bmw_x5__504561189f.jpg https://cdn4.riastatic.com/photosnew/auto/photo/bmw_x5__504561189m.jpg https://cdn4.riastatic.com/photosnew/auto/photo/bmw_x5__504561189b.jpg https://cdn4.riastatic.com/photosnew/auto/photo/bmw_x5__504561189sx.jpg auto/photo/bmw_x5__504561189.jpg /auto_bmw_x5_34809199.html 1 тис. км USD 1 bu hide 110 800 102 406 4 122 868 75 2023-08-13T11:00:14.000Z [(097) 744 26 95]
9 False 1 34979610 12336674 9 96 5 12 BMW X5 2022 р. / Одесі https://cdn1.riastatic.com/photosnew/auto/photo/bmw_x5__508825896f.jpg https://cdn1.riastatic.com/photosnew/auto/photo/bmw_x5__508825896m.jpg https://cdn1.riastatic.com/photosnew/auto/photo/bmw_x5__508825896b.jpg https://cdn1.riastatic.com/photosnew/auto/photo/bmw_x5__508825896sx.jpg auto/photo/bmw_x5__508825896.jpg /auto_bmw_x5_34979610.html 1 тис. км USD 1 bu hide 89 800 82 997 3 341 458 74 2023-08-11T09:50:50.000Z [(066) 588 88 01]