As title,I could get the values in just first page, but I can't get values page by page with for-in-loop. I've chek my code, but I'm still confused with it. How could I get that values in every page?
# Imports Required
!pip install selenium
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
import requests
from bs4 import BeautifulSoup
browser = webdriver.Chrome(executable_path='./chromedriver.exe')
wait = WebDriverWait(browser,5)
output = list()
for i in range(1,2):
browser.get("https://www.rakuten.com.tw/shop/watsons/product/?l-id=tw_shop_inshop_cat&p={}".format(i))
# Wait Until the product appear
wait.until(EC.presence_of_element_located((By.XPATH,"//div[@class='b-content b-fix-2lines']")))
# Get the products link
product_links = browser.find_elements(By.XPATH,"//div[@class='b-content b-fix-2lines']/b/a")
# Iterate over 'product_links' to get all the 'href' values
for link in (product_links):
print(link.get_attribute('href'))
browser.get(link.get_attribute('href'))
soup = BeautifulSoup(browser.page_source)
products =[]
product = {}
product['商品名稱'] = soup.find('div',class_="b-subarea b-layout-right shop-item ng-scope").h1.text.replace('\n','')
product['價錢'] = soup.find('strong',class_="b-text-xlarge qa-product-actualPrice").text.replace('\n','')
all_data=soup.find_all("div",class_="b-container-child")[2]
main_data=all_data.find_all("span")[-1]
product['購買次數'] = main_data.text
products.append(product)
print(products)
product_links = browser.find_elements(By.XPATH,"//div[@class='b-content b-fix-2lines']/b/a")
# Iterate over 'product_links' to get all the 'href' values
for link in (product_links):
print(link.get_attribute('href'))
browser.get(link.get_attribute('href'))
The problem is that when you do browser.get()
, it invalidates the HTML element referred to by product_links
because it no longer exists in the current page. You should get all of the 'href'
attributes into an array. One way is with a list comprehension:
links = [link.get_attribute('href') for link in product_links]
Now you can loop over the strings in links
to load new pages.
With that said, you should look at the library scrapy
which can do a lot of the heavy lifting for you.