First, I got the total url of all pages. However, When I want to get into each page (page by page),it failed. How could I get into each page?
!pip install selenium
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import urllib as url
from urllib.parse import urlparse
import time
browser = webdriver.Chrome(executable_path='./chromedriver.exe')
wait = WebDriverWait(browser,5)
output = []
for i in range(1,2): # Iterate from page 1 to the last page
browser.get("https://tw.mall.yahoo.com/search/product?p=%E5%B1%88%E8%87%A3%E6%B0%8F&pg={}".format(i))
# Wait Until the product appear
wait.until(EC.presence_of_element_located((By.XPATH,"//ul[@class='gridList']")))
# Get the products
product_links = browser.find_elements(By.XPATH,"//ul[@class='gridList']/li/a")
# Iterate over 'product_links' to get all the 'href' values
for link in (product_links):
print(f"{link.get_attribute('href')}")
output.append([link.get_attribute('href')])
for b in output:
browser.get(b)
output
InvalidArgumentException: Message: invalid argument: 'url' must be a string
(Session info: chrome=96.0.4664.45)
i
is of type int
. You need to convert int
into string
to apply String Interpolation using the str()
function as follows:
format(str(i))
So effective line of code will be:
for i in range(1,2): # Iterate from page 1 to the last page
browser.get("https://tw.mall.yahoo.com/search/product?p=%E5%B1%88%E8%87%A3%E6%B0%8F&pg={}".format(str(i)))