I am trying to scrape the sku and description on this site: https://www.dewalt.com/products/power-tools/
but, it wont scrape the desired elements despite the code being able to run. Does anyone know why? Am I grabbing the wrong elements?
import undetected_chromedriver as uc
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd
import time
from selenium.common.exceptions import NoSuchElementException
options = Options()
driver = uc.Chrome()
website = 'https://www.dewalt.com/products/power-tools/drills'
driver.get(website)
WebDriverWait(driver, 5).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, '#product-list')))
prod_num = []
prod_desc = []
container = driver.find_element(By.CSS_SELECTOR, '#product-list')
for _ in range(4):
driver.execute_script("window.scrollBy(0, 2000);")
time.sleep(2)
skus = container.find_elements(By.CLASS_NAME, 'coh-inline-element product-sku')
descriptions = container.find_elements(By.CLASS_NAME, 'coh-link subtitle card-link product-title')
for sku in skus:
prod_num.append(sku.text)
for desc in descriptions:
prod_desc.append(desc.text)
driver.quit()
print(len(prod_num))
print(len(prod_desc))
# Create a DataFrame from the scraped data
df = pd.DataFrame({'code': prod_num, 'description': prod_desc})
# Save the DataFrame to a CSV file
df.to_csv('dewtest1.csv', index=False)
print(df)
The content you are after is static, so you should go for the requests module instead. Here is how you can achieve this using the requests module:
import requests
import pandas as pd
from bs4 import BeautifulSoup
link = 'https://www.dewalt.com/products/power-tools/drills'
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
}
res = requests.get(link,headers=headers)
soup = BeautifulSoup(res.text,"html.parser")
df = pd.DataFrame(columns= ['sku','desc'])
for item in soup.select(".coh-column > article[role='article']"):
sku = item.select_one(".product-sku").get_text(strip=True)
desc = item.select_one(".product-title").get_text(strip=True)
df = pd.concat([df, pd.DataFrame({'sku': [sku], 'desc': [desc]})], ignore_index=True)
print(sku,desc)
df.to_csv("dewalt.csv",index=False)
Your existing approach should also work if you consider the following changes within the selector:
product_list = WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".coh-column > article[role='article']")))
prod_num = []
prod_desc = []
for container in product_list:
sku = container.find_element(By.CSS_SELECTOR, '.product-sku').text
description = container.find_element(By.CSS_SELECTOR, '.product-title').text
prod_num.append(sku)
prod_desc.append(description)
df = pd.DataFrame({'code': prod_num, 'description': prod_desc})
df.to_csv('dewtest1.csv', index=False)