Как скрести eBay в 2024 году: Руководство для начинающих

import re
import csv
import time

import requests
from bs4 import BeautifulSoup

proxies = {
    "http": "http://username:[email protected]:6060",
    "https": "http://username:[email protected]:6060",
}

def get_product_information(product_url) -> dict:
    r = requests.get(product_url, proxies=proxies)
    soup = BeautifulSoup(r.text, features="html.parser")

    product_title = soup.find("h1", {"class": "x-item-title__mainTitle"}).text
    product_price = soup.find("div", {"class": "x-price-primary"}).text.split(" ")[-1]
    currency = soup.find("div", {"class": "x-price-primary"}).text.split(" ")[0]

    # locate the element that holds quanity number of product
    quantity_available = soup.find("div", {"class":"x-quantity__availability"})
    if quantity_available is not None:
        # Using regex check if we can locate the strings that holds this number
        regex_pattern = r"\d+\savailable"
        if re.search(regex_pattern, quantity_available.text) is not None:
            quantity_available = re.search(regex_pattern, quantity_available.text).group()
            # After string is located we extract the number by splitting it by space and selecting the first element.
            quantity_available = quantity_available.split(" ")[0]
        else:
            quantity_available = "NA"

    total_reviews = soup.find("span", {"class":"ux-summary__count"})
    if total_reviews is not None:
        total_reviews = total_reviews.text.split(" ")[0]
    else:
        total_reviews = "NA"

    rating = soup.find("span", {"class":"ux-summary__start--rating"})
    if rating is not None:
        rating = rating.text
    else:
        rating = "NA"

    product_info = {
        "product_url": product_url,
        "title": product_title,
        "product_price": product_price,
        "currency": currency,
        "availability": quantity_available,
        "nr_reviews": total_reviews,
        "rating": rating
    }

    return product_info

def save_to_csv(products, csv_file_name="products.csv"):

    # Write the list of dictionaries to a CSV file
    with open(csv_file_name, mode='w', newline='') as csv_file:
        # Create a csv.DictWriter object
        writer = csv.DictWriter(csv_file, fieldnames=products[0].keys())

        # Write the header (keys of the dictionary)
        writer.writeheader()

        # Write the rows (values of the dictionaries)
        writer.writerows(products)

    print(f"Data successfully written to {csv_file_name}")

def main(keyword_to_search: str):
    products = []

    r = requests.get(f"https://www.ebay.com/sch/i.html?_nkw={keyword_to_search}", proxies=proxies)

    soup = BeautifulSoup(r.text, features="html.parser")
    for item in soup.find_all("div", {"class": "s-item__info clearfix"})[2::]:
        item_url = item.find("a").get("href")

        product_info: dict = get_product_information(item_url)
        print(product_info)
        # Adding a 1-second delay between requests to avoid overloading the server and reduce the risk of being blocked
        time.sleep(2)

        products.append(product_info)
    # save data to csv
    save_to_csv(products)


if __name__ == '__main__':
    keywords = "laptop bag"
    main(keywords)

mkdir ebay_scraping
cd ebay_scraping
python -m venv venv
source env/bin/activate # В Windows используйте: venv\Scripts\activate
pip install requests bs4

импорт csv
импорт времени
импортировать запросы
из bs4 import BeautifulSoup

proxies = {
   "http": "http://username:[email protected]:6060",
   "https": "http://username:[email protected]:6060",
}

def main(keyword_to_search: str):
   products = []

   r = requests.get(f"https://www.ebay.com/sch/i.html?_nkw={keyword_to_search}", proxies=proxies)

   soup = BeautifulSoup(r.text, features="html.parser")
   for item in soup.find_all("div", {"class": "s-item__info clearfix"})[2::]:
       item_url = item.find("a").get("href")

       product_info: dict = get_product_information(item_url)
		# Adding a 1-second delay between requests to avoid overloading the server and reduce the risk of being blocked
       time.sleep(1)

       products.append(product_info)
   # save data to csv
   save_to_csv(products)

def get_product_information(product_url) -> dict:
   r = requests.get(product_url, proxies=proxies)
   soup = BeautifulSoup(r.text, features="html.parser")

   product_title = soup.find("h1", {"class": "x-item-title__mainTitle"}).text
   product_price = soup.find("div", {"class": "x-price-primary"}).text.split(" ")[-1]
   currency = soup.find("div", {"class": "x-price-primary"}).text.split(" ")[0]

   # locate the element that holds quanity number of product
   quantity_available = soup.find("div", {"class":"x-quantity__availability"})
   if quantity_available is not None:
       # Using regex check if we can locate the strings that holds this number
       regex_pattern = r"\d+\savailable"
       if re.search(regex_pattern, quantity_available.text) is not None:
           quantity_available = re.search(regex_pattern, quantity_available.text).group()
           # After string is located we extract the number by splitting it by space and selecting the first element.
           quantity_available = quantity_available.split(" ")[0]
       else:
           quantity_available = "NA"

   total_reviews = soup.find("span", {"class":"ux-summary__count"})
   if total_reviews is not None:
       total_reviews = total_reviews.text.split(" ")[0]
   else:
       total_reviews = "NA"

   rating = soup.find("span", {"class":"ux-summary__start--rating"})
   if rating is not None:
       rating = rating.text
   else:
       rating = "NA"

   product_info = {
       "product_url": product_url,
       "title": product_title,
       "product_price": product_price,
       "currency": currency,
       "availability": quantity_available,
       "nr_reviews": total_reviews,
       "rating": rating
   }

   return product_info

def save_to_csv(products, csv_file_name="products.csv"):

   # Write the list of dictionaries to a CSV file
   with open(csv_file_name, mode='w', newline='') as csv_file:
       # Create a csv.DictWriter object
       writer = csv.DictWriter(csv_file, fieldnames=products[0].keys())

       # Write the header (keys of the dictionary)
       writer.writeheader()

       # Write the rows (values of the dictionaries)
       writer.writerows(products)

   print(f"Data successfully written to {csv_file_name}")