【发布时间】：2021-11-26 19:49:23
【问题描述】：
我认为是缓存或 cookie 问题。当我在一个页面（任何页面）上运行我的代码时，它会给我该页面上的结果（大约 40 个列表）。例如，当我从范围 (1,4) 进行迭代以获取前 3 页时，第一页提供 40 个列表，但我得到第 2、3、4 页的 0 个结果。如果我只在第 2 页上运行我的代码，它会给我 40 个列表。我已经尝试实现重试方法并重试链接 10 次，但它没有工作。我还尝试在我的标题中旋转用户代理以在每个页面上使用不同的浏览器，但仍然无法正常工作。
我的代码如下，如果有任何见解，我将不胜感激。
import requests, pandas, random
from bs4 import BeautifulSoup
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
user_agent_list = [
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:77.0) Gecko/20100101 Firefox/77.0',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36',
]
domain = "https://www.kijiji.ca"
base_url = ("https://www.kijiji.ca/b-cars-trucks/ontario/used/")
listCars = [] #empty list for final dictionaries
#100 pages constitutes ~17 hour cycles - to check once per 24h
for x in range(2,4):
user_agent = random.choice(user_agent_list)
getHeader = {'User-Agent': user_agent}
new_url = base_url + f"page-{x}" + "/c174l9004a49"
# r = session.get(new_url)
try:
r = requests.get(new_url, timeout=10, headers = getHeader)
print("")
print("Https Request Success: FULL PAGE {}: {}".format(x, new_url))
print("")
except requests.exceptions.ConnectionError as e:
print("")
print("ERROR: Can't Get Page {}".format(x))
print("")
continue
page = r.content
soup = BeautifulSoup(page, "html.parser")
#list of every ad on page which should be looped through
rows = soup.find_all("div", {"class": "regular-ad"})
print("{} Results on Page {}.".format(len(rows), x))
#looping through each single ad on index page
for row in rows:
carDictionary = {}
ad_id = row.get("data-listing-id")
ad_url = domain + row.get("data-vip-url")
if "kijijiautos.ca" in ad_url:
print("Skipped Invalid URL Ad")
continue
carDictionary["Ad ID"] = ad_id
carDictionary["Ad URL"] = ad_url
#getting price value
carPriceRaw = row.find("div", {"class": "price"}).text.replace("\n", "").replace(" ","")
try:
if carPriceRaw == "Please Contact":
carPrice = None
elif carPriceRaw == "PleaseContact":
carPrice = None
else:
carPrice = int(float(row.find("div", {"class": "price"}).text.replace("\n", "").replace(" ","").replace('$', '').replace(',', '')))
carDictionary["Price"] = carPrice
except (ValueError):
continue
#https request for individual ad to dissect specific information
try:
singlePgContent = requests.get(ad_url, timeout=5, headers = getHeader).content
print("Https Request Success: SINGLE AD: {}".format(ad_url))
except requests.exceptions.ConnectionError as e:
print("ERROR: LINK FAILED - SKIPPED: {}".format(ad_url))
continue
single_page = BeautifulSoup(singlePgContent, "html.parser")
try:
attributes = single_page.find("div", {"class": "attributeListWrapper-2108313769"}) #both columns of attributes - holds 5 children nodes, we just need the first 2 columns
print("retrieved attributes")
columns = attributes.find_all("ul", {"class": "itemAttributeList-1090551278"}) #each is a column
first_col = columns[0] #year, make, model, trim
second_col = columns[1] #body type, drivetrain, transmission, kilometers
first_col_attributes = first_col.find_all("li", {"class": "itemAttributeWrapper-37588635"})
second_col_attributes = second_col.find_all("li", {"class": "itemAttributeWrapper-37588635"})
except (AttributeError):
print("Could not retrieve attributes")
continue
for attribute_li in first_col_attributes:
try: #to check and see if dt (attribute name) exists, otherwise skip iteration -eg. carfax report has no attribute name
label = attribute_li.find("dl", {"class": "itemAttribute-3080139557"}).find("dt", {"class": "attributeLabel-240934283"}).text
except:
continue
try:
if label == "Year":
carYear = attribute_li.find("dl", {"class": "itemAttribute-3080139557"}).find("dd", {"class": "attributeValue-2574930263"}).text
carDictionary["Year"] = int(carYear)
except (AttributeError, ValueError):
continue
try:
if label == "Make":
carMake = attribute_li.find("dl", {"class": "itemAttribute-3080139557"}).find("dd", {"class": "attributeValue-2574930263"}).text
carDictionary["Make"] = carMake
except (AttributeError, ValueError):
continue
try:
if label == "Model":
carModel = attribute_li.find("dl", {"class": "itemAttribute-3080139557"}).find("dd", {"class": "attributeValue-2574930263"}).text
carDictionary["Model"] = carModel
except (AttributeError, ValueError):
continue
try:
if label == "Trim":
carTrim = attribute_li.find("dl", {"class": "itemAttribute-3080139557"}).find("dd", {"class": "attributeValue-2574930263"}).text
carDictionary["Trim"] = carTrim
except (AttributeError):
carDictionary["Trim"] = None
for attribute_li in second_col_attributes:
try: #to check and see if dt (attribute name) exists, otherwise skip iteration -eg. carfax report has no attribute name
label = attribute_li.find("dl", {"class": "itemAttribute-3080139557"}).find("dt", {"class": "attributeLabel-240934283"}).text
except:
continue
try:
if label == "Body Type":
carBody = attribute_li.find("dl", {"class": "itemAttribute-3080139557"}).find("dd", {"class": "attributeValue-2574930263"}).text
carDictionary["Body"] = carBody
except (AttributeError):
carDictionary["Body"] = None
try:
if label == "Transmission":
carTrans = attribute_li.find("dl", {"class": "itemAttribute-3080139557"}).find("dd", {"class": "attributeValue-2574930263"}).text
carDictionary["Transmission"] = carTrans
except (AttributeError):
carDictionary["Transmission"] = None
try:
if label == "Kilometers":
carKM = int(attribute_li.find("dl", {"class": "itemAttribute-3080139557"}).find("dd", {"class": "attributeValue-2574930263"}).text.replace(',', ''))
carDictionary["Kilometers"] = carKM
except (AttributeError, ValueError):
carDictionary["Kilometers"] = None
try:
if label == "Drivetrain":
carDrivetrain = attribute_li.find("dl", {"class": "itemAttribute-3080139557"}).find("dd", {"class": "attributeValue-2574930263"}).text
carDictionary["Drivetrain"] = carDrivetrain
except (AttributeError):
carDictionary["Drivetrain"] = None
print(carDictionary)
listCars.append(carDictionary)
df = pandas.DataFrame(listCars)
df.to_csv('cars.csv')
print(df)
输出给我：
第 2 页有 40 个结果。 第 3 页有 0 个结果。 然后打印包含汽车列表数据的字典列表的 pandas 数据框。
【问题讨论】：
标签： python beautifulsoup python-requests