Excel 中缺少 2 个产品的数据

Question

我正在编写一个代码，需要从所有机车的网站上抓取数据。 导出到Excel时，2个产品没有出现： Excel 中的第 6 行（产品：63256）和第 7 行（产品：69256） 有人可以提示我为什么吗？ 这是代码： . . . .

import requests
from bs4 import BeautifulSoup
import pandas as pd
import xlsxwriter

baseurl = 'https://www.roco.cc/'

headers = {
    'Accept-Encoding': 'gzip, deflate, sdch',
    'Accept-Language': 'en-US,en;q=0.8',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
    'Cache-Control': 'max-age=0',
    'Connection': 'keep-alive',
}

productlinks = []

for x in range(1,2):
    r = requests.get(
        f'https://www.roco.cc/ren/products/locomotives/steam-locomotives.html?p={x}&verfuegbarkeit_status=41%2C42%2C43%2C45%2C44')
    soup = BeautifulSoup(r.content, 'lxml')
    productlist = soup.find_all('li', class_='item product product-item')
    
    for item in productlist:
        for link in item.find_all('a', class_='product-item-link', href=True):
            productlinks.append(link['href'])

Loco_list = []
Spare_parts_list = []

for link in productlinks:
    r = requests.get(link, allow_redirects=False)
    soup = BeautifulSoup(r.content, 'lxml')
    try:
        Manufacturer_name = soup.find(
            'div', class_='product-head-name').h1.text.strip()
    except:
        Manufacturer_name = ''
    try:
        Reference = soup.find('span', class_='product-head-artNr').text.strip()
    except:
        Reference = ''
        
    try:  
        Price = soup.find('div', class_='product-head-price').text.strip()
    except:
        Price = ''
        
    Type = 'Steam locomotive'
        
    try:
        Scale = soup.find('td', {'data-th': 'Scale'}).text.strip()
    except:
        Scale = ''

    try:  
        Current = soup.find('td', {'data-th': 'Control'}).text.split(' ')[0]
    except:
        Current = ''


    try:  
        Control = soup.find('td', {'data-th': 'Control'}).text.strip()
    except:
        Control = ''

    try:
        Interface = soup.find('td', {'data-th': 'Interface'}).text.strip()
    except:
        Interface = ''

    try:
        Digital_decoder = soup.find(
            'td', {'data-th': 'Digital decoder'}).text.strip()
    except:
        Digital_decoder = ''

    try:
        Decoder_Type = soup.find(
            'td', {'data-th': 'Decoder-Type'}).text.strip()
    except:
        Decoder_Type = ''

    try:
        Motor = soup.find('td', {'data-th': 'Motor'}).text.strip()
    except:
        Motor = ''

    try:
        Flywheel = soup.find('td', {'data-th': 'Flywheel'}).text.strip()
    except:
        Flywheel = ''

    try:
        Minimum_radius = soup.find(
            'td', {'data-th': 'Minimum radius'}).text.strip()
    except:
        Minimum_radius = ''

    try:
        Length_over_buffer = soup.find(
            'td', {'data-th': 'Length over buffer'}).text.strip()
    except:
        Length_over_buffer = ''

    try:
        Number_of_driven_axles = soup.find(
        'td', {'data-th': 'Number of  driven axles'}).text.strip()
    except:
        Number_of_driven_axles = ''

    try:
        Number_of_axles_with_traction_tyres = soup.find(
        'td', {'data-th': 'Number of  axles with traction tyres'}).text.strip()
    except:
        Number_of_axles_with_traction_tyres = ''

    try:
        Coupling = soup.find('td', {'data-th': 'Coupling'}).text.strip()
    except:
        Coupling = ''

    try:
        LED_lighting = soup.find(
            'td', {'data-th': 'LED lighting'}).text.strip()
    except:
        LED_lighting = ''

    try:
        Head_light = soup.find('td', {'data-th': 'Head light'}).text.strip()
    except:
        Head_light = ''

    try:
        LED_head_light = soup.find(
            'td', {'data-th': 'LED head light'}).text.strip()
    except:
        LED_head_light = ''

    try:
        Country = soup.find(
            'td', {'data-th': 'Original (country)'}).text.strip()
    except:
        Country = ''

    try:
        Railway_company = soup.find(
            'td', {'data-th': 'Railway Company'}).text.strip()
    except:
        Railway_company = ''

    try:
        Epoch = soup.find('td', {'data-th': 'Epoch'}).text.strip()
    except:
        Epoch = ''

    try:
        Description = soup.find(
            'div', class_='product-add-form-text').text.strip()
    except:
        Description = ''


    Locomotives = {
        'Manufacturer_name': Manufacturer_name,
        'Reference': Reference,
        'Price': Price,
        'Type': Type,
        'Scale': Scale,
        'Current': Current,
        'Control': Control,
        'Interface': Interface,
        'Digital_decoder': Digital_decoder,
        'Decoder_Type': Decoder_Type,
        'Motor': Motor,
        'Flywheel': Flywheel,
        'Minimum_radius': Minimum_radius,
        'Length_over_buffer': Length_over_buffer,
        'Number_of_driven_axles': Number_of_driven_axles,
        'Number_of_axles_with_traction_tyres': Number_of_axles_with_traction_tyres,
        'Coupling': Coupling,
        'LED_lighting': LED_lighting,
        'Head_light': Head_light,
        'LED_head_light': LED_head_light,
        'Country': Country,
        'Railway_company': Railway_company,
        'Epoch': Epoch,
        'Description': Description,
    } 

    Loco_list.append(Locomotives)

print(Locomotives)

# Manufacturer_name = 
# Reference = 

# Spare_part_number = soup.find('td', {'data-th': 'Art. No.:'}).text.strip()
# Spare_part_name = soup.find('td', {'data-th': 'Description'}).text.strip()
# Price = soup.find('td', {'data-th': 'Price:'}).text.strip()

# Spare_parts = {
#     'Manufacturer_name': Manufacturer_name,
#     'Reference': Reference,
#     'Spare_part_number': Spare_part_number,
#     'Spare_part_name': Spare_part_name,
#     'Price': Price
#         }


# Spare_parts_list.append(Spare_parts)


# print(Spare_parts_list)

df1 = pd.DataFrame(Loco_list)
# df2 = pd.DataFrame(Spare_parts_list)
# # df3 = pd.DataFrame()
# # df4 = pd.DataFrame()
writer = pd.ExcelWriter('Roco - locomotives.xlsx', engine='xlsxwriter')
df1.to_excel(writer, sheet_name='Model')
# df2.to_excel(writer, sheet_name='Spare parts')
# # df3.to_excel(writer, sheet_name='Documents')
# # df4.to_excel(writer, sheet_name='Photos')
writer.save()

print('Saved to file')


  

Answer 1

不要像跳过错误那样使用except，而是打印它并做一些研究来处理这个问题：

except Exception as e: 
    print(e)

你不允许重定向，所以在某些情况下你不会得到 soup - 启用重定向在某些情况下会导致无限重定向，在我看来这是网站的问题。

如果数量不是那么高，只需记录此 url 并手动添加数据。

---

如果您无法修复requests 与服务器之间的通信行为，请使用更像浏览器的方法。不是我最喜欢的，主要是因为太多和更慢，但它会解决这个问题 - 试试selenium

import pandas as pd
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
from bs4 import BeautifulSoup

url = 'https://www.roco.cc/ren/products/locomotives/steam-locomotives.html?verfuegbarkeit_status=42%2C45%2C41%2C43%2C44'
driver.get(url)

soup = BeautifulSoup(driver.page_source)

loco_list = []
spare_part_list = []

for link in soup.select('[id^="product-item-info_"]>a'):
    driver.get(link.get('href'))
    soup = BeautifulSoup(driver.page_source)
    d = dict(e.stripped_strings for e in soup.select('#product-attribute-specs-table tr'))
    d.update({'Reference': soup.select_one('.product-head-artNr').text.strip()})
    loco_list.append(d)
    
    spare_parts = pd.read_html(str(soup.select('#product-attribute-et-table')))[0].iloc[:,:3]
    spare_parts['Reference'] = soup.select_one('.product-head-artNr').text.strip()
    spare_part_list.append(spare_parts)

df1 = pd.DataFrame(loco_list)
df2 = pd.concat(spare_part_list, ignore_index=True)