数据框显示为空

Question

将熊猫导入为 pd 从 bs4 导入 BeautifulSoup 导入请求 baseurl = "https://www.amazon.com/" headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36", "Accept-language":"en-美国，恩；q=0.5'} 对于范围内的 x (1,75)： main_page = requests.get(f'https://www.amazon.in/books-for-1-year-old/s?k=books+for+1+year+old&i=stripbooks&rh=n%3A1318073031%2Cp_n_age_range% 3A1318384031&page={x}'，标题 = 标题） 汤= BeautifulSoup（main_page.content，'lxml'） booklist = soup.find_all('div', attrs={'class':'a-section a-spacing-none'}) # 打印（书单）

for book in booklist:
    reference = book.find_all('a',attrs = {'class': 'a-link-normal s-no-outline'}, href=True)
    for item in reference:
        link = baseurl+item['href']
        #print(link)
        webpage = requests.get(link, headers = headers)
        soup2 = BeautifulSoup(webpage.content, "lxml")
        title_parent = soup2.find('span', attrs = {'id': 'productTitle'})
        if title_parent is not None:
            title = title_parent.text
            print(link)
            print(title)
            desc_parent1 = soup2.find('div', attrs = {'id': 'bookDescription_feature_div'})
            desc_parent2 = soup2.find('div', attrs = {'id': 'iframeContent'})
            if desc_parent1 is not None:
                desc = desc_parent1.find('div').text
            elif desc_parent1 is not None:
                desc = desc_parent2.find('div').text
            print(desc)

testlink = 'https://www.amazon.in/Wonder-House-Books/dp/9388369882/ref=sr_1_1_sspa?keywords=books+for+1+year+old&qid=1637501815&refinements=p_n_age_range%3A1318384031&s=books&sr=1-1-spons&psc=1&spLa=ZW5jcnlwdGVkUXVhbGlmaWVyPUFUVDk0RFk2WEpFTTAmZW5jcnlwdGVkSWQ9QTA2MTk3MDg3WUpXR0E2RzJXSCZlbmNyeXB0ZWRBZElkPUEwNDY1NzE5M0xRQzE4TUhYUkQwSiZ3aWRnZXROYW1lPXNwX2F0ZiZhY3Rpb249Y2xpY2tSZWRpcmVjdCZkb05vdExvZ0NsaWNrPXRydWU='
for book in booklist:
    
    r = requests.get(testlink, headers=headers)
    soup = BeautifulSoup(r.content, 'lxml')
    details = soup.find('ul', attrs = {'class':'a-unordered-list a-nostyle a-vertical a-spacing-none detail-bullet-list'}).text.strip()
    print(details)
    book_data = {
        'links': ['link'],
        'title':['title'],
        'description':['desc_parent1'],
        'description2' : ['desc_parent2'] ,
        'deatils': ['details'],
    }
print(book_data)

df = pd.DataFrame(book_data)
print(df.head())

当我执行print(book_data) 时，我得到了预期的字典数据，但是当它被转换为带有熊猫的数据框时，它显示为空。有人可以帮我解决这个问题吗？

Answer 1

抓取代码似乎工作正常，唯一的问题是book_data 字典变量。变量链接、标题等无法按照您认为的方式工作，这就是为什么您会得到一个空数据框，因为字典变量是空的/错误的。

import pandas as pd
from bs4 import BeautifulSoup
import requests

baseurl = "https://www.amazon.com/"
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36", "Accept-language":"en-US, en;q=0.5'}

for x in range(1,75):
    main_page = requests.get(f'https://www.amazon.in/books-for-1-year-old/s?k=books+for+1+year+old&i=stripbooks&rh=n%3A1318073031%2Cp_n_age_range%3A1318384031&page={x}', headers = headers)
    soup = BeautifulSoup(main_page.content, 'lxml')
    booklist = soup.find_all('div', attrs={'class':'a-section a-spacing-none'})
#     print(booklist)

df = pd.DataFrame(columns=['links', 'title', 'description'])

for book in booklist:
    reference = book.find_all('a',attrs = {'class': 'a-link-normal s-no-outline'}, href=True)
    for item in reference:
        link = baseurl+item['href']
        # print(link)
        webpage = requests.get(link, headers = headers)
        soup2 = BeautifulSoup(webpage.content, "lxml")
        title_parent = soup2.find('span', attrs = {'id': 'productTitle'})
        if title_parent is not None:
            title = title_parent.text
            # print(link)
            # print(title)
            desc_parent1 = soup2.find('div', attrs = {'id': 'bookDescription_feature_div'})
            desc_parent2 = soup2.find('div', attrs = {'id': 'iframeContent'})
            if desc_parent1 is not None:
                desc = desc_parent1.find('div').text
            elif desc_parent1 is not None:
                desc = desc_parent2.find('div').text
            # print(desc)
        df.loc[len(df.index)] = [link, title, desc]

print(df.head())