使用我用来抓取其他页面的相同代码显示两个不同的错误

Question

我使用代码从tripadvisor 中抓取了两页，效果非常好。但是现在，它向我显示了两个不同的错误：

with open("iletaitunsquare1.csv", "w", encoding="utf-8-sig", newline='') as csv_file:
    w = csv.writer(csv_file, delimiter = ";", quoting=csv.QUOTE_MINIMAL)
    w.writerow(["inf_rest_name", "rest_eclf", "name_client", "date_rev_cli", "opinion_cl"])

    with requests. Session() as s:
        for offset in range (270,1230,10):
            url = f'https://www.tripadvisor.fr/Restaurant_Review-g187147-d6575305-Reviews-or{offset}-Il_Etait_Un_Square-Paris_Ile_de_France.html'
            r = s.get(url)
            soup = bs(r.content, 'lxml')
            reviews = soup.select('.reviewSelector')
            ids = [review.get('data.reviewid') for review in reviews]
            r = s.post(
                    'https://www.tripadvisor.fr/OverlayWidgetAjax?Mode=EXPANDED_HOTEL_REVIEWS_RESP&metaReferer=',
                    data = {'reviews': ','.join(ids), 'contextChoice': 'DETAIL'},
                    headers = {'Referer': r.url}
                    )

            soup = bs(r.content, 'lxml')
            if not offset:
                inf_rest_name = soup.select_one('.heading').text.replace("\n","").strip()
                rest_eclf = soup.select_one('.header_links a').text.strip()

            for review in soup.select('.reviewSelector'):
                name_client = review.select_one('.info_text > div:first-child').text.strip()
                date_rev_cl = review.select_one('.ratingDate')['title'].strip()
                titre_rev_cl = review.select_one('.noQuotes').text.strip()
                opinion_cl = review.select_one('.partial_entry').text.replace("\n","").strip()
                row = [f"{inf_rest_name}", f"{rest_eclf}", f"{name_client}", f"{date_rev_cl}", f"{titre_rev_cl}", f"{opinion_cl}"]
                w.writerow(row)

执行错误：

"data = {'reviews': ','.join(ids), 'contextChoice': 'DETAIL'} TypeError: sequence item 0: expected str instance, NoneType found"

在我决定只更改第 6 行（站点页面）和第 7 行（Url）中的值之后：

with open("boutary.csv", "w", encoding="utf-8-sig", newline='') as csv_file:
    w = csv.writer(csv_file, delimiter = ";", quoting=csv.QUOTE_MINIMAL)
    w.writerow(["inf_rest_name", "rest_eclf", "name_client", "date_rev_cl", "titre_rev_cl", "opinion_cl"])

    with requests.Session() as s:
        for offset in range(40, 290, 10):
            url = f'https://www.tripadvisor.fr/Restaurant_Review-g187147-d9783452-Reviews-or{offset}-Boutary-Paris_Ile_de_France.html'
            r = s.get(url)
            soup = bs(r.content, 'lxml')
            reviews = soup.select('.reviewSelector')
            ids = [review.get('data-reviewid') for review in reviews]
            r = s.post(
                'https://www.tripadvisor.fr/OverlayWidgetAjax?Mode=EXPANDED_HOTEL_REVIEWS_RESP&metaReferer=',
                data = {'reviews': ','.join(ids), 'contextChoice': 'DETAIL'},
                headers = {'referer': r.url}
                )

            soup = bs(r.content, 'lxml')

            if not offset:
                    inf_rest_name = soup.select_one('.heading').text.replace("\n","").strip()
                    rest_eclf = soup.select_one('.header_links a').text.strip()

            for review in soup.select('.reviewSelector'):
                name_client = review.select_one('.info_text > div:first-child').text.strip()
                date_rev_cl = review.select_one('.ratingDate')['title'].strip()
                titre_rev_cl = review.select_one('.noQuotes').text.strip()
                opinion_cl = review.select_one('.partial_entry').text.replace("\n","").strip()
                row = [f"{inf_rest_name}", f"{rest_eclf}", f"{name_client}", f"{date_rev_cl}" , f"{titre_rev_cl}", f"{opinion_cl}"]
                w.writerow(row)

它告诉我

"row = [f"{inf_rest_name}", f"{rest_eclf}", f"{name_client}", f"{date_rev_cl}" , f"{titre_rev_cl}", f"{opinion_cl}"]

NameError: name 'inf_rest_name' is not defined"

这些错误很奇怪，因为之前我在其他 URL 中使用了相同的代码并且它运行良好。 你能告诉我发生了什么吗？我怎样才能正确运行它？感谢您的帮助。

Answer 1

这是因为在未在此处发布的原始代码中，它依赖于偏移量 0 的真/假值，在您之前的问题中是第一个偏移量。

例如，用：

for offset in range(0, 10, 10):
    if not offset:

第一个值 0 是 Falsy 与大于 0 的数字（在这种情况下）将被视为 Truthy。如果不是 True，即 False，即如果偏移量为 0，则设置 inf_rest_name 的值。这确保了它的值只在第一个循环而不是每次设置。它的值不会改变，所以不需要再次读取。

以下所有值都是真值，因此inf_rest_name 永远不会被设置。

for offset in range(40, 290, 10):
    if not offset:

你可以改成：

if offset == firstvalue:

例如

if offset == 40:
    inf_rest_name = soup.select_one('.heading').text.replace("\n","").strip()
    rest_eclf = soup.select_one('.header_links a').text.strip()

请参阅this 了解更多信息。

这些行还需要与第一汤而不是后来的汤一起使用（因为那只是评论）

import requests
from bs4 import BeautifulSoup as bs

with requests.Session() as s:
        for offset in range(40, 290, 10):
            url = f'https://www.tripadvisor.fr/Restaurant_Review-g187147-d9783452-Reviews-or{offset}-Boutary-Paris_Ile_de_France.html'
            r = s.get(url)
            soup = bs(r.content, 'lxml')
            if offset == 40:
                inf_rest_name = soup.select_one('.heading').text.replace("\n","").strip()
                rest_eclf = soup.select_one('.header_links a').text.strip()
            reviews = soup.select('.reviewSelector')
            ids = [review.get('data-reviewid') for review in reviews]
            r = s.post(
                'https://www.tripadvisor.fr/OverlayWidgetAjax?Mode=EXPANDED_HOTEL_REVIEWS_RESP&metaReferer=',
                data = {'reviews': ','.join(ids), 'contextChoice': 'DETAIL'},
                headers = {'referer': r.url}
                )

            soup = bs(r.content, 'lxml')

            for review in soup.select('.reviewSelector'):
                name_client = review.select_one('.info_text > div:first-child').text.strip()
                date_rev_cl = review.select_one('.ratingDate')['title'].strip()
                titre_rev_cl = review.select_one('.noQuotes').text.strip()
                opinion_cl = review.select_one('.partial_entry').text.replace("\n","").strip()
                row = [f"{inf_rest_name}", f"{rest_eclf}", f"{name_client}", f"{date_rev_cl}" , f"{titre_rev_cl}", f"{opinion_cl}"]

---

对于您的第一个代码块，您使用了无效属性。应该是

ids = [review.get('data-reviewid') for review in reviews]

注意我添加了一个 is None 测试来处理未找到。这也应该添加到顶级版本中。

import requests
from bs4 import BeautifulSoup as bs

with requests. Session() as s:
        for offset in range (270, 1230, 10):
            url = f'https://www.tripadvisor.fr/Restaurant_Review-g187147-d6575305-Reviews-or{offset}-Il_Etait_Un_Square-Paris_Ile_de_France.html'
            r = s.get(url)
            soup = bs(r.content, 'lxml')
            if offset == 270:
                inf_rest_name = soup.select_one('.heading').text.replace("\n","").strip()
                rest_eclf = soup.select_one('.header_links a').text.strip()
            reviews = soup.select('.reviewSelector')
            ids = [review.get('data-reviewid') for review in reviews]
            r = s.post(
                    'https://www.tripadvisor.fr/OverlayWidgetAjax?Mode=EXPANDED_HOTEL_REVIEWS_RESP&metaReferer=',
                    data = {'reviews': ','.join(ids), 'contextChoice': 'DETAIL'},
                    headers = {'Referer': r.url}
                    )

            soup = bs(r.content, 'lxml')

            for review in soup.select('.reviewSelector'):
                name_client= review.select_one('.info_text > div:first-child')
                if name_client is None:
                    name_client = 'N/A'
                else:
                    name_client = name_client.text.strip()

                date_rev_cl = review.select_one('.ratingDate')
                if date_rev_cl is None:
                    date_rev_cl = 'N/A'
                else:
                    date_rev_cl  = date_rev_cl['title'].strip()

                titre_rev_cl = review.select_one('.noQuotes')
                if titre_rev_cl is None:
                    titre_rev_cl = 'N/A'
                else:
                    titre_rev_cl = titre_rev_cl.text.strip()

                opinion_cl = review.select_one('.partial_entry')
                if opinion_cl is None:
                     opinion_cl = 'N/A'
                else:
                     opinion_cl =  opinion_cl.text.replace("\n","").strip()

                row = [f"{inf_rest_name}", f"{rest_eclf}", f"{name_client}", f"{date_rev_cl}", f"{titre_rev_cl}", f"{opinion_cl}"]
                print(row)