如何连续运行 webscraper 直到被中断

Question

import pandas as pd
import requests
from bs4 import BeautifulSoup

page = requests.get("https://forecast.weather.gov/MapClick.php?lat=40.71455000000003&lon=-74.00713999999994#.XxWVcSgzbIU")
soup = BeautifulSoup(page.content, 'html.parser')
week = soup.find(id='seven-day-forecast-list')
items = week.find_all(class_='tombstone-container')

'''
print(items[1].find(class_='period-name').get_text())
print(items[1].find(class_='short-desc').get_text())
print(items[1].find(class_='temp').get_text())
'''
#doing the above with list comprehesion
while(1):
    period_names=[item.find(class_='period-name').get_text() for item in items]
    short_descrpition=[item.find(class_='short-desc').get_text() for item in items]
    temp_names=[item.find(class_='temp').get_text() for item in items]

    weather_stuff = pd.DataFrame({
        'period':period_names,
        'short_descrpition': short_descrpition,
        'temperature':temp_names,
        })
    weather_stuff.to_csv('weather.csv')

我可以使用 while(1) 继续更新 weather.csv 文件，直到我中断程序吗？

Answer 1

为了不断更新，您需要不断向网站发送请求并更新 csv。在您的情况下，您正在覆盖 csv 文件

import pandas as pd
import requests, time
from bs4 import BeautifulSoup

count = 0

while 1:
    page = requests.get("https://forecast.weather.gov/MapClick.php?lat=40.71455000000003&lon=-74.00713999999994#.XxWVcSgzbIU")
    soup = BeautifulSoup(page.content, 'html.parser')
    week = soup.find(id='seven-day-forecast-list')
    items = week.find_all(class_='tombstone-container')

    period_names=[item.find(class_='period-name').get_text() for item in items]
    short_descrpition=[item.find(class_='short-desc').get_text() for item in items]
    temp_names=[item.find(class_='temp').get_text() for item in items]

    weather_stuff = pd.DataFrame({
        'period':period_names,
        'short_descrpition': short_descrpition,
        'temperature':temp_names,
        })
    weather_stuff.to_csv('weather.csv')
    count+=1
    print(count)
    time.sleep(2)

输出是发送请求的次数。我还在 2 次连续通话之间进行了睡眠