如何让 phantomJS webdriver 等到加载特定的 HTML 元素然后返回 page.source？

Question

我为一个网络爬取对象开发了下面的代码。

它需要两个日期作为输入。然后在这两个日期之间创建一个日期列表，并将每个日期附加到包含某个位置的天气信息的网页 url。然后它将 HTML 数据表转换为 Dataframe，然后将数据作为 csv 文件存储在存储中（基本链接为：https://www.wunderground.com/history/daily/ir/mashhad/OIMM/date/2019-1-3，如您在此示例中所见，日期为 2019-1-3）：

from datetime import timedelta, date
from bs4 import BeautifulSoup
from selenium import webdriver
import pandas as pd
from furl import furl
import os
import time

class WebCrawler():
    def __init__(self, st_date, end_date):
        if not os.path.exists('Data'):
            os.makedirs('Data')
        self.path = os.path.join(os.getcwd(), 'Data')
        self.driver = webdriver.PhantomJS()
        self.base_url = 'https://www.wunderground.com/history/daily/ir/mashhad/OIMM/date/'
        self.st_date = st_date
        self.end_date = end_date

    def date_list(self):
        # Create list of dates between two dates given as inputs.
        dates = []
        total_days = int((self.end_date - self.st_date).days + 1)

        for i in range(total_days):
            date = self.st_date + timedelta(days=i)
            dates.append(date.strftime('%Y-%m-%d'))

        return dates

    def create_link(self, attachment):
        # Attach dates to base link
        f = furl(self.base_url)
        f.path /= attachment
        f.path.normalize()

        return f.url

    def open_link(self, link):
        # Opens link and visits page and returns html source code of page
        self.driver.get(link)
        html = self.driver.page_source

        return html

    def table_to_df(self, html):
        # Finds table of weather data and converts it into pandas dataframe and returns it
        soup = BeautifulSoup(html, 'lxml')
        table = soup.find("table",{"class":"tablesaw-sortable"})

        dfs = pd.read_html(str(table))
        df = dfs[0]

        return df

    def to_csv(self, name, df):
        # Save the dataframe as csv file in the defined path
        filename = name + '.csv'
        df.to_csv(os.path.join(self.path,filename), index=False)

这就是我想要使用WebCrawler 对象的方式：

date1 = date(2018, 12, 29)
date2 = date(2019, 1, 1)

# Initialize WebCrawler object
crawler = WebCrawler(st_date=date1, end_date=date2)
dates = crawler.date_list()

for day in dates:
    print('**************************')
    print('PROCESSING : ', day)
    link = crawler.create_link(day)
    print('WAITING... ')
    time.sleep(3)
    print('VISIT WEBPAGE ... ')
    html = crawler.open_link(link)
    print('DATA RETRIEVED ... ')
    df = crawler.table_to_df(html)
    print(df.head(3))
    crawler.to_csv(day, df)
    print('DATA SAVED ...')

出现的问题是循环的第一次迭代运行完美，但第二次循环停止并出现错误，显示No tables where found（发生在table = soup.find("table",{"class":"tablesaw-sortable"}) 行中），这是因为页面源由WebCrawler.open_link 在之前返回网页完全加载网页的内容，包括表格（包含天气信息）。网站也有可能因为服务器太忙而拒绝请求。

我们是否可以构建一个循环，不断尝试打开链接，直到它可以找到表格，或者至少等到表格加载然后返回表格？

Answer 1

您可以让 selenium 等待特定元素。在您的情况下，它将是类名为“tablesaw-sortable”的表。我强烈建议您使用 CSS 选择器来查找此元素，因为它可以快速获取所有表格元素且不易出错。

这是 CSS 选择器，为您预制 table.tablesaw-sortable。将 selenium 设置为等到该元素加载完毕。

来源：https://stackoverflow.com/a/26567563/4159473

Answer 2

我使用@mildmelon 建议的https://stackoverflow.com/a/26567563/4159473 解决方案重写了代码，并且我还在每次向服务器发送请求和请求页面源之间使用了一些延迟：

from datetime import timedelta, date
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException
import pandas as pd
from furl import furl
import os
import time
class WebCrawler():
    def __init__(self, st_date, end_date):
        if not os.path.exists('Data'):
            os.makedirs('Data')
        self.path = os.path.join(os.getcwd(), 'Data')
        self.driver = webdriver.PhantomJS()
        self.delay_for_page = 7
        self.base_url = 'https://www.wunderground.com/history/daily/ir/mashhad/OIMM/date/'
        self.st_date = st_date
        self.end_date = end_date

    def date_list(self):
        # Create list of dates between two dates given as inputs.
        dates = []
        total_days = int((self.end_date - self.st_date).days + 1)

        for i in range(total_days):
            date = self.st_date + timedelta(days=i)
            dates.append(date.strftime('%Y-%m-%d'))

        return dates

    def create_link(self, attachment):
        # Attach dates to base link
        f = furl(self.base_url)
        f.path /= attachment
        f.path.normalize()

        return f.url

    def open_link(self, link):
        # Opens link and visits page and returns html source code of page
        self.driver.get(link)
        myElem = WebDriverWait(self.driver, self.delay_for_page)\
        .until(EC.presence_of_element_located((By.CLASS_NAME, 'tablesaw-sortable')))


    def table_to_df(self, html):
        # Finds table of weather data and converts it into pandas dataframe and returns it
        soup = BeautifulSoup(html, 'lxml')
        table = soup.find("table",{"class":"tablesaw-sortable"})

        dfs = pd.read_html(str(table))
        df = dfs[0]

        return df

    def to_csv(self, name, df):
        # Save the dataframe as csv file in the defined path
        filename = name + '.csv'
        df.to_csv(os.path.join(self.path,filename), index=False)

date1 = date(2019, 2, 1)
date2 = date(2019, 3, 5)


# Initialize WebCrawler object
crawler = WebCrawler(st_date=date1, end_date=date2)
dates = crawler.date_list()
for day in few_dates:
    print('**************************')
    print('DATE : ', day)
    link = crawler.create_link(day)
    print('WAITING ....')
    print('')
    time.sleep(12)
    print('OPENING LINK ... ')

    try:
        crawler.open_link(link)
        html = crawler.driver.page_source
        print( "DATA IS FETCHED")
        df = crawler.table_to_df(html)
        print(df.head(3))
        crawler.to_csv(day, df)
        print('DATA SAVED ...')
    except TimeoutException:
        print( "NOT FETCHED ...!!!")

获取天气信息没有问题。我猜每个请求之间的延迟会导致更好的性能。 myElem = WebDriverWait(self.driver, self.delay_for_page)\.until(EC.presence_of_element_located((By.CLASS_NAME, 'tablesaw-sortable'))) 行也提高了速度。