如何单击“下一步”并使用 Selenium 浏览所有页面？

Question

我有以下代码来抓取网站：

from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from selenium import webdriver
from functools import reduce

def stats_canada():
     driver = webdriver.Chrome('/Users/wwds/Desktop/chromedriver')
     driver.get('https://www150.statcan.gc.ca/n1/en/type/data?count=100&p=0-data/tables%2C-All#tables')
     WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, "tables-lnk"))).click()
     elements = WebDriverWait(driver, 30).until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, "#all a[target='_self']")))
     linkTitles = pd.DataFrame([title.text for title in elements]).rename(columns = {0 : 'Name'})
     links = pd.DataFrame([link.get_attribute("href") for link in elements]).rename(columns = {0 : 'Link'})
     elements = WebDriverWait(driver, 30).until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, "#all span[class='ndm-result-date']")))
     release_date = pd.DataFrame([date.text for date in elements]).rename(columns = {'0' : 'Release Date'}).rename(columns = {0 : 'Release Date'})
     elements = WebDriverWait(driver, 30).until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, "#all div[class='ndm-result-productid']")))
     table_id = pd.DataFrame([table.text for table in elements]).rename(columns = {0 : 'Table ID'})
     table_id['Table ID'] = table_id['Table ID'].str.replace("Table: ", "")
     data = reduce(lambda x,y: pd.merge(x, y, left_index = True, right_index = True), [linkTitles, links, release_date, table_id])
     return data


stats_canada()

我也有这个导航到下一页：

driver.find_element_by_partial_link_text('Next').click()

但是我想知道如何将它修改到我的函数中，以便它会抓取所有页面并将其附加到一个数据框中？

Answer 1

您只需要获取页数并对其进行迭代：

import re
import time

from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium import webdriver


def stats_canada():
     driver = webdriver.Chrome()
     url = 'https://www150.statcan.gc.ca/n1/en/type/data?count=100&p={}-data/tables%2C-All#tables'
     driver.get(url.format(0))
     WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.ID, "tables-lnk"))).click()

     pagination = [i for i in driver.find_elements_by_css_selector('#tables .tgl-panel .pagination.pagination-sm li')][-2]
     count_pages = re.search(r'to page (\d+) of', pagination.text).group(1)

     for page in range(1, int(count_pages)):
         driver.get(url.format(page))

stats_canada()