WebScraping & python：在 html 中渲染 javascript？

Question

我正在尝试构建一个网络爬虫来从 tsx 页面获取趋势股票。我目前获得了所有趋势链接，现在我正在尝试抓取各个页面上的信息。根据我的代码，当我尝试在 getStockDetails() 中输出“quote_wrapper”时，它返回一个空列表。我怀疑这是因为 JavaScript 还没有在页面上呈现？不确定这是否是一回事。无论如何，我试图输出页面上的所有html进行调试，我也没有看到它。我读到“渲染” JavaScript 的唯一方法是使用 Selenium 并使用 browser.execute_script("return document.documentElement.outerHTML")。它适用于索引页面，所以我尝试在其他页面上使用它。我也在代码中对此进行了评论。感谢您的帮助，如果可以的话。

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup as soup
from urllib2 import urlopen as uReq

import time
import random
import requests


def getTrendingQuotes(source_code):
    # grabs all the trending quotes for that day
    links = []
    page_soup = soup(source_code, "lxml")
    trendingQuotes = page_soup.findAll("div", {"id": "trendingQuotes"})
    all_trendingQuotes = trendingQuotes[0].findAll('a')
    for link in all_trendingQuotes:
        url = link.get('href')
        name = link.text
        # print(name)
        links.append(url)
    return links


def getStockDetails(url, browser):
    print(url)
    source_code = browser.execute_script(
        "return document.documentElement.outerHTML")

    #What is the correct syntax here?
    #I'm trying to get the innerHTML of whole page in selenium driver
    #It seems I can only access the JavaScript for the entire page this way

    # source_code = browser.execute_script(
    #    "return" + url +".documentElement.outerHTML")

    page_soup = soup(source_code, "html.parser")
    # print(page_soup)
    quote_wrapper = page_soup.findAll("div", {"class": "quoteWrapper"})
    print(quote_wrapper)


def trendingBot(browser):

    while True:
        source_code = browser.execute_script(
            "return document.documentElement.outerHTML")
        trending = getTrendingQuotes(source_code)
        for trend in trending:
            browser.get(trend)
            getStockDetails(trend, browser)
        break
        # print(trend)


def Main():

    url = 'https://www.tmxmoney.com/en/index.html'
    browser = webdriver.Chrome(
        r"C:\Users\austi\OneDrive\Desktop\chromeDriver\chromedriver_win32\chromedriver.exe")
    browser.get(url)

    print("[+] Success! Bot Starting!")
    trendingBot(browser)
    browser.quit()


if __name__ == "__main__":
    Main()

Answer 1

请不要将 BeautifulSoup 和 Selenium 混合使用，这是不必要的。要使用 javascript 呈现页面，您需要等到元素生成，使用 WebDriverWait 并使用 browser.page_source 获取页面源，但此处未使用。

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait 

def getTrendingQuotes(browser):
    # wait until trending links appear, not really needed only for example
    all_trendingQuotes = WebDriverWait(browser, 10).until(
        lambda d: d.find_elements_by_css_selector('#trendingQuotes a')
    ) 
    return [link.get_attribute('href') for link in all_trendingQuotes]

def getStockDetails(url, browser):
    print(url)
    browser.get(url)
    quote_wrapper = browser.find_element_by_css_selector('div.quote-wrapper')
    print(quote_wrapper.text)
    #print(quote_wrapper.get_attribute('outerHTML'))

def trendingBot(url, browser):
    browser.get(url)
    trending = getTrendingQuotes(browser)
    for trend in trending:
        getStockDetails(trend, browser)

def Main():
    url = 'https://www.tmxmoney.com/en/index.html'
    browser = webdriver.Chrome(
        r"C:\Users\austi\OneDrive\Desktop\chromeDriver\chromedriver_win32\chromedriver.exe")
    print("[+] Success! Bot Starting!")
    trendingBot(url, browser)
    browser.quit()

if __name__ == "__main__":
    Main()