无法在不发送请求的情况下在方法之间传递参数

Question

我在 python 中创建了一个脚本，使用 scrapy 和 selenium 来解析不同餐厅的主页中的链接，然后从它们的内页解析每家餐厅的名称。

当我将 scrapy 与 selenium 结合使用时，回调（或在方法之间传递参数）如何在不发送请求的情况下工作？

以下脚本使用我无法摆脱的self.driver.get(response.url) 覆盖回调：

import scrapy
from selenium import webdriver
from scrapy.crawler import CrawlerProcess
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support import expected_conditions as EC

class YPageSpider(scrapy.Spider):
    name = "yellowpages"
    link = 'https://www.yellowpages.com/search?search_terms=Pizza+Hut&geo_location_terms=San+Francisco%2C+CA'

    def start_requests(self):
        self.driver = webdriver.Chrome()
        self.wait = WebDriverWait(self.driver, 10)
        yield scrapy.Request(self.link,callback=self.parse)

    def parse(self,response):
        self.driver.get(response.url)
        for elem in self.wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".v-card .info a.business-name"))):
            yield scrapy.Request(elem.get_attribute("href"),callback=self.parse_info)

    def parse_info(self,response):
        self.driver.get(response.url)
        elem = self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, ".sales-info > h1"))).text
        yield {"title":elem}

if __name__ == '__main__':
    c = CrawlerProcess()
    c.crawl(YPageSpider)
    c.start()

Answer 1

你的意思是在函数之间传递变量吗？为什么不为此使用meta？不管有没有 Selenium，它都可以正常工作。我使用和你一样的代码，只是两个小更新：

def parse(self,response):
    self.driver.get(response.url)
    for elem in self.wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".v-card .info a.business-name"))):
        yield scrapy.Request(elem.get_attribute("href"),
                             callback=self.parse_info,
                             meta={'test': 'test'})  # <- pass anything here

def parse_info(self,response):
    self.driver.get(response.url)
    elem = self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, ".sales-info > h1"))).text
    yield {"title": elem, 'data': response.meta['test']}  # <- getting it here

所以，它输出：

...
2019-05-16 17:40:52 [scrapy.core.scraper] DEBUG: Scraped from <200 https://www.yellowpages.com/san-francisco-ca/mip/pizza-hut-473437740?lid=473437740>
{'data': 'test', 'title': u'Pizza Hut'}
...

Answer 2

@vezunchik 已经指出的链接答案几乎可以让您到达那里。唯一的问题是，当您使用完全相同的代码时，您将拥有多个 chromedriver 实例。要多次重复使用同一个驱动程序，您可以尝试如下。

在您的scrapy项目middleware.py中创建一个文件并粘贴以下代码：

from scrapy.http import HtmlResponse
from selenium import webdriver

class SeleniumMiddleware(object):
    def __init__(self):
        chromeOptions = webdriver.ChromeOptions()
        chromeOptions.add_argument("--headless")
        self.driver = webdriver.Chrome(options=chromeOptions)

    def process_request(self, request, spider):
        self.driver.get(request.url)
        body = self.driver.page_source
        return HtmlResponse(self.driver.current_url, body=body, encoding='utf-8', request=request)

如果你想看看 chmoedriver 如何在可见模式下遍历，我想提出一个更新。要让浏览器明显地移动，请尝试以下操作：

from scrapy import signals
from selenium import webdriver
from scrapy.http import HtmlResponse
from scrapy.xlib.pydispatch import dispatcher

class SeleniumMiddleware(object):
    def __init__(self):
        self.driver = webdriver.Chrome()
        dispatcher.connect(self.spider_closed, signals.spider_closed)

    def process_request(self, request, spider):
        self.driver.get(request.url)
        body = self.driver.page_source
        return HtmlResponse(self.driver.current_url, body=body, encoding='utf-8', request=request)

    def spider_closed(self):
        self.driver.quit()

使用以下脚本获取所需内容。通过中间件使用 selenium 的每个 url 将只有一个请求（导航）。现在您可以在您的蜘蛛中使用Selector() 来获取如下所示的数据。

import sys
# The hardcoded address leads to your project location which ensures that
# you can add middleware reference within crawlerprocess
sys.path.append(r'C:\Users\WCS\Desktop\yourproject')
import scrapy
from scrapy import Selector
from scrapy.crawler import CrawlerProcess

class YPageSpider(scrapy.Spider):
    name = "yellowpages"
    start_urls = ['https://www.yellowpages.com/search?search_terms=Pizza+Hut&geo_location_terms=San+Francisco%2C+CA']

    def parse(self,response):
        items = Selector(response)
        for elem in items.css(".v-card .info a.business-name::attr(href)").getall():
            yield scrapy.Request(url=response.urljoin(elem),callback=self.parse_info)

    def parse_info(self,response):
        items = Selector(response)
        title = items.css(".sales-info > h1::text").get()
        yield {"title":title}

if __name__ == '__main__':
    c = CrawlerProcess({
            'DOWNLOADER_MIDDLEWARES':{'yourspider.middleware.SeleniumMiddleware': 200},
        })
    c.crawl(YPageSpider)
    c.start()