用硒在scrapy中循环网址答案

【问题标题】：Looping Urls in scrapy with selenium用硒在scrapy中循环网址
【发布时间】：2021-01-24 20:58:30
【问题描述】：

我想使用循环加载所有 url 并尝试从中获取数据但我无法做到。有人对如何在 scrapy 中处理多个 url 有任何想法。

这是我的代码：

    import scrapy
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from shutil import which
from scrapy.selector import Selector
import time
from scrapy.http import Request

class TrSpider(scrapy.Spider):
    name = 'tr'
    allowed_domains = ['trc.com']
    start_urls = ['https://www.google.com/travel/things-to-do?g2lb=2502548,2503806,4258168,4270442,4306835,4317915,4328159,4371334,4401769,4419364,4463263,4463666,4464463,4482194,4482438,4486153,4270859,4284970,4291517&hl=en-PK&gl=pk&un=1&dest_mid=/m/0rtv&dest_state_type=main&sa=X&ved=2ahUKEwjvk6PXqLXuAhWN2BQKHZs1BNsQhdIBegQIARAW',
'https://www.google.com/travel/things-to-do?g2lb=2502548,2503806,4258168,4270442,4306835,4317915,4328159,4371334,4401769,4419364,4463263,4463666,4464463,4482194,4482438,4486153,4270859,4284970,4291517&hl=en-PK&gl=pk&un=1&dest_mid=/m/012gg6&dest_state_type=main&sa=X&ved=2ahUKEwjvk6PXqLXuAhWN2BQKHZs1BNsQhdIBegQIARAc',
'https://www.google.com/travel/things-to-do?g2lb=2502548,2503806,4258168,4270442,4306835,4317915,4328159,4371334,4401769,4419364,4463263,4463666,4464463,4482194,4482438,4486153,4270859,4284970,4291517&hl=en-PK&gl=pk&un=1&dest_mid=/m/027f5q&dest_state_type=main&sa=X&ved=2ahUKEwjvk6PXqLXuAhWN2BQKHZs1BNsQhdIBegQIARAi',
'https://www.google.com/travel/things-to-do?g2lb=2502548,2503806,4258168,4270442,4306835,4317915,4328159,4371334,4401769,4419364,4463263,4463666,4464463,4482194,4482438,4486153,4270859,4284970,4291517&hl=en-PK&gl=pk&un=1&dest_mid=/m/02d4_q&dest_state_type=main&sa=X&ved=2ahUKEwjvk6PXqLXuAhWN2BQKHZs1BNsQhdIBegQIARAo',
'https://www.google.com/travel/things-to-do?g2lb=2502548,2503806,4258168,4270442,4306835,4317915,4328159,4371334,4401769,4419364,4463263,4463666,4464463,4482194,4482438,4486153,4270859,4284970,4291517&hl=en-PK&gl=pk&un=1&dest_mid=/m/0b1nv7&dest_state_type=main&sa=X&ved=2ahUKEwjvk6PXqLXuAhWN2BQKHZs1BNsQhdIBegQIARAu',
'https://www.google.com/travel/things-to-do?g2lb=2502548,2503806,4258168,4270442,4306835,4317915,4328159,4371334,4401769,4419364,4463263,4463666,4464463,4482194,4482438,4486153,4270859,4284970,4291517&hl=en-PK&gl=pk&un=1&dest_mid=/m/04m7fh&dest_state_type=main&sa=X&ved=2ahUKEwjvk6PXqLXuAhWN2BQKHZs1BNsQhdIBegQIARA0',
'https://www.google.com/travel/things-to-do?g2lb=2502548,2503806,4258168,4270442,4306835,4317915,4328159,4371334,4401769,4419364,4463263,4463666,4464463,4482194,4482438,4486153,4270859,4284970,4291517&hl=en-PK&gl=pk&un=1&dest_mid=/m/0b95m1&dest_state_type=main&sa=X&ved=2ahUKEwjvk6PXqLXuAhWN2BQKHZs1BNsQhdIBegQIARA6',
'https://www.google.com/travel/things-to-do?g2lb=2502548,2503806,4258168,4270442,4306835,4317915,4328159,4371334,4401769,4419364,4463263,4463666,4464463,4482194,4482438,4486153,4270859,4284970,4291517&hl=en-PK&gl=pk&un=1&dest_mid=/m/0b95tg&dest_state_type=main&sa=X&ved=2ahUKEwjvk6PXqLXuAhWN2BQKHZs1BNsQhdIBegQIARBA',
'https://www.google.com/travel/things-to-do?g2lb=2502548,2503806,4258168,4270442,4306835,4317915,4328159,4371334,4401769,4419364,4463263,4463666,4464463,4482194,4482438,4486153,4270859,4284970,4291517&hl=en-PK&gl=pk&un=1&dest_mid=/m/02p77j&dest_state_type=main&sa=X&ved=2ahUKEwjvk6PXqLXuAhWN2BQKHZs1BNsQhdIBegQIARBG']

    def __init__(self):
        chrome_option = Options()
        chrome_option.add_argument('--headless')

        chrome_path = which('chromedriver')

        driver = webdriver.Chrome(executable_path=chrome_path)

        for a in self.start_urls:
            driver.get(a)
            driver.find_element_by_xpath('//button[@class="VfPpkd-LgbsSe VfPpkd-LgbsSe-OWXEXe-INsAgc VfPpkd-LgbsSe-OWXEXe-dgl2Hf Rj2Mlf OLiIxf PDpWxe J3Eqid"]').click()
            time.sleep(1)
            self.html = driver.page_source

    def parse(self, response):
        ab = response.url
        resp = Selector(text = self.html)
        
        for a in resp.xpath('//div[@class="GwjAi "]'):
            yield{
                'Name': a.xpath('./div[1]/div/text()').get(),
                'Rating': a.xpath('./div[2]/span/span/span[1]/text()').get(),
                'Number of reviews': a.xpath('./div[2]/span/span/span[2]/text()').get(),
                'Discription': a.xpath('./div[3]/text()').extract(),
                'Url': ab
            }

如果我在没有循环的情况下运行它，但它不会与多个链接一起工作。谢谢

【问题讨论】：

您是否遇到任何错误或您在哪里遇到问题？

标签： python selenium web-scraping scrapy

【解决方案1】：

尝试以下方法：

import scrapy
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from scrapy.selector import Selector
from selenium.common.exceptions import TimeoutException

class TrSpider(scrapy.Spider):
    name = 'tr'
    start_urls = [
        'https://www.google.com/travel/things-to-do?g2lb=2502548,2503806,4258168,4270442,4306835,4317915,4328159,4371334,4401769,4419364,4463263,4463666,4464463,4482194,4482438,4486153,4270859,4284970,4291517&hl=en-PK&gl=pk&un=1&dest_mid=/m/0rtv&dest_state_type=main&sa=X&ved=2ahUKEwjvk6PXqLXuAhWN2BQKHZs1BNsQhdIBegQIARAW',
        'https://www.google.com/travel/things-to-do?g2lb=2502548,2503806,4258168,4270442,4306835,4317915,4328159,4371334,4401769,4419364,4463263,4463666,4464463,4482194,4482438,4486153,4270859,4284970,4291517&hl=en-PK&gl=pk&un=1&dest_mid=/m/012gg6&dest_state_type=main&sa=X&ved=2ahUKEwjvk6PXqLXuAhWN2BQKHZs1BNsQhdIBegQIARAc',
        'https://www.google.com/travel/things-to-do?g2lb=2502548,2503806,4258168,4270442,4306835,4317915,4328159,4371334,4401769,4419364,4463263,4463666,4464463,4482194,4482438,4486153,4270859,4284970,4291517&hl=en-PK&gl=pk&un=1&dest_mid=/m/027f5q&dest_state_type=main&sa=X&ved=2ahUKEwjvk6PXqLXuAhWN2BQKHZs1BNsQhdIBegQIARAi',
        'https://www.google.com/travel/things-to-do?g2lb=2502548,2503806,4258168,4270442,4306835,4317915,4328159,4371334,4401769,4419364,4463263,4463666,4464463,4482194,4482438,4486153,4270859,4284970,4291517&hl=en-PK&gl=pk&un=1&dest_mid=/m/02d4_q&dest_state_type=main&sa=X&ved=2ahUKEwjvk6PXqLXuAhWN2BQKHZs1BNsQhdIBegQIARAo',
        'https://www.google.com/travel/things-to-do?g2lb=2502548,2503806,4258168,4270442,4306835,4317915,4328159,4371334,4401769,4419364,4463263,4463666,4464463,4482194,4482438,4486153,4270859,4284970,4291517&hl=en-PK&gl=pk&un=1&dest_mid=/m/0b1nv7&dest_state_type=main&sa=X&ved=2ahUKEwjvk6PXqLXuAhWN2BQKHZs1BNsQhdIBegQIARAu',
        'https://www.google.com/travel/things-to-do?g2lb=2502548,2503806,4258168,4270442,4306835,4317915,4328159,4371334,4401769,4419364,4463263,4463666,4464463,4482194,4482438,4486153,4270859,4284970,4291517&hl=en-PK&gl=pk&un=1&dest_mid=/m/04m7fh&dest_state_type=main&sa=X&ved=2ahUKEwjvk6PXqLXuAhWN2BQKHZs1BNsQhdIBegQIARA0',
        'https://www.google.com/travel/things-to-do?g2lb=2502548,2503806,4258168,4270442,4306835,4317915,4328159,4371334,4401769,4419364,4463263,4463666,4464463,4482194,4482438,4486153,4270859,4284970,4291517&hl=en-PK&gl=pk&un=1&dest_mid=/m/0b95m1&dest_state_type=main&sa=X&ved=2ahUKEwjvk6PXqLXuAhWN2BQKHZs1BNsQhdIBegQIARA6',
        'https://www.google.com/travel/things-to-do?g2lb=2502548,2503806,4258168,4270442,4306835,4317915,4328159,4371334,4401769,4419364,4463263,4463666,4464463,4482194,4482438,4486153,4270859,4284970,4291517&hl=en-PK&gl=pk&un=1&dest_mid=/m/0b95tg&dest_state_type=main&sa=X&ved=2ahUKEwjvk6PXqLXuAhWN2BQKHZs1BNsQhdIBegQIARBA',
        'https://www.google.com/travel/things-to-do?g2lb=2502548,2503806,4258168,4270442,4306835,4317915,4328159,4371334,4401769,4419364,4463263,4463666,4464463,4482194,4482438,4486153,4270859,4284970,4291517&hl=en-PK&gl=pk&un=1&dest_mid=/m/02p77j&dest_state_type=main&sa=X&ved=2ahUKEwjvk6PXqLXuAhWN2BQKHZs1BNsQhdIBegQIARBG'
    ]

    def start_requests(self):
        driver = webdriver.Chrome()
        wait = WebDriverWait(driver,10)
        for item_link in self.start_urls:
            driver.get(item_link)
            try:
                button = wait.until(EC.presence_of_element_located((By.XPATH, "//span[contains(.,'See all top sights')]")))
                driver.execute_script("arguments[0].click();",button)
            except TimeoutException:
                pass
            htmlelements = driver.page_source
            wait.until(EC.staleness_of(button))
            yield scrapy.Request(item_link,meta={"htmlelements":htmlelements})

    def parse(self, response):
        ab = response.url
        resp = Selector(text=response.meta.get("htmlelements"))
        for a in resp.xpath('//div[starts-with(@class,"GwjAi")]'):
            yield {
                'Name': a.xpath('./div[1]/div/text()').get(),
                'Rating': a.xpath('./div[2]/span/span/span[1]/text()').get(),
                'Number of reviews': a.xpath('./div[2]/span/span/span[2]/text()').get(),
                'Discription': a.xpath('./div[3]/text()').extract(),
                'Url': ab
            }

【讨论】：