如何设置scrapy spider在多个页面上运行 - 没有下一页按钮

Question

我正在努力设置从头到尾生成结果的多个页面。不幸的是，该网站没有“下一页”按钮，这使事情变得更加困难。这是网站结果https://www.imot.bg/pcgi/imot.cgi?act=3&slink=5fuby1&f1=1。

import scrapy
import logging

class ApartmentsSpider(scrapy.Spider):
    name = 'apartments'
    allowed_domains = ["www.imot.bg"]
    start_urls = ["https://www.imot.bg/pcgi/imot.cgi?act=3&slink=5f8c9v&f1=1"
                , "https://www.imot.bg/pcgi/imot.cgi?act=3&slink=5f8c9v&f1=2"]

    def parse(self, response):
        # apartments = response.xpath("//div[@class='price']/text()").get()
        apartments = response.xpath("//a[@class='lnk1']")
        for apartment in apartments:
            apartment_type = apartment.xpath(".//text()").get()
            apartment_link = apartment.xpath(".//@href").get()

            yield response.follow(url = apartment_link, callback=self.parse_apartment, meta={"apartment_type":apartment_type,
                                                                                            "apartment_link" : apartment_link}) 

    def parse_apartment(self, response):
        apartment_type = response.request.meta["apartment_type"]
        apartment_address = response.xpath("//form/div[2]/span[1]/text()").get()
        apartment_price = response.xpath("//td/div[1]/strong[1]/text()").get()
        apartment_sqm = response.xpath("//ul[@class='imotData']/li[2]/text()").get()
        offered_by = response.xpath("//div/b[1]/text()").get()
        contact = response.xpath("//div[8]/span[1]/span[1]/text()").get()
        apartment_link = response.request.meta["apartment_link"]

        yield {
            "apartment_type" : apartment_type,
            "apartment_address" : apartment_address,
            "apartment_price" : apartment_price,
            "apartment_sqm" : apartment_sqm,
            "apartment_link" : apartment_link,
            "offer" : offered_by,
            "contact" : contact
        }

Answer 1

还有另一种方法可以做到这一点。如果您继续单击数字并转到下一页，您可以查看 URL 并了解它的情况。有了这个网站，下一个更改只需在 URL 上添加 +1。我希望代码更有意义。将这 4 行添加到您的 parse 方法中。我使用了数字 26，因为这是列表的最后一页。

  def parse(self, response):
        # apartments = response.xpath("//div[@class='price']/text()").get()
        apartments = response.xpath("//a[@class='lnk1']")
        for apartment in apartments:
            apartment_type = apartment.xpath(".//text()").get()
            apartment_link = apartment.xpath(".//@href").get()

            yield response.follow(url = apartment_link, callback=self.parse_apartment, meta={"apartment_type":apartment_type,
                                                                                            "apartment_link" : apartment_link})

        next_page = self.base_url + str(self.num)
        if self.num <= 26:
            self.num += 1
            yield response.follow(next_page, callback=self.parse)