Scrapy 不遵循提取的链接

Question

Scrapy 不跟随抓取的链接。当我调用应该从之前抓取的链接中提取详细信息的函数时，它不起作用，而是从 start_urls 中抓取内容。

import scrapy


class OlxSpider(scrapy.Spider):
    name = 'olx'
    start_urls = ['https://www.olx.pl/nieruchomosci/mieszkania/wynajem/wroclaw/']
    allowed_domains = ['www.olx.pl']

    def parse(self, response):

        urls = response.css('a.detailsLink::attr(href)').extract()
        for url in urls:
            url = response.urljoin(url)
            yield response.follow(url, callback=self.parse_indetail)

    def parse_indetail(self, response):
        item = {}
        item['price'] = response.xpath('//h3/text()').extract()

        yield item

Answer 1

你需要显式添加start_requests，然后遍历：

这是一个例子：

class Olx(scrapy.Spider):
name = 'olx'

url = 'https://www.olx.in/api/relevance/search?category=1725&facet_limit=100&location=1000001&location_facet_limit=20&.....etc...etc...'

headers = {
    'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36'
}

def __init__(self):
    with open('results.csv', 'w') as csv_file:
        csv_file.write('title,description,location,features,date,price\n')

def start_requests(self):
    for page in range(0, 5):
        yield scrapy.Request(url=self.url + '&page=' + str(page), headers=self.headers, callback=self.parse)

def parse(self, res):