我在哪里错了，试图递归刮？

Question

我想用 scrapy 抓取一个网站，在类别中列出其产品他们进一步刮，但似乎我错过了一些东西。

有人回答在这里修复我的代码是最新版本，因为我认为我今天会再学习一次scrapy，但它仍然没有递归扫描它只是似乎循环遍历所有页面但从未解析项目

似乎从来没有进入 else 语句

yield scrapy.Request(url = response.url,callback = self.parse_item)

如果我强制它在不循环的情况下输出项目，我可以对其进行调试以检查项目是否正确解析

如果我更改以下内容

if product_pages:
            for product_url in product_pages:
                product_url2 = str(self.base_url + product_url)
                self.log("Queued up: %s" % product_url2)
                yield scrapy.Request(url = product_url2,callback = self.parse_product_pages)
        else: 
            yield scrapy.Request(url = response.url,callback = self.parse_item) 

到

if product_pages:
            for product_url in product_pages:
                product_url2 = str(self.base_url + product_url)
                self.log("Queued up: %s" % product_url2)
                yield scrapy.Request(url = product_url2,callback = self.parse_item)
        else: 
            yield scrapy.Request(url = response.url,callback = self.parse_product_pages)  

这是我在 python 2.7 中工作的代码

    import scrapy

from scrapy.spiders import CrawlSpider, Rule
from scrapy.selector import HtmlXPathSelector
from ybscrape.items import Product
from scrapy.linkextractors import LinkExtractor
from scrapy.linkextractors.sgml import SgmlLinkExtractor

class ybracingSpider(CrawlSpider):
    name = 'ybscrape2'
    download_delay = 0.75


    def __init__(self, *args, **kwargs):
        super(ybracingSpider, self).__init__(*args, **kwargs)
        self.allowed_domains = ['http://www.ybracing.com/', 'www.ybracing.com', 'www.esellepro.com']
        self.base_url = 'http://www.ybracing.com'
        self.start_urls = ['http://www.ybracing.com/karting/']

    def parse_start_url(self, response):
        category = response.xpath("//h2/a/@href").extract()


        #loop over catagory pages take the product link and add all pages url
        for product in category:
            all_pages = '?itemsperpage=99999'
            category_url = str(self.base_url + product + all_pages)
            self.log("Queued up: %s" % category_url)
            yield scrapy.Request(url = category_url,callback = self.parse_product_pages)

    def parse_product_pages(self, response):
        product_pages = response.xpath("//li/div/div/h3/a/@href").extract()


    #print("debug pause")
    #print(product_pages)
    #wait = input("PRESS ENTER TO CONTINUE.")
    #print("continue")

        if product_pages:
            for product_url in product_pages:
                product_url2 = str(self.base_url + product_url)
                self.log("Queued up: %s" % product_url2)
                yield scrapy.Request(url = product_url2,callback = self.parse_product_pages)
        else: 
            yield scrapy.Request(url = response.url,callback = self.parse_item)  

    def parse_item(self, response):
        item = Product()


    item['description'] = response.xpath("//div[@id='Tabbed-Container-Details']/div[2]/div/text()").extract()
    item['product_title'] = response.xpath("//h3[@class='Product-Heading']/text()").extract()
        item['price'] = response.xpath("//div[@id='Product-Price']/text()").extract()
    table_rows = response.xpath("//table[@id='SpecificationTab']/tr[*]/td[1]//text()").extract()



        yield item

我的物品.py

from scrapy.item import Item, Field

class Product(Item):
  product_title = Field()
  description = Field()
  price = Field() 

我期望我的代码在步骤中执行的操作

1. 获取第一个导出（类别）中的所有链接（这有效）
2. 查看每个类别中的所有 9999 种产品并导出列表（可行）
3. 从导出中获取产品 url 将其附加到基本 url 以访问每个产品页面。 （这有效） 4.然后从产品页面中读取数据以添加到项目中（永远不会在这里），除非我跳过 if 语句但那不是递归的，它不会处理这样的子类别

Answer 1

在这里，我对您的代码进行了一些更改，现在它可以工作了

import scrapy

from scrapy.spiders import CrawlSpider, Rule
from scrapy.selector import HtmlXPathSelector
from demo.items import DemoItem
from scrapy.linkextractors import LinkExtractor
from scrapy.linkextractors.sgml import SgmlLinkExtractor

class DemoSpider(CrawlSpider):
    name = 'ybracing2'

    def __init__(self, *args, **kwargs):
        super(DemoSpider, self).__init__(*args, **kwargs)
        self.allowed_domains = ['http://www.ybracing.com/', 'www.ybracing.com', 'www.esellepro.com']
        self.base_url = 'http://www.ybracing.com'
        self.start_urls = ['http://www.ybracing.com/racewear/']

    def parse_start_url(self, response):
        category = response.xpath("//h2/a/@href").extract()


        #loop over catagory pages take the product link and add all pages url
        for product in category:
            all_pages = '?itemsperpage=99999'
            category_url = str(self.base_url + product + all_pages)
            self.log("Queued up: %s" % category_url)
            yield scrapy.Request(url = category_url,callback = self.parse_product_pages)

    def parse_product_pages(self, response):
        product_pages = response.xpath("//div[@class='Product']/a/@href").extract()

        if product_pages:
            for product_url in product_pages:
                product_url2 = str(self.base_url + product_url)
                self.log("Queued up: %s" % product_url2)
                yield scrapy.Request(url = product_url2,callback = self.parse_item)
        else: 
            yield scrapy.Request(url = response.url,callback = self.parse_product_pages)  

    def parse_item(self, response):
        item = DemoItem()
        dirty_data ={}
        item['product_title'] = response.xpath("//h3[@class='Product-Heading']/text()").extract()
        item['price'] = response.xpath("//div[@id='Product-Price']/text()").extract()
        item['description'] = response.xpath("//div[@id='Tabbed-Container-Details']/div[2]/div/text()").extract()
        #image['product_image'] = 

        # for variable in dirty_data.keys():
            # if dirty_data[variable]: 
                # if variable == 'price':
                    # item[variable] = float(''.join(dirty_data[variable]).strip().replace('$', '').replace(',', ''))
                # else: 
                    # item[variable] = ''.join(dirty_data[variable]).strip()
        yield item