Scrapy 使用 CSS 提取数据和 excel 将所有内容导出到一个单元格中

Question

这是蜘蛛

import scrapy
import re

from ..items import HomedepotSpiderItem



class HomedepotcrawlSpider(scrapy.Spider):
    name = 'homeDepotCrawl'
    allowed_domains = ['homedepot.com']
    start_urls = ['https://www.homedepot.com/b/ZLINE-Kitchen-and-Bath/N-5yc1vZhsy/Ntk-ProductInfoMatch/Ntt-zline?NCNI-5&storeSelection=3304,3313,3311,3310,8560&experienceName=default']



    def parse(self, response):

        items = HomedepotSpiderItem()

        #get model
        productName = response.css('.pod-plp__description.js-podclick-analytics').css('::text').getall()

        productName = [x.strip(' ') for x in productName if len(x.strip())] 
        productName = [x.strip('\n') for x in productName if len(x.strip())] 
        productName = [x.strip('\t') for x in productName if len(x.strip())] 
        productName = [x.strip(',') for x in productName if len(x.strip())] 

        #productName = productName[0].split(',') tried to split the list into indiviudal elements


        productSKU = response.css('.pod-plp__model::text').getall()

        #get rid of all the stuff i dont need
        productSKU = [x.strip(' ') for x in productSKU] #whiteSpace
        productSKU = [x.strip('\n') for x in productSKU] 
        productSKU = [x.strip('\t') for x in productSKU] 
        productSKU = [x.strip(' Model# ') for x in productSKU] #gets rid of the model name 
        productSKU = [x.strip('\xa0') for x in productSKU] #gets rid of the model name 


        #get the price
        productPrice = response.css('.price__numbers::text').getall()

        #get rid of all the stuff i dont need
        productPrice = [x.strip(' ') for x in productPrice if len(x.strip())] 
        productPrice = [x.strip('\n') for x in productPrice if len(x.strip())] 
        productPrice = [x.strip('\t') for x in productPrice if len(x.strip())] 
        productPrice = [x.strip('$') for x in productPrice if len(x.strip())] 

        ## All prices are printing out twice, so take every other price
        productPrice = productPrice[::2]



        items['productName'] = productName
        items['productSKU'] = productSKU
        items['productPrice'] = productPrice

        yield items

Items.py

import scrapy


class HomedepotSpiderItem(scrapy.Item):
     #create items
     productName = scrapy.Field()
     productSKU = scrapy.Field()
     productPrice = scrapy.Field()
     #prodcutNumRating = scrapy.Field()

     pass

我的问题

我现在正在用 Scrapy 做一些练习，我使用 CSS 从 Home Depot 的网站上提取了所有这些数据。提取后，我手动剥离了所有我不需要的数据，并且在终端上看起来很好。 但是，在将所有内容导出到 Excel 后，我提取的所有数据都打印到每行一列中。例如：产品名称->所有模型进入一个单元格。我查看了一些scrapy文档，发现 .getall() 将所有内容作为列表返回，所以我尝试将列表拆分为单个元素，认为这会很好，但是，这将摆脱我抓取的所有数据。

任何帮助都将不胜感激，如果需要任何澄清，请告诉我！

编辑 我正在使用以下方法导出到 excel：scrapy crawl homeDepotCrawl -o test.csv -t csv

Answer 1

问题是您将所有项目加载到一个 scrapy.Item 实例中。有关详细信息，请参阅代码 cmets。

另外，值得注意的是，您可以使用项目加载器或创建项目管道来清理字段，而不是重复如此多的代码。在处理单个项目时，您不需要使用太多的列表理解。即使是一个简单的函数，你可以调用它们来运行它们，也比完成所有这些列表理解要好。

[1]https://docs.scrapy.org/en/latest/topics/loaders.html

[2]https://docs.scrapy.org/en/latest/topics/item-pipeline.html

[3]https://docs.scrapy.org/en/latest/topics/items.html

import scrapy
import re

from ..items import HomedepotSpiderItem

class HomedepotcrawlSpider(scrapy.Spider):
    name = 'homeDepotCrawl'
    allowed_domains = ['homedepot.com']
    start_urls = ['https://www.homedepot.com/b/ZLINE-Kitchen-and-Bath/N-5yc1vZhsy/Ntk-ProductInfoMatch/Ntt-zline?NCNI-5&storeSelection=3304,3313,3311,3310,8560&experienceName=default']


def parse(self, response):
    '''
    Notice when we set items variable we are not using .get or .extract yet
    We collect the top level of each item into a list of selectors. 
    Then we loop through the selectors creating a new scrapy.Item instance for each selector/item on the page. 
    The for product in items loop will step through each item selector individually.
    You can then chain .css to your variable product.css now to access each section of each 
    item individually and export them separately. 
    This will give you a new row for each item.
    '''
    items = response.css('.plp-pod')
    for product in items:
        # Create new scrapy.Item for each product in our selector list.
        item = HomedepotSpiderItem()
        item['productName'] = product.css('.pod-plp__description.js-podclickanalytics::text').get()
        # Notice we are yielding item inside of the loop.
        yield item