从 Scrapy 管道中删除重复项？

Question

我的 scrapy 爬虫从一组 url 中收集数据，但是当我再次运行它以添加新内容时，旧内容会保存到我的 Mongodb 数据库中。有没有办法检查这个项目是否已经在我的 Mongodb 数据库中找到（重复项目具有相同的标题字段），如果是，请将其从管道中删除。另外，保存后最好从数据库中删除它们吗？如果是，我将如何在我的项目中实现它。

这是我的管道：

import logging
import pymongo
from scrapy.exceptions import DropItem


class MongoPipeline(object):

collection_name = 'articles'

def __init__(self, mongo_uri, mongo_db):
    self.mongo_uri = mongo_uri
    self.mongo_db = mongo_db

@classmethod
def from_crawler(cls, crawler):
    ## pull in information from settings.py
    return cls(
        mongo_uri=crawler.settings.get('MONGO_URI'),
        mongo_db=crawler.settings.get('MONGO_DATABASE')
    )

def open_spider(self, spider):
    ## initializing spider
    ## opening db connection
    self.client = pymongo.MongoClient(self.mongo_uri)
    self.db = self.client[self.mongo_db]

def close_spider(self, spider):
    ## clean up when spider is closed
    self.client.close()



def process_item(self, item, spider):
    ## how to handle each post


    bbcDict = {}
    if item['art_content'] != []:
        bbcDict['art_content'] = item['art_content']
        bbcDict['date'] = item['date']
        bbcDict['date_str'] = item['date_str']
        bbcDict['title'] = item['title']
        bbcDict['url'] = item['url']
        self.db[self.collection_name].insert_one(dict(bbcDict))
        return item

   # self.db[self.collection_name].insert(dict(item))
   # logging.debug("Post added to MongoDB")
   # return item

这是我的爬虫

from datetime import datetime as dt
import scrapy
from ArtScraper.items import ArtscraperItem

class PostSpider(scrapy.Spider):


article = ""
name = 'crawly'
allowed_domains = []

start_urls = ['http://feeds.bbci.co.uk/arabic/rss.xml']



def parse(self, response):
    # get the subreddit from the URL
    #sub = response.url.split('/')[4]

    #Get the title

    # parse thru each of the posts
    #for post in response.css('div.thing'):
    articles = response.xpath('//channel/item')
    for article in articles:
        item = ArtscraperItem()
        print ('hello')
        item['date'] = dt.today()
        item['date_str'] = article.xpath('pubDate/text()').extract_first()
        item['url'] = article.xpath('link/text()').extract_first()
        item['title'] = article.xpath('title/text()').extract_first()
        url = item['url']
        yield scrapy.Request(
            url,
            callback=self.parse_article,
            meta={'item': item},  # carry over our item
        )
        #request = scrapy.Request(url, callback=self.parse_article)
        #request.meta['item'] = item
        #yield request

def parse_article(self, response):
    item = response.meta['item']
    pars = response.xpath("//div[@class='story-body']/div[@class='story-body__inner']/p/text()").extract()
    item['art_content'] = '-'.join(pars)
    print ("HHHH")
    yield item

提前致谢。

Answer 1

您可以通过在处理项目时在您的MongoPipeline 类上创建titles 列表来过滤重复项，并在process_items 期间使用DropItem 删除项目。 official docs 提供了一个很好的例子。然后，您可以在返回项目时保存到 MongoDB。

在您的情况下，这将是您的管道中重复过滤器的实现：

import logging
import pymongo
from scrapy.exceptions import DropItem


class MongoPipeline(object):

collection_name = 'articles'

def __init__(self, mongo_uri, mongo_db):
    self.mongo_uri = mongo_uri
    self.mongo_db = mongo_db
    self.titles_seen = set()

@classmethod
def from_crawler(cls, crawler):
    ## pull in information from settings.py
    return cls(
        mongo_uri=crawler.settings.get('MONGO_URI'),
        mongo_db=crawler.settings.get('MONGO_DATABASE')
    )

def open_spider(self, spider):
    ## initializing spider
    ## opening db connection
    self.client = pymongo.MongoClient(self.mongo_uri)
    self.db = self.client[self.mongo_db]

def close_spider(self, spider):
    ## clean up when spider is closed
    self.client.close()

def process_item(self, item, spider):
    if item['title'] in self.titles_seen:
        raise DropItem("Duplicate item title found: %s" % item)
    else:
        self.titles_seen.add(item['title'])
        return item

Answer 2

对我来说有必要导入 ItemAdapter 以将 Item 转换为数组

from itemadapter import ItemAdapter

def process_item(self, item, spider):
            adapter = ItemAdapter(item)
            if self.db[self.collection_name].find_one({'id':adapter['id']}) != None:
                dado = self.db[self.collection_name].find_one_and_update({'id':adapter['id']})
                ## ----> raise DropItem(f"Duplicate item found: {item!r}") <------
                print(f"Duplicate item found: {dado!r}")
            else:
                self.db[self.collection_name].insert_one(ItemAdapter(item).asdict())
            return item

我更喜欢更新而不是触发 dropitem。