爬虫框架Scrapy之完整例子

w-bb

创建一个新项目comment：

scrapy startproject comment

items.py

import scrapy


class CommentItem(scrapy.Item):
    comments = scrapy.Field()

spider.py

from scrapy import Request,Spider
from lxml import etree
from comment.items import CommentItem



#你想爬取哪一部电影，将其余的两部注释即可。
class CommentSpider(Spider):
    \'\'\'
    这个是三生三世十里桃花
    \'\'\'
    name = \'comment\'

    def start_requests(self):
        templateurl = \'https://movie.douban.com/subject/25823277/comments?start={}&limit=20&sort=new_score&status=P\'
        for i in range(3201):
            url = templateurl.format(str(i * 20))
            yield Request(url=url, callback=self.parse)

    def parse(self, response):
        selector = etree.HTML(response.text)
        item = SanshengItem()
        item[\'comments\'] = selector.xpath(\'//div[@class="comment"]/p/text()\')
        yield item


# class CommentSpider(Spider):
#     \'\'\'
#     这个是二十二
#     \'\'\'
#     name = \'comment\'
# 
#     def start_requests(self):
#         templateurl = \'https://movie.douban.com/subject/26430107/comments?start={}&limit=20&sort=new_score&status=P\'
#         for i in range(1601):
#             url = templateurl.format(str(i * 20))
#             yield Request(url=url, callback=self.parse)
# 
#     def parse(self, response):
#         selector = etree.HTML(response.text)
#         item = SanshengItem()
#         item[\'comments\'] = selector.xpath(\'//div[@class="comment"]/p/text()\')
#         yield item
# 
# class CommentSpider(Spider):
#     \'\'\'
#     这个是战狼
#     \'\'\'
#     name = \'comment\'
# 
#     def start_requests(self):
#         templateurl = \'https://movie.douban.com/subject/26363254/comments?start={}&limit=20&sort=new_score&status=P\'
#         for i in range(9001):
#             url = templateurl.format(str(i * 20))
#             yield Request(url=url, callback=self.parse)
# 
#     def parse(self, response):
#         selector = etree.HTML(response.text)
#         item = SanshengItem()
#         item[\'comments\'] = selector.xpath(\'//div[@class="comment"]/p/text()\')
#         yield item

pipelines.py

import pymongo


class MongoPipeline(object):

    collection = \'sansheng\'    # 三生的数据表
    # collection = \'ershier\'   # 二十二的数据表
    # collection = \'zhanlang\'  # 战狼的数据表


    def __init__(self, mongo_uri, mongo_db):
        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            mongo_uri = crawler.settings.get(\'MONGO_RUI\'),
            mongo_db = crawler.settings.get(\'MONGO_DB\')
        )

    def open_spider(self, spider):
        self.client = pymongo.MongoClient(self.mongo_uri)
        self.db = self.client[self.mongo_db]

    def close_spider(self, spider):
        self.client.close()

    def process_item(self, item, spider):
        table = self.db[self.collection]
        for com in item[\'comments\']:
            data = dict()
            data[\'comment\'] = com.strip().replace("\n", "")
            table.insert_one(data)
        return item

middlewares.py

import scrapy
from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware
import random


class MyUseragentMiddleware(UserAgentMiddleware):
    \'\'\'
    设置User-Agent
    \'\'\'

    def __init__(self, user_agent):
        self.user_agent = user_agent

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            user_agent=crawler.settings.get(\'USER_AGENTS\')
        )

    def process_request(self, request, spider):
        agent = random.choice(self.user_agent)
        request.headers[\'User-Agent\'] = agent

settings.py

BOT_NAME = \'comment\'

SPIDER_MODULES = [\'comment.spiders\']
NEWSPIDER_MODULE = \'comment.spiders\'


ROBOTSTXT_OBEY = False


DOWNLOAD_DELAY = 2


COOKIES_ENABLED = False


DEFAULT_REQUEST_HEADERS = {
    \'Accept\': \'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\',
    \'Accept-Language\': \'-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3\',
    \'Cookie\':\'浏览器复制粘贴你的cookie\'
}

DOWNLOADER_MIDDLEWARES = {
   \'comment.middlewares.MyUseragentMiddleware\': 400,
}

USER_AGENTS = [
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
    "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
    "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
    "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
    "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
    "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
    "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)",
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)",
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; 360SE)",
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
    "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
    "Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5",
    "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b13pre) Gecko/20110307 Firefox/4.0b13pre",
    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:16.0) Gecko/20100101 Firefox/16.0",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11",
    "Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
]

ITEM_PIPELINES = {
   \'comment.pipelines.MongoPipeline\': 300,
}

MONGO_URI = \'mongodb://localhost:27017\'
MONGO_DB = "douban"

注意：

请把COOKIES_ENABLED设置为 False，你可能觉得奇怪，为什么我们使用了cookie却需要把它设置为False，原因在于，我们直接把cookie放在了请求头里面，但是scrapy默认自己拥有一套处理cookie的中间件，当你把它设置为True的时候，两者会产生影响，从而请求失败，你可以自己尝试一下。那如果我执意要把他设置为True呢，难道就不能解决了么？当然是可以的，但是我们今天就不在深入的讨论这个问题，以后可以单独解释。

我们这里抓取评论数据是为了之后的分析所用。

 

参考：

http://www.cnblogs.com/cnkai/p/7418330.html

https://blog.csdn.net/topleeyap/article/details/78911246

分类：

技术点：

相关文章：

  • 2021-09-05
  • 2021-08-21
  • 2021-11-30
  • 2021-12-10
  • 2021-11-30
  • 2021-11-30
  • 2021-12-10
  • 2021-11-20
猜你喜欢
  • 2022-01-18
  • 2021-11-30
  • 2021-12-10
  • 2021-06-29
  • 2021-10-06
相关资源
相似解决方案
热门标签
Java Python linux javascript Mysql C# Docker 算法 前端 SpringBoot Redis Vue spring 设计模式 .net core .net kubernetes c++ 数据库 数据结构 大数据 js 机器学习 微服务 Android Go 程序员 面试 JVM ASP.net core 云原生 人工智能 后端 PHP git CSS golang k8s Nginx Django mybatis 深度学习 多线程 React 架构 devops 爬虫 云计算 Spring Boot LeetCode