# -*- coding: utf-8 -*- import scrapy from ..items import QutoutiaoItem import json import re from ..settings import CATEGORY_INFO, LIST_LIMIT class QutoutiaoSpider(scrapy.Spider): name = \'qutoutiao\' #allowed_domains = [\'qutoutiao.net\'] start_urls = [] # 各类小标题 categoryInfo = CATEGORY_INFO limit = LIST_LIMIT for value in categoryInfo: url = BASE_API + "cid=%s&tn=1&page=1&limit=%s" % ( str(value[\'cid\']), str(limit)) start_urls.append(url) def parse(self, response): response_url = response.url # 分类id从url获取一次 searchObj = re.search(r\'(.*)cid=(\d+)\', response_url) cid = searchObj and searchObj.group(2) or 0 data = json.loads(response.text)[\'data\'][\'data\'] for value in data: # 初始化模型对象 item = QutoutiaoItem() # 来源 item[\'source_name\'] = value[\'source_name\'] # 标题 item[\'title\'] = value[\'title\'] # 详细页url url = item[\'url\'] = value[\'url\'] # url = url[0:url.find(\'?\')] # 简介 item[\'introduction\'] = value[\'introduction\'] # 封面图 item[\'cover\'] = value[\'cover\'] # 发布时间 item[\'publish_time\'] = value[\'publish_time\'] # 分类 item[\'cid\'] = cid # 爬取详情页 yield scrapy.Request(url=item[\'url\'], meta={\'meta_item\': item}, callback=self.detail_parse) # 详情页 def detail_parse(self, response): # 提取每次Response的meta数据 meta_item = response.meta[\'meta_item\'] # 取内容 content_selector = response.xpath(\'//div[@class="content"]\') meta_item[\'content_images\'] = content_selector.xpath( \'//img/@src|//img/@data-src\').extract() meta_item[\'content\'] = content_selector.extract()[0] yield meta_item # 列表API BASE_API = \'http://api.1sapp.com/content/outList?\' # 爬取地址
# -*- coding: utf-8 -*- # Define here the models for your scraped items # # See documentation in: # http://doc.scrapy.org/en/latest/topics/items.html import scrapy class QutoutiaoItem(scrapy.Item): # define the fields for your item here like: # 文章id aid = scrapy.Field() # 来源 source_name = scrapy.Field() # 标题 title = scrapy.Field() # 详细页url url = scrapy.Field() # 简介 introduction = scrapy.Field() # 封面图 cover = scrapy.Field() # 发布时间 publish_time = scrapy.Field() # 分类ID cid = scrapy.Field() # 内容 content = scrapy.Field() # 内容-中的图片 content_images = scrapy.Field()
# -*- coding: utf-8 -*- # Define here the models for your spider middleware # # See documentation in: # http://doc.scrapy.org/en/latest/topics/spider-middleware.html from scrapy import signals class QutoutiaoSpiderMiddleware(object): # Not all methods need to be defined. If a method is not defined, # scrapy acts as if the spider middleware does not modify the # passed objects. @classmethod def from_crawler(cls, crawler): # This method is used by Scrapy to create your spiders. s = cls() crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) return s def process_spider_input(self, response, spider): # Called for each response that goes through the spider # middleware and into the spider. # Should return None or raise an exception. return None def process_spider_output(self, response, result, spider): # Called with the results returned from the Spider, after # it has processed the response. # Must return an iterable of Request, dict or Item objects. for i in result: yield i def process_spider_exception(self, response, exception, spider): # Called when a spider or process_spider_input() method # (from other spider middleware) raises an exception. # Should return either None or an iterable of Response, dict # or Item objects. pass def process_start_requests(self, start_requests, spider): # Called with the start requests of the spider, and works # similarly to the process_spider_output() method, except # that it doesn’t have a response associated. # Must return only requests (not items). for r in start_requests: yield r def spider_opened(self, spider): spider.logger.info(\'Spider opened: %s\' % spider.name) from fake_useragent import UserAgent import logging class UserAgent_CookiesMiddleware(object): # 随机更换user-agent def __init__(self, crawler): super(UserAgent_CookiesMiddleware, self).__init__() self.ua = UserAgent() self.ua_type = crawler.settings.get("RANDOM_UA_TYPE", "random") self.logger = logging.getLogger(__name__) @classmethod def from_crawler(cls, crawler): return cls(crawler) def process_request(self, request, spider): def get_ua(): return getattr(self.ua, self.ua_type) random_agent = get_ua() if random_agent: # 记录 request.headers[\'User-Agent\'] = random_agent request.headers[\'Accept\'] = \'application/json, text/javascript, */*; q=0.01\' request.headers[\'Origin\'] = \'http://home.qutoutiao.net\' request.headers[\'Referer\'] = \'http://home.qutoutiao.net/pages/home.html\' self.logger.debug(\'Current UserAgent: \' + random_agent)
# -*- coding: utf-8 -*- # Define your item pipelines here # # Don\'t forget to add your pipeline to the ITEM_PIPELINES setting # See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html import scrapy from scrapy.utils.project import get_project_settings from scrapy.pipelines.images import ImagesPipeline import os from .qttutils import QttUtils # 封面下载 class CoverImagePipeline(ImagesPipeline): # 获取settings中的常量 IMAGES_STORE = get_project_settings().get(\'IMAGES_STORE\') # 下载图片 def get_media_requests(self, item, info): cover_images = item[\'cover\'] if cover_images: for image_url in cover_images: yield scrapy.Request(url=image_url) # 下载完成 def item_completed(self, results, item, info): # print(\'*\'*20,results,item,info) image_path = [x[\'path\'] for ok, x in results if ok] # 获取自定义存储路径 store_path = QttUtils.getStorePath() coverImages = [] # 将图片移动到新的路径 print(\'------------------image_path-\',image_path) print(\'-----------------type(image_path)--\', type(image_path)) if image_path: for image_url in image_path: file_name = os.path.split(str(image_url)) print(\'------------------file_name-\', file_name) print(\'------------------file_name-\', type(file_name)) new_image = store_path + \'/\' + file_name[1] coverImages.append(new_image) os.rename(self.IMAGES_STORE + \'/\' + image_url, new_image) item[\'cover\'] = coverImages return item # 内容图片下载 class ContentImagePipeline(ImagesPipeline): # 获取settings中的常量 IMAGE_STORE = get_project_settings().get(\'IMAGES_STORE\') # 下载图片 def get_media_requests(self, item, info): content_images = item[\'content_images\'] if content_images: for image_url in content_images: yield scrapy.Request(image_url) # 下载完成 def item_completed(self, results, item, info): image_path = [x[\'path\'] for ok, x in results if ok] print(\'---------------------image_path\', image_path) # 获取自定义存储路径 store_path = QttUtils.getStorePath() contentImages = [] # 将图片移动到新的路径 if image_path: for base_path in image_path: print(\'----------------value\', base_path) file_name = os.path.split(str(base_path)) new_image = store_path + "/" + file_name[1] contentImages.append(new_image) os.rename(self.IMAGE_STORE + "/" + base_path, new_image) item[\'content_images\'] = contentImages return item # # 下载完成 方法一 # def item_completed(self, results, item, info): # image_info = [(x[\'path\'], x[\'url\']) for ok, x in results if ok] # print(\'---------------------image_info\', image_info) # # 获取自定义存储路径 # store_path = QttUtils.getStorePath() # contentImages = [] # # 将图片移动到新的路径 # if image_info: # for value in image_info: # print(\'----------------value\', value) # image_url = value[0] # image_source = value[1] # # file_name = os.path.split(str(image_url)) # new_image = store_path + "/" + file_name[1] # contentImages.append((new_image, image_source)) # os.rename(self.IMAGE_STORE + "/" + image_url, new_image) # item[\'content_images\'] = contentImages # return item import json from .qttutils import QttUtils class QutoutiaoPipeline(object): def __init__(self): # 获取自定义的存储路径 store_path = QttUtils.getStorePath() json_path = store_path + \'/\' + \'qutoutiao.json\' self.filename = open(json_path, \'wb\') def process_item(self, item, spider): text = json.dumps(dict(item), ensure_ascii=False) + \'\n\' self.filename.write(text.encode(\'utf-8\')) return item def close_spider(self, spider): self.filename.close()
# -*- coding: utf-8 -*- # @Time : 2018-6-1 11:01 # @Author : Amir # @Site : # @File : qttutils.py # @Software: PyCharm \'\'\' 趣头条工具类 \'\'\' import time import os import shutil from .settings import DATA_PATH class QttUtils: # 获取存储路径 # # @param [string] action [remove删除目录，默认create] # @return [string] path/year/month/day/* @staticmethod def getStorePath(action=\'create\'): localtimes = time.localtime() year = time.strftime("%Y", localtimes) month = time.strftime(\'%m\', localtimes) day = time.strftime(\'%d\', localtimes) store_path = DATA_PATH + "/%s/%s/%s"%(year,month,day) # 删除目录 if os.path.exists(store_path) and action == \'remove\': shutil.rmtree(store_path) # 创建多级目录 if not os.path.exists(store_path) and action == \'create\': os.makedirs(store_path) return store_path
# -*- coding: utf-8 -*- # Scrapy settings for QuTouTiao project # # For simplicity, this file contains only settings considered important or # commonly used. You can find more settings consulting the documentation: # # http://doc.scrapy.org/en/latest/topics/settings.html # http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html # http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html BOT_NAME = \'QuTouTiao\' SPIDER_MODULES = [\'QuTouTiao.spiders\'] NEWSPIDER_MODULE = \'QuTouTiao.spiders\' #列表记录数 LIST_LIMIT = 10 # 储存路径 DATA_PATH = r\'./data\' IMAGES_STORE = r\'./image\' #分类 CATEGORY_INFO = [ {"cid":255,"name":"推荐"}, {"cid":1,"name":"热点"}, {"cid":6,"name":"娱乐"}, {"cid":5,"name":"养生"}, {"cid":2,"name":"搞笑"}, {"cid":7,"name":"科技"}, {"cid":8,"name":"生活"}, {"cid":10,"name":"财经"}, {"cid":9,"name":"汽车"}, ] # Crawl responsibly by identifying yourself (and your website) on the user-agent #USER_AGENT = \'QuTouTiao (+http://www.yourdomain.com)\' # Obey robots.txt rules ROBOTSTXT_OBEY = False # Configure maximum concurrent requests performed by Scrapy (default: 16) #CONCURRENT_REQUESTS = 32 # Configure a delay for requests for the same website (default: 0) # See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay # See also autothrottle settings and docs DOWNLOAD_DELAY = 3 # The download delay setting will honor only one of: #CONCURRENT_REQUESTS_PER_DOMAIN = 16 #CONCURRENT_REQUESTS_PER_IP = 16 # Disable cookies (enabled by default) COOKIES_ENABLED = False # Disable Telnet Console (enabled by default) #TELNETCONSOLE_ENABLED = False # Override the default request headers: #DEFAULT_REQUEST_HEADERS = { # \'Accept\': \'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\', # \'Accept-Language\': \'en\', #} # Enable or disable spider middlewares # See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html #SPIDER_MIDDLEWARES = { # \'QuTouTiao.middlewares.QutoutiaoSpiderMiddleware\': 543, #} # Enable or disable downloader middlewares # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html DOWNLOADER_MIDDLEWARES = { \'QuTouTiao.middlewares.UserAgent_CookiesMiddleware\': 299, } # Enable or disable extensions # See http://scrapy.readthedocs.org/en/latest/topics/extensions.html #EXTENSIONS = { # \'scrapy.extensions.telnet.TelnetConsole\': None, #} # Configure item pipelines # See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html ITEM_PIPELINES = { \'QuTouTiao.pipelines.QutoutiaoPipeline\': 300, \'QuTouTiao.pipelines.ContentImagePipeline\': 301, \'QuTouTiao.pipelines.CoverImagePipeline\': 302 } # Enable and configure the AutoThrottle extension (disabled by default) # See http://doc.scrapy.org/en/latest/topics/autothrottle.html #AUTOTHROTTLE_ENABLED = True # The initial download delay #AUTOTHROTTLE_START_DELAY = 5 # The maximum download delay to be set in case of high latencies #AUTOTHROTTLE_MAX_DELAY = 60 # The average number of requests Scrapy should be sending in parallel to # each remote server #AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 # Enable showing throttling stats for every response received: #AUTOTHROTTLE_DEBUG = False # Enable and configure HTTP caching (disabled by default) # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings #HTTPCACHE_ENABLED = True #HTTPCACHE_EXPIRATION_SECS = 0 #HTTPCACHE_DIR = \'httpcache\' #HTTPCACHE_IGNORE_HTTP_CODES = [] #HTTPCACHE_STORAGE = \'scrapy.extensions.httpcache.FilesystemCacheStorage\'