【爬虫】随机获取UA

使用模块  fake-useragent

https://github.com/hellysmile/fake-useragent

 

1.安装模块

2.配置

# settings.py

'''下载器中间件'''
DOWNLOADER_MIDDLEWARES = {
   'Lagou.middlewares.RandomUserAgentMiddleware': 543,
    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
}

'''UA的类型'''
RANDOM_UA_TYPE = "random"

# middlewares.py
'''模仿middlewares的UserAgentMiddleware写的类'''
class RandomUserAgentMiddleware(object):
    """This middleware allows spiders to override the user_agent"""

    def __init__(self, crawler):
        # 实例化UserAgent()，从配置文件读取ua的类型
        super().__init__()
        self.ua = UserAgent()
        self.ua_type = crawler.settings.get("RANDOM_UA_TYPE","random")

    @classmethod
    def from_crawler(cls, crawler):
        return cls(crawler)

    # def spider_opened(self, spider):
    #     self.user_agent = getattr(spider, 'user_agent', self.user_agent)

    def process_request(self, request, spider):
        def get_ua():
            # 通过反射获取随机UA
            random_ua =  getattr(self.ua,self.ua_type)
            return random_ua
        request.headers.setdefault("User-Agent", get_ua())