grequests python 爬虫 并发

GitHub

安装

pip install gevent grequests

import grequests

urls = [
    'http://www.heroku.com',
    'http://python-tablib.org',
    'http://httpbin.org',
    'http://python-requests.org',
    'http://fakedomain/',
    'http://kennethreitz.com'
]

rs = (grequests.get(u) for u in urls)

reps = grequests.map(rs)

# 添加回调
def exception_handler(request, exception):
    print("Request failed")

reqs = [
   grequests.get('http://httpbin.org/delay/1', timeout=0.001),
   grequests.get('http://fakedomain/'),
   grequests.get('http://httpbin.org/status/500')]
grequests.map(reqs, exception_handler=exception_handler)