Python学习之实现简单的高并发爬虫爬取网页

import gevent,time
from  urllib import request
#urllib的io操作gevent不会识别，不会自动切换，以下方法解决
from gevent import monkey
monkey.patch_all() #把当前程序的所有Io操作给我单独的做上标记

def f(url):
    print(\'GET: %s\' % url)
    resp = request.urlopen(url)
    data = resp.read()
    # f = open("url.html","wb")
    # f.write(data)
    # f.close()
    print(\'%d bytes received from %s.\' % (len(data), url))

#串行
urls = [
    \'https://www.python.org/\',
    \'https://www.nginx.org/\',
    \'https://github.com/\',
]
time_start = time.time()
for url in urls:
    f(url)
print("同步cost",time.time()-time_start)

#协程并行
async_time_start = time.time()
gevent.joinall([
    gevent.spawn(f, \'https://www.python.org/\'),
    gevent.spawn(f, \'https://www.yahoo.com/\'),
    gevent.spawn(f, \'https://github.com/\'),
])
print("异步cost",time.time()-async_time_start)