今日概要：
1、爬汽车之家的新闻资讯
2、爬github和chouti
3、requests和beautifulsoup
4、轮询和长轮询
5、django request.POST和request.body
一、HTTP知识扫盲
1、http的get请求 是没有请求体,所有的参数都放在请求头的url里
2、http的post请求 将请求内容放到请求体里
3、http = 请求头+请求体 响应头+响应体
4、http是无状态请求,一个请求，一次响应就会结束
二、爬取汽车之家的新闻页
#!/usr/bin/python
# -*- coding:utf-8 -*-
import requests
from bs4 import BeautifulSoup
response = requests.get('http://www.autohome.com.cn/news')
response.encoding = 'gbk' #汽车之家的中文是gbk编码
# print(response.text)
soup = BeautifulSoup(response.text,'html.parser')
tag = soup.find(name='div',attrs={'id':'auto-channel-lazyload-article'})
li_list = tag.find_all('li')
for li in li_list:
if li.find(name='h3'):
print(li.find(name='h3').text)
2.1爬取鲜花网
import requests
from bs4 import BeautifulSoup
response = requests.get('http://www.hua.com/aiqingxianhua/')
root = BeautifulSoup(response.text,'html.parser') #实例化soup对象
#通过.进行查找,所有的标签都为对象
div_list = root.find_all(attrs={"class":"grid-item"})
for div in div_list:
img_dir = div.find(name='img').get('src')
title = div.find(name='span',attrs={"class":"product-title"})
price = div.find(name='span',attrs={"class":"price-num"})
print(img_dir,title.text,price.text)
'''
//img01.hua.com/uploadpic/newpic/9012247.jpg_220x240.jpg 鲜花/幸福的约定-苏醒玫瑰33枝、紫罗兰、银叶菊 339
//img01.hua.com/uploadpic/newpic/9012246.jpg_220x240.jpg 鲜花/邻家女孩-红玫瑰33枝、红色小雏菊 296
//img01.hua.com/uploadpic/newpic/9010011.jpg_220x240.jpg 鲜花/一心一意-玫瑰11枝，粉色勿忘我0.3扎 126
//img01.hua.com/uploadpic/newpic/9012011.jpg_220x240.jpg 鲜花/阳光海岸-19枝香槟玫瑰 218
//img01.hua.com/uploadpic/newpic/9010966.jpg_220x240.jpg 鲜花/一往情深-精品玫瑰礼盒:19枝红玫瑰，勿忘我适量 235
//img01.hua.com/uploadpic/newpic/9012042.jpg_220x240.jpg 鲜花/热恋-红玫瑰50枝 359
//img01.hua.com/uploadpic/newpic/9012041.jpg_220x240.jpg 鲜花/浪漫缤纷-戴安娜粉玫瑰50枝 359
//img01.hua.com/uploadpic/newpic/9012175.jpg_220x240.jpg 鲜花/月光女神-白玫瑰11枝，绿色桔梗5枝，小菊3枝，白色石竹梅4枝 228
//img01.hua.com/uploadpic/newpic/9010947.jpg_220x240.jpg 鲜花/真爱如初-雪山玫瑰11枝、深紫色勿忘我0.3扎 186
//img01.hua.com/uploadpic/newpic/9012177.jpg_220x240.jpg 鲜花/不变的承诺-99枝红玫瑰 519
'''
三、爬取gitlab和chouti的新闻页
github自动登录
#!/usr/bin/python
# -*- coding:utf-8 -*-
import requests
from bs4 import BeautifulSoup
r1 = requests.get(url='https://github.com/login')
b1 = BeautifulSoup(r1.text,'html.parser')
auth_token = b1.find(attrs={'name':'authenticity_token'}).get('value')
r1_cookies_data = r1.cookies.get_dict()
print(auth_token)
r2 = requests.post('https://github.com/session', data={
"commit": "Sign in",
"utf8": '✓',
"authenticity_token": auth_token,
"login": "xxxx",
"password": "xxxx",
},
cookies=r1_cookies_data)
r2_cookies_data = r2.cookies.get_dict()
print(r1_cookies_data)
print(r2_cookies_data)
all_cookies = {}
all_cookies.update(r1_cookies_data)
all_cookies.update(r2_cookies_data)
#github直接用带token之后的cookies就行
r3 = requests.get('https://github.com/settings/emails',cookies=r2_cookies_data)
print(r3.text)
登录抽屉并自动点赞
#!/usr/bin/python
# -*- coding:utf-8 -*-
import requests
r1 = requests.get(url='http://dig.chouti.com/')
r1_cookies_data = r1.cookies.get_dict()
r2 = requests.post('http://dig.chouti.com/login',data={'phone':'xxx',"password":"xxx","oneMonth":1}
,cookies=r1_cookies_data)
r2_cookies_data = r2.cookies.get_dict()
print(r1_cookies_data)
print(r2_cookies_data)
all_cookies = {}
all_cookies.update(r1_cookies_data)
all_cookies.update(r2_cookies_data)
'''
session_id 在第一次请求
{'JSESSIONID': 'aaaIZQdBA4siraQ2m0t8v', 'route': '0c5178ac241ad1c9437c2aafd89a0e50', 'gpsd': 'dd55c4cda0a45f6bc3274a79a7e50316'}
{'puid': '417d102e3c72e88cd6003bc984c569b4', 'gpid': '4c91ec17bd8340bdb75116916e19bc20'}
'''
r3 = requests.post('http://dig.chouti.com/link/vote?linksId=14708906',cookies=r1_cookies_data)
print(r3.text)
'''
{"result":{"code":"9999", "message":"推荐成功", "data":{"jid":"cdu_50096919787","likedTime":"1508043437615000","lvCount":"6","nick":"congratula","uvCount":"3","voteTime":"小于1分钟前"}}}
'''
注意:有的登录页面,登录的时候不一定会给cookie,需要get一次才给cookie,而登录的时候仅仅是授权,get的时候的cookie,这样就不需要带第二次的cookie去请求
四、轮询和长轮询
-
轮询：客户端定时向服务器发送Ajax请求，服务器接到请求后马上返回响应信息并关闭连接。
优点：后端程序编写比较容易。
缺点：请求中有大半是无用，浪费带宽和服务器资源。
实例：适于小型应用。 -
长轮询：客户端向服务器发送Ajax请求，服务器接到请求后hold住连接，直到有新消息才返回响应信息并关闭连接，客户端处理完响应信息后再向服务器发送新的请求，服务器端会设置超时时间,当出现超时的时候，服务端会断开链接，客户端会再次请求服务端hold住
优点：在无消息的情况下不会频繁的请求。
缺点：服务器hold连接会消耗资源。
实例：WebQQ、Hi网页版、Facebook IM。
另外，对于长连接和socket连接也有区分：
-
长连接：在页面里嵌入一个隐蔵iframe，将这个隐蔵iframe的src属性设为对一个长连接的请求，服务器端就能源源不断地往客户端输入数据。
优点：消息即时到达，不发无用请求。
缺点：服务器维护一个长连接会增加开销。
实例：Gmail聊天
五、requests的用法
1、GET请求：
requests.get(url="http://www.oldboyedu.com")
# data="http GET / http1.1\r\nhost:oldboyedu.com\r\n....\r\n\r\n"
requests.get(url="http://www.oldboyedu.com/index.html?p=1")
# data="http GET /index.html?p=1 http1.1\r\nhost:oldboyedu.com\r\n....\r\n\r\n"
requests.get(url="http://www.oldboyedu.com/index.html",params={'p':1})
# data="http GET /index.html?p=1 http1.1\r\nhost:oldboyedu.com\r\n....\r\n\r\n"
2、POST请求：
requests.post(url="http://www.oldboyedu.com",data={'name':'alex','age':18}) # 默认请求头：application/x-www-form-urlencoded
data="http POST / http1.1\r\nhost:oldboyedu.com\r\n....\r\n\r\nname=alex&age=18"
requests.post(url="http://www.oldboyedu.com",json={'name':'alex','age':18}) # 默认请求头：application/json
data="http POST / http1.1\r\nhost:oldboyedu.com\r\n....\r\n\r\n{"name": "alex", "age": 18}"
requests.post(
url="http://www.oldboyedu.com",
params={'p':1},
json={'name':'alex','age':18}
) # 默认请求头：application/json
data="http POST /?p=1 http1.1\r\nhost:oldboyedu.com\r\n....\r\n\r\n{"name": "alex", "age": 18}"
3、更多参数
def request(method, url, **kwargs): """Constructs and sends a :class:`Request <Request>`. :param method: method for the new :class:`Request` object. :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. :param files: (optional) Dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``) for multipart encoding upload. ``file-tuple`` can be a 2-tuple ``('filename', fileobj)``, 3-tuple ``('filename', fileobj, 'content_type')`` or a 4-tuple ``('filename', fileobj, 'content_type', custom_headers)``, where ``'content-type'`` is a string defining the content type of the given file and ``custom_headers`` a dict-like object containing additional headers to add for the file. :param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple :param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed. :type allow_redirects: bool :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. :param verify: (optional) whether the SSL cert will be verified. A CA_BUNDLE path can also be provided. Defaults to ``True``. :param stream: (optional) if ``False``, the response content will be immediately downloaded. :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair. :return: :class:`Response <Response>` object :rtype: requests.Response Usage:: >>> import requests >>> req = requests.request('GET', 'http://httpbin.org/get') <Response [200]> """ 参数列表