Python抓取微博评论(二)
对于新浪微博评论的抓取，首篇做的时候有些考虑不周，然后现在改正了一些地方，因为有人问，抓取评论的时候“爬前50页的热评，或者最新评论里的前100页“，这样的数据看了看，好像每条微博的评论都只能抓取到前100页，当page=101时，xhr数据就成空，然后没有内容，所以现在是抓取每条微博最近的100页的评论，即1000条评论，
代码有些改动，但是思路都是一样
# -*- coding: utf-8 -*- import re import urllib import urllib2 import os import stat import itertools import re import sys import requests import json import time import socket import urlparse import csv import random from datetime import datetime, timedelta import lxml.html from wordcloud import WordCloud import jieba import PIL import matplotlib.pyplot as plt import numpy as np from zipfile import ZipFile from StringIO import StringIO from downloader import Downloader from bs4 import BeautifulSoup from HTMLParser import HTMLParser from itertools import product import sys reload(sys) sys.setdefaultencoding(\'utf8\') import json,urllib2 def download(url, headers, num_try=2): while num_try >0: num_try -= 1 try: content = requests.get(url, headers=headers) return content.text except urllib2.URLError as e: print \'Download error\', e.reason return None header_dict = { \'Content-Type\':\'application/json; charset=utf-8\', \'Accept\':\'application/json, text/plain, */*\', \'Accept-Encoding\':\'gzip, deflate, br\', \'Accept-Language\':\'zh-CN,zh;q=0.9\', \'Connection\':\'keep-alive\', \'Cookie\':\'...\', \'Host\':\'m.weibo.cn\', \'Referer\':\'https://m.weibo.cn/u/1241148864?display=0&retcode=6102\', \'User-Agent\':\'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36\', \'X-Requested-With\':\'XMLHttpRequest\' } def wordcloudplot(txt): path = \'/Users/cy/Downloads/msyh.ttf\' path = unicode(path, \'utf8\').encode(\'gb18030\') alice_mask = np.array(PIL.Image.open(\'/Users/cy/Desktop/1.jpg\')) wordcloud = WordCloud(font_path=path, background_color="white", margin=5, width=1800, height=800, mask=alice_mask, max_words=2000, max_font_size=60, random_state=42) wordcloud = wordcloud.generate(txt) wordcloud.to_file(\'/Users/cy/Desktop/2.jpg\') plt.imshow(wordcloud) plt.axis("off") plt.show() def main(): a = [] f = open(r\'/Users/cy/Downloads/a.json\', \'r\').read() words = list(jieba.cut(f)) for word in words: if len(word) > 1: a.append(word) txt = r\' \'.join(a) wordcloudplot(txt) def get_comment(que): f = open(\'/Users/cy/Downloads/a.json\', \'w\') total_number = 10 for each in que: for i in range(1,total_number): textmood = {"id": each, "page": i} textmood = json.dumps(textmood) uu = \'https://m.weibo.cn/status/\' + str(each) header = {\'Connection\': \'keep-alive\', \'Cookie\': \'.......\', \'Accept-Language\': \'zh-CN,zh;q=0.8\', \'Host\': \'m.weibo.cn\', \'Referer\':uu, \'User-Agent\': \'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36\', \'X-Requested-With\': \'XMLHttpRequest\' } url = \'https://m.weibo.cn/api/comments/show?id=%s&page=%s\'%(str(each),str(i)) print url req = urllib2.Request(url=url, data=textmood, headers=header) res = urllib2.urlopen(req) res = res.read() contents = res d = json.loads(contents, encoding="utf-8") total_numbers = d[\'total_number\'] print total_numbers tto = total_numbers / 10 + 1 if total_number > tto: total_number = min(tto,10) if \'data\' in d: data = d[\'data\'] if data != "": for each_one in data: if each_one != "": if each_one[\'text\'] != "": mm = each_one[\'text\'].split(\'<\') if r\'回复\' not in mm[0]: index = mm[0]#filter(lambda x: x not in \'0123456789\', mm[0]) print index f.write(index.encode("u8")) def get_identified(): que = [] url = \'https://m.weibo.cn/api/container/getIndex?uid=1241148864&luicode=10000011&lfid=100103type%3D3%26q%3D%E5%BC%A0%E6%9D%B0&featurecode=20000180&type=uid&value=1241148864&containerid=1076031241148864\' for i in range(1,3): if i > 1: url = \'https://m.weibo.cn/api/container/getIndex?uid=1241148864&luicode=10000011&lfid=100103type%3D3%26q%3D%E5%BC%A0%E6%9D%B0&featurecode=20000180&type=uid&value=1241148864&containerid=1076031241148864&page=\'+str(i) print url req = download(url, header_dict,2) print req d = json.loads(req,encoding="utf-8") print d try: data = d[\'data\'][\'cards\'] print data except KeyError,e: print e.message if data != "": for each in data: print each[\'itemid\'] mm = each[\'itemid\'] if mm != "": identity = mm.split(\'-\') num = identity[1][1:] que.append(num) print num get_comment(que) if __name__ == \'__main__\': get_identified() main()