概览页抓取链接
1 import requests 2 import re 3 import pymysql 4 import hashlib 5 import datetime 6 7 8 class Demo(object): 9 def __init__(self): 10 self.host = \'127.0.0.1\' 11 self.db = \'app_mark\' 12 self.user = \'root\' 13 self.passwd = \'123456\' 14 self.charset = \'utf8mb4\' 15 self.headers = { 16 \'User-Agent\': \'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36\', 17 } 18 self.url = \'http://www.lvmama.com/\' 19 self.channel_link = [ 20 \'http://s.lvmama.com/group/H13K110000?keyword=%E6%99%AE%E5%90%89%E5%B2%9B&k=0#list\', # 海岛 21 \'http://s.lvmama.com/route/H13K310000?keyword=%E6%96%B0%E5%8A%A0%E5%9D%A1&k=0#list\', # 东南亚 22 \'http://s.lvmama.com/route/H13K310000?keyword=%E9%A6%99%E6%B8%AF&k=0#list\', # 中国港澳台 23 \'http://s.lvmama.com/group/H13K110000?keyword=%E8%BF%AA%E6%8B%9C&k=0#list\', # 迪拜 24 \'http://s.lvmama.com/group/C262H13K110000?keyword=%E4%BF%84%E7%BD%97%E6%96%AF&tabType=group#list\', # 俄罗斯 25 \'http://s.lvmama.com/group/H13K110000Y4?keyword=%E8%B6%8A%E5%8D%97#list#list\', # 越南 26 \'http://s.lvmama.com/group/C265H13K110000?keyword=%E6%B3%95%E5%9B%BD&tabType=group#list%22\', # 法瑞意德 27 \'http://s.lvmama.com/group/H13K110000?keyword=%E5%B7%B4%E5%8E%98%E5%B2%9B&k=0#list\', # 巴厘岛 28 \'http://s.lvmama.com/route/H13K310000?keyword=%E6%97%A5%E6%9C%AC&k=0#list\', # 日本 29 \'http://s.lvmama.com/route/H13K310000?keyword=%E6%AC%A7%E6%B4%B2&k=0#list\', # 欧美 30 \'http://s.lvmama.com/route/H13K440100?keyword=%E6%96%B0%E5%8A%A0%E5%9D%A1&k=0#list\', # 新加坡 31 \'http://s.lvmama.com/route/H13K310000?keyword=%E9%A6%99%E6%B8%AF&k=0#list\', # 香港 32 \'http://s.lvmama.com/route/H13K310000?keyword=%E6%BE%B3%E6%B4%B2&k=0#list\', # 澳洲 33 \'http://s.lvmama.com/route/H13K310000?keyword=%E6%B3%B0%E5%9B%BD&k=0#list\', # 泰国 34 \'http://s.lvmama.com/route/H13K440300?keyword=%E4%B8%89%E4%BA%9A&k=0#list\', # 三亚 35 \'http://s.lvmama.com/route/H13K440300P2?keyword=%E4%B8%89%E4%BA%9A&tabType=route350\', # 三亚p2 36 \'http://s.lvmama.com/route/H13K440300P3?keyword=%E4%B8%89%E4%BA%9A&tabType=route350\', # 三亚p3 37 \'http://s.lvmama.com/route/H13K440300P4?keyword=%E4%B8%89%E4%BA%9A&tabType=route350\', # 三亚p4 38 \'http://s.lvmama.com/route/H13K440300?keyword=%E5%8E%A6%E9%97%A8&k=0#list\', # 厦门 39 \'http://s.lvmama.com/route/H13K440300?keyword=%E5%B9%BF%E4%B8%9C&k=0#list\', # 广东 40 \'http://s.lvmama.com/route/H13K440300?keyword=%E4%BA%91%E5%8D%97&k=0#list\', # 云南 41 \'http://s.lvmama.com/route/H13K440300?keyword=%E4%B8%8A%E6%B5%B7&k=0#list\', # 上海 42 \'http://s.lvmama.com/route/H13K440300?keyword=%E8%A5%BF%E5%AE%89&k=0#list\', # 西安 43 \'http://s.lvmama.com/route/H13K440300?keyword=%E6%88%90%E9%83%BD&k=0#list\', # 成都 44 \'http://s.lvmama.com/route/H13K440300?keyword=%E5%90%89%E6%9E%97&k=0#list\', # 吉林 45 \'http://s.lvmama.com/route/H13K440300?keyword=%E8%A5%BF%E5%8C%97&k=0#list\', # 西北 46 \'http://s.lvmama.com/scenictour/K110000?keyword=%E5%8C%97%E4%BA%AC&k=0#list\', # 北京 47 \'http://s.lvmama.com/scenictour/K110000?keyword=%E5%B1%B1%E4%B8%9C&k=0#list\', # 山东 48 \'http://s.lvmama.com/scenictour/K110000?keyword=%E5%B1%B1%E8%A5%BF&k=0#list\', # 山西 49 \'http://s.lvmama.com/scenictour/K110000?keyword=%E6%B2%B3%E5%8C%97&k=0#list\', # 河北 50 \'http://s.lvmama.com/scenictour/K110000?keyword=%E8%BE%BD%E5%AE%81&k=0#list\', # 辽宁 51 ] 52 self.channel_name = [ 53 \'海岛\', 54 \'东南亚\', 55 \'中国港澳台\', 56 \'迪拜\', 57 \'俄罗斯\', 58 \'越南\', 59 \'法瑞意德\', 60 \'巴厘岛\', 61 \'日本\', 62 \'欧洲\', 63 \'新加坡\', 64 \'香港\', 65 \'澳洲\', 66 \'泰国\', 67 \'三亚\', 68 \'三亚p2\', 69 \'三亚p3\', 70 \'三亚p4\', 71 \'厦门\', 72 \'广东\', 73 \'云南\', 74 \'上海\', 75 \'西安\', 76 \'成都\', 77 \'吉林\', 78 \'西北\', 79 \'北京\', 80 \'山东\', 81 \'山西\', 82 \'河北\', 83 \'辽宁\', 84 ] 85 86 def get_html(self, url): 87 response = requests.get(url, headers=self.headers) 88 response.encoding = response.apparent_encoding 89 html = response.text 90 return html 91 92 def get_data(self): 93 # 首页抓取 94 # html = self.get_html(self.url) 95 # datas = re.findall(\'<li data-mmurl=.*?<div class="footLink">\', html, re.S)[0] 96 # lis = re.findall(\'(<li data-mmurl=.*?</li>)\', datas, re.S) 97 # for li in lis: 98 # # detail_url = re.findall(\'<li data-mmurl="(.*?)"\', li, re.S) # 详情页app链接 99 # detail_url = re.findall(\'href="(.*?)"\', li, re.S)[0] # 详情页网页链接 100 # self.save_data(detail_url) 101 # print(datas) 102 103 # 频道抓取 104 urls = [] 105 # 正则匹配链接 106 for index, channel in enumerate(self.channel_link): 107 html = self.get_html(channel) 108 divs = re.findall(\'<div class="product-left".*<div class="paging orangestyle"\', html, re.S)[0] 109 divs = re.findall(\'<div class="product-section">.*?</div>\', divs, re.S) 110 for div in divs: 111 print(self.channel_name[index]) 112 url = re.findall(\'<a href="(.*?)"\', div, re.S)[0] 113 self.save_data(url) 114 115 def save_data(self, url): 116 print(url) 117 hkey = hashlib.md5(url.encode(encoding=\'utf-8\')).hexdigest() 118 sitename = \'驴妈妈旅游\' 119 lasttime = datetime.datetime.now().strftime(\'%Y-%m-%d %H:%M:%S\') 120 tag = \'0\' 121 list_sql = [url, hkey, tag, sitename, lasttime] 122 con = pymysql.connect(host=self.host, db=self.db, user=self.user, passwd=self.passwd, charset=self.charset) 123 cur = con.cursor() 124 sql = \'insert into gly(link, hkey, tag, sitename, lasttime) values (%s, %s, %s, %s, %s)\' 125 try: 126 cur.execute(sql, list_sql) 127 print(\'insert success\') 128 except Exception as e: 129 con.rollback() 130 print(\'error~\', e) 131 else: 132 con.commit() 133 cur.close() 134 con.close() 135 136 137 if __name__ == \'__main__\': 138 demo = Demo() 139 demo.get_data()
细览页解析字段
1 import pymysql 2 import re 3 import requests 4 from multiprocessing.dummy import Pool as ThreadPool 5 import datetime 6 7 8 class XLY(object): 9 def __init__(self): 10 self.host = \'127.0.0.1\' 11 self.db = \'app_mark\' 12 self.user = \'root\' 13 self.passwd = \'123456\' 14 self.charset = \'utf8mb4\' 15 self.headers = { 16 \'User-Agent\': \'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36\' 17 } 18 self.start = datetime.datetime.now() 19 20 def get_data(self): 21 # 从gly表中拿链接 22 con = pymysql.connect(host=self.host, db=self.db, user=self.user, passwd=self.passwd, charset=self.charset) 23 cur = con.cursor() 24 sql = \'select link from gly where tag = "1" and sitename="驴妈妈旅游"\' 25 after_sql = \'update gly set tag="1" where tag="0" and sitename = "驴妈妈旅游"\' 26 try: 27 cur.execute(sql) 28 results = cur.fetchall() 29 cur.execute(after_sql) 30 except Exception as e: 31 con.rollback() 32 results = None 33 print(\'error~\', e) 34 else: 35 con.commit() 36 cur.close() 37 con.close() 38 return results 39 40 def parse_data(self, url): 41 # 正则匹配各个字段 42 print(url) 43 url = url[0] 44 # 匹配id 45 id = url.split(\'/\')[-1] 46 id = re.sub(\'\?.*\', \'\', id) 47 # print(id) 48 response = requests.get(url, headers=self.headers) 49 html = response.text 50 if \'scenic\' not in url and \'hotel\' not in url: 51 # 去掉酒店和景点 52 # 匹配标题 53 title = re.findall(\'<h.*?tit">(.*?)</h.*?>\', html, re.S) 54 if title: 55 title = title[0] 56 title = re.sub(\'\n|\r| |自营|<[\s\S]*?>\', \'\', title) 57 title = title.strip() 58 else: 59 title = re.findall(\'<p class="nchtitle">(.*?)</p>\', html, re.S) 60 if title: 61 title = title[0] 62 title = re.sub(\'\n|\r| |自营|<[\s\S]*?>\', \'\', title) 63 title = title.strip() 64 # 匹配价格 65 price = re.findall(\'<dfn.*?>(\d+)</dfn>\', html, re.S) 66 if price: 67 price = price[0] 68 else: 69 price = re.findall(\'<span class="product_price">.*?(\d+).*?</span>\', html, re.S) 70 if price: 71 price = price[0] 72 else: 73 price = re.findall(\'￥<em>(\d+)</em>\', html, re.S) 74 if price: 75 price = price[0] 76 else: 77 price = re.findall(\'<span class="product-price-value">.*?(\d+).*?</span>\', html, re.S) 78 if price: 79 price = price[0] 80 else: 81 price = None 82 # 匹配好评率 83 praise = re.findall(\'<p class="product_top_dp">[\s\S]*?<span>([\s\S]*?)</span>[\s\S]*?</p>\', html, re.S) 84 if praise: 85 praise = praise[0] 86 praise = re.sub(\'<.*?>\', \'\', praise) 87 praise = praise.strip() 88 else: 89 praise = re.findall(\'<a href="#pro_comment".*?<span>([\s\S]*?)</span>\', html, re.S) 90 if praise: 91 praise = praise[0] 92 else: 93 praise = re.findall(\'<span class="c_f60">([\s\S]*?)</span>\', html, re.S) 94 if praise: 95 praise = praise[0] 96 praise = praise.strip() 97 else: 98 praise = re.findall(\'<p class="product_top_dp">[\s\S]*?<span>([\s\S]*?)<small>%</small>[\s\S]*?</span>\', html, re.S) 99 if praise: 100 praise = praise[0] 101 praise = praise.strip() 102 else: 103 praise = re.findall(\'<span class="val">([\s\S]*?)</span>\', html, re.S) 104 if praise: 105 praise = praise[0] 106 if praise: 107 if \'%\' in praise: 108 praise = re.sub(\'%\', \'\', praise) 109 praise = float(praise) 110 if praise > 100: 111 praise = None 112 print(\'好评率抓取错误\') 113 else: 114 pass 115 else: 116 praise = None 117 # 匹配出发地 118 starting_city = re.findall(\'<dl class="info-city">[\s\S]*?出发城市[\s\S]*?<ii>([\s\S]*?)</ii></dd>\', html, re.S) 119 target_city = re.findall(\'<dt>目的地[\s\S]*?<dd>([\s\S]*?)</dd>\', html, re.S) 120 if starting_city: 121 starting_city = starting_city[0] 122 starting_city = re.sub(\'<.*?>\', \'\', starting_city) 123 # 匹配目的地 124 target_city = target_city[0] 125 target_city = re.sub(\'<.*?>\', \'\', target_city) 126 # 匹配天数 127 days_spent = re.findall(\'<dt>出游天数[\s\S]*?<dd>([\s\S]*?)</dd>\', html, re.S)[0] 128 days_spent = re.sub(\'<.*?>\', \'\', days_spent) 129 # print(days_spent) 130 else: 131 starting_city = target_city = days_spent = None 132 # 匹配类型 133 type_ = re.findall(\'<i class="t-category">([\s\S]*?)</i>\', html, re.S) 134 if type_: 135 type_ = type_[0] 136 else: 137 type_ = re.findall(\'<span class="product_top_type product_type_zyx">([\s\S]*?)</span>\', html, re.S) 138 if type_: 139 type_ = type_[0] 140 else: 141 type_ = re.findall(\'<span class="dpn_group">([\s\S]*?)</span>\', html, re.S) 142 if type_: 143 type_ = type_[0] 144 else: 145 type_ = None 146 # print(type_) 147 list_data = [id, title, price, praise, starting_city, target_city, days_spent, type_, url] 148 self.save_data(list_data) 149 150 def save_data(self, list_data): 151 # 写入数据库 152 con = pymysql.connect(host=self.host, db=self.db, user=self.user, passwd=self.passwd, charset=self.charset) 153 cur = con.cursor() 154 sql = \'insert into lvmama(id_num, title, price, praise, starting_city, target_city, days_spent, type_, link) values (%s, %s, %s, %s, %s, %s, %s, %s, %s)\' 155 # cur.execute(sql, list_data) 156 # con.commit() 157 try: 158 cur.execute(sql, list_data) 159 print(\'insert success\') 160 except Exception as e: 161 con.rollback() 162 print(\'error~\', e) 163 else: 164 con.commit() 165 cur.close() 166 con.close() 167 168 169 if __name__ == \'__main__\': 170 xly = XLY() 171 urls = xly.get_data() 172 if urls: 173 # 开启多线程 174 pool = ThreadPool(20) 175 pool.map(xly.parse_data, urls) 176 pool.close() 177 pool.join() 178 end = datetime.datetime.now() 179 print(\'耗时:\', (end-xly.start)) 180 # for url in urls: 181 # url = url[0] 182 # xly.parse_data(url) 183 # break