爬取某电影网站最新电影

 1 # -*- coding: utf-8 -*-
 2 """
 3 Created on Wed Oct 12 16:48:33 2016
 4 
 5 @author: fuzzier
 6 """
 7 
 8 import requests
 9 from bs4 import BeautifulSoup
10 import re
11 import os
12 import codecs
13 
14 URL = \'http://www.xxxxx.net\'
15 
16 def download_page(url):
17     headers = {\'User_Agent\':\'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1581.2 Safari/537.36\'}
18     html = requests.get(url,headers=headers).content
19     return html
20     
21 def parser_html(data):
22     soup = BeautifulSoup(data,\'html.parser\')
23     films = []
24     trs = soup.find(\'div\',class_=\'bd3rl\').find(\'div\',class_=\'co_content8\').find_all(\'tr\')
25     for i in trs:
26         tr = i.find(\'a\',href=re.compile(r\'/\w+?/\w+?/\w+?/\d+?/\d+?.html\')).string
27         if tr:
28             films.append(tr)
29         else:
30             films.append(\'None\')
31     return films
32 
33 if __name__ == \'__main__\':
34     html = download_page(URL)
35     film_list = parser_html(html)
36     with codecs.open(os.getcwd()+\'\\dytt8_hot.txt\',\'w\',encoding=\'utf8\') as f:
37         for i in film_list:
38             f.write(i+\'\r\n\')