python 爬虫demo

# coding: utf-8

import datetime
import urllib.parse
import urllib.request
from urllib.error import *
from bs4 import BeautifulSoup
import re
import os


def get_html(url, values):
    html = \'\'
    status_code = 200
    user_agent = \'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.82 Safari/537.36\'
    headers = {\'User-Agent\': user_agent}
    data = urllib.parse.urlencode(values).encode(encoding=\'UTF8\')
    for i in range(1,3):
        req = urllib.request.Request(url=url, headers=headers, data=data)
        try:
            response = urllib.request.urlopen(req)
        except HTTPError as e:
            print(url, values)
            print(\'The server couldn\\'t fulfill the request.\')
            print(\'HTTP Error,code: \', e.code)
            status_code = int(e.code)
            break
        except URLError as e:
            status_code = int(e.code)
            print(\'We failed to reach a server.Reason: \', e.reason)
            print(\'url: %s, status code:%d, retry count:%d\' % (url + \'?\' + bytes.decode(data), status_code, i))
        else:
            html = response.read( ).decode(\'gbk\')
            break

    return html, status_code


def request_page(page):
    url = \'http://cmispub.cicpa.org.cn/cicpa2_web/PersonIndexAction.do\'
    values = {
        \'method\': \'indexQuery\',
        \'queryType\': \'2\',
        \'isStock\': \'00\',
        \'pageSize\': \'\',
        \'pageNum\': page,
        \'offName\': \'\',
        \'ascGuid\': \'\',
        \'perCode\': 0,
        \'perName\': \'\'
    }
    return get_html(url, values)


def parse_cicpa_page(html):
    soup = BeautifulSoup(html, \'html.parser\')
    items = soup.select("#tabDetail a")
    return items


def request_detail(code):
    print(\'request code:\', code)
    url = \'http://cmispub.cicpa.org.cn/cicpa2_web/07/\' + code + \'.shtml\'
    values = {}
    return get_html(url, values)


def parse_detail_header(html):
    soup = BeautifulSoup(html, \'html.parser\')
    headers = soup.select("#detailtb td.tdl")
    line = \'\'
    for item in headers:
        line = line + item.get_text( ).strip( ) + \',\'
    line = line.strip(\',\')
    return line


def parse_detail_content(html):
    soup = BeautifulSoup(html, \'html.parser\')
    headers = soup.select("#detailtb td.data_tb_content")
    line = \'\'
    for item in headers:
        line = line + item.get_text( ).strip( ) + \',\'
    line = line.strip(\',\')
    return line


def create_file(filepath, header):
    file_dir = os.path.split(filepath)[0]
    if not os.path.isdir(file_dir):
        os.makedirs(file_dir)
    if not os.path.exists(filepath):
        f = open(filepath, \'w\')
        if len(header) > 0:
            f.write(header + \'\n\')
        f.close( )


def is_down_exists(code):
    return False


def main():
    start_time = datetime.datetime.now( )

    html_dir = \'D:/crawl_data/cicpa/html/\'
    if not os.path.isdir(html_dir):
        os.makedirs(html_dir)

    header_file = \'D:/crawl_data/cicpa/header.csv\'
    need_header = not os.path.exists(header_file)

    datafile = \'D:/crawl_data/cicpa/data_%s.csv\' % start_time.strftime("%Y%m%d_%H%M%S_%f")
    page_error_file = \'D:/crawl_data/cicpa/error_page_%s.txt\' % start_time.strftime("%Y%m%d_%H%M%S_%f")
    detail_error_file = \'D:/crawl_data/cicpa/error_detail_%s.txt\' % start_time.strftime("%Y%m%d_%H%M%S_%f")

    create_file(datafile, \'\')
    create_file(page_error_file, \'page,status\')
    create_file(detail_error_file, \'code,status\')

    data_file_object = open(datafile, \'+w\')
    page_error_file_object = open(page_error_file, \'+w\')
    detail_error_file_object = open(detail_error_file, \'+w\')

    for i in range(1, 6912):
        print(\'request:\', i)
        result, status = request_page(i)
        if status != 200:
            page_error_file_object.write(str(i) + \',\' + str(status) + \'\n\')
            page_error_file_object.flush( )
            continue
        items = parse_cicpa_page(result)
        for item in items:
            code = re.findall(r"javascript:viewDetail\(\\'(\w+?)\\',", str(item))[0]
            html_file_path = html_dir + code + \'.html\'
            if os.path.exists(html_file_path):
                continue
            detail_html, status = request_detail(code)
            if len(detail_html) == 0:
                detail_error_file_object.write(code + \',%d\n\' % status)
                detail_error_file_object.flush( )
                continue
            if need_header:
                header = parse_detail_header(detail_html)
                f = open(header_file, \'w\')
                f.write(header + \'\n\')
                f.close()
                need_header = False
            # save base data
            line = parse_detail_content(detail_html)
            data_file_object.write(line + \'\n\')
            data_file_object.flush( )
            # save html
            html_file_object = open(html_file_path, \'w\')
            html_file_object.write(detail_html + \'\n\')
            html_file_object.close( )
            print(line)

    data_file_object.close( )
    page_error_file_object.close( )
    detail_error_file_object.close( )
    print(\'finished in\', (datetime.datetime.now( ) - start_time).microseconds, \'ms\')

if __name__ == \'__main__\':
    main( )