爬虫案例-爬运维工单

 

源代码：

# coding=utf-8
import requests
from lxml import etree


class ChaxunSpdier:
    def __init__(self):
        self.start_url = \'http://111.40.232.237:9000/eoms35/sheet/complaint/complaint.do?method=performQuery\'
        self.part_url = \'http://111.40.232.237:9000/eoms35/sheet/complaint/\'
        self.headers = {
            \'Connection\': \'keep-alive\',
            \'Cookie\': \'TSJSESSIONID=0000YvxNFfPYx8EBo8lsKNrKIl6:1bkt8lo7d\',#每次都得换一下
            \'Host\': \'111.40.232.237:9000\',
            \'Referer\': \'http://111.40.232.237:9000/eoms35/sheet/complaint/complaint.do?method=showQueryPage&type=interface&urlType=complaint&userName=liuhaoce&workSerial=0&isDutyMaster=false&workSerialTime=&startDuty=&endDuty=\',
            \'User-Agent\': \'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.116 Safari/537.36\'}

    def parse_url(self, url):
        formdata = {
            \'sheetIdStringExpression\': \'like\',
            \'main.sheetId\': \'\',  # 工单流水号
            \'titleStringExpression\': \'like\',
            \'main.title\': \'\',
            \'main.status\': \'\',
            \'statusChoiceExpression\': \'0\',
            \'task.taskName\': \'\',
            \'sendRoleIdStringExpression\': \'in\',
            \'main.sendRoleId\': \'\',
            \'sendDeptIdStringExpression\': \'in\',
            \'main.sendDeptId\': \'\',
            \'sendUserIdStringExpression\': \'in\',
            \'main.sendUserId\': \'\',
            \'operateRoleIdStringExpression\': \'in\',
            \'link.operateRoleId\': \'\',
            \'operateDeptIdStringExpression\': \'in\',
            \'link.operateDeptId\': \'\',
            \'operateUserIdStringExpression\': \'in\',
            \'link.operateUserId\': \'\',
            \'toDeptIdStringExpression\': \'in\',
            \'showArea\': \'大庆, 铁通\',  # 投诉受理省份
            \'main.toDeptId\': \'1005, 1021\',
            \'main.complaintType1\': \'\',
            \'complaintType1ChoiceExpression\': \'1010615100202\',  # 投诉类型一：家宽业务
            \'main.complaintType2\': \'\',
            \'complaintType2ChoiceExpression\': \'\',
            \'main.complaintType\': \'\',
            \'main.complaintType4\': \'\',
            \'main.complaintType5\': \'\',
            \'main.complaintType6\': \'\',
            \'main.complaintType7\': \'\',
            \'complaintNumStringExpression\': \'\',
            \'main.complaintNum\': \'\',
            \'parentCorrelationStringExpression\': \'\',
            \'main.parentCorrelation\': \'\',
            \'customAttributionStringExpression\': \'like\',
            \'main.customAttribution\': \'\',
            \'repeatComplaintTimesStringExpression\': \'>=\',
            \'main.repeatComplaintTimes\': \'\',
            \'complaintDescStringExpression\': \'like\',
            \'main.complaintDesc\': \'\',
            \'main.sendTime\': \'\',
            \'sendTimeStartDateExpression\': \'>=\',
            \'sendTimeStartDate\': \'2020-02-02 20:13:35\',  # 开始时间
            \'sendTimeLogicExpression\': \'and\',
            \'sendTimeEndDateExpression\': \'<=\',
            \'sendTimeEndDate\': \'2020-02-23 20:13:35\',  # 结束时间
            \'queryType\': \'record\'
        }
        response = requests.post(url, data=formdata, headers=self.headers)
        return response.content

    def get_content_list(self, html_raw):
        html = etree.HTML(html_raw)
        tr_list = html.xpath(\'//tbody/tr\')  # 每一个tr里放了一行投诉
        content_list = []
        for content in tr_list:
            item = {}
            zineirong = content.xpath(\'./td\')  # 每行投诉都封装在n个td标签下
            item[\'工单主题\'] = zineirong[0].xpath(\'.//text()\')[0]
            item[\'工单流水号\'] = zineirong[1].xpath(\'./a/text()\')[0]
            # item[\'处理时限\'] = zineirong[3].xpath(\'./text()\')[0]
            detail_link = self.part_url + zineirong[1].xpath(\'./a/@href\')[0]
            detail_dict = self.get_gongdan_detail(detail_link)
            item[\'xiangqing\'] = detail_dict
            content_list.append(item)
        next_gongdan_url = self.part_url + html.xpath("//a[text()=\'下一页\']/@href")[0] if len(html.xpath("//a[text()=\'下一页\']/@href")) > 0 else None  # 下一页工单列表明细
        return content_list, next_gongdan_url

    def get_gongdan_detail(self, url):
        html_raw = self.parse_url(url)
        html = etree.HTML(html_raw)
        xiangqing_dict = {}
        xiangqing_dict[\'投诉内容\'] = html.xpath(\'//*[@id="complainttext"]/text()\')
        xiangqing_dict[\'派往对象\'] = html.xpath(\'//div[@id="ext-gen47"]/table/tbody/tr[4]/td[4]/text()\')#ifram里了，查不到
        xiangqing_dict[\'qita\'] = html.xpath(\'//*[@id="ext-gen47"]/text()\')

        return xiangqing_dict

    def save_content_list(self, content_list):
        for i, v in enumerate(content_list, start=1):
            print(i, v)

    def run(self):
        next_url = self.start_url#工单查询主界面
        content_total_list = []
        while next_url is not None:
            html_raw = self.parse_url(next_url)  # 获取访问每一页工单源数据
            content_list, next_url = self.get_content_list(html_raw)  # 提取url具体内容放在里列表里，获取下一页链接
            content_total_list = content_total_list + content_list  #将提取每一页内容加载到列表中
        self.save_content_list(content_total_list)  # 每一条工单内容打印一下

if __name__ == \'__main__\':
    Spdier = ChaxunSpdier()
    Spdier.run()