链家深圳租房信息爬取练习附加源码

from urllib import request
from time import sleep
from  lxml import  etree
import csv
# import random    #sleep(random.random(1)*2) 随机秒数
# 参数部分
# sz_url = \'https://sz.lianjia.com/zufang/\'
#
# header = {
#
# \'Referer\': \'https://sz.lianjia.com/zufang/\',
# \'User-Agent\': \'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36\',
# }
# # 请求部分
# res = request.Request(sz_url,headers=header)
#
# response = request.urlopen(res)
# result = response.read().decode()
# # print(result)
# # 筛选部分
# html = etree.HTML(result)
# name_list = html.xpath(\'//ul[@id="house-lst"]/li/div[@class="info-panel"]/h2/a\')
# with open(\'house.csv\',"wb") as f:
#     for name in name_list:
#         title=name.attrib["title"]
#         f.write(title.encode())
#         f.write(\'\n\'.encode())
#         print(title)


# --------------------------------------------------------------------------------------------------------------
# # 参数部分
# sz_url = \'https://sz.lianjia.com/zufang/105101400296.html\'
#
# header = {
#
# \'User-Agent\': \'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36\',
# }
# # 请求部分
# res = request.Request(sz_url,headers=header)
#
# response = request.urlopen(res)
# result = response.read().decode()
# # print(result)
#
# html = etree.HTML(result)
# name_list = html.xpath(\'//div[@class="brokerName"]/a\')
#
# for name in name_list:
#     text = name.text
#     print(text)

# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@code  tree@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
# 请求测试
def getRequet(url,xpath,**headers):
    default_headers = {
        \'Connection\': \'keep-alive\',
        \'User-Agent\': \'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTM, like Gecko) Chrome/67.0.3396.62 Safari/537.36\',
    }
    if headers :
        headers ={ **headers, **default_headers}
    else:
        headers = default_headers
    req = request.Request(url,headers=headers)
    response = request.urlopen(req)
    result = response.read().decode()
    html = etree.HTML(result)
    name_list = html.xpath(xpath)
    return name_list
def main():

    with open(\'house.csv\',"wb") as f:#打开csv文件 写入数据
        # csv_file = open(\'house.csv\',\'wb\')
        # csv_write = csv.writer(csv_file,dialect=\'excel\')
        zf_url=\'https://sz.lianjia.com/zufang/\'#要访问的url地址
        zf_xpath=\'//ul[@id="house-lst"]/li/div[@class="info-panel"]/h2/a\'#租房xpath地址
        name_xpath=\'//div[@class="brokerName"]/a\'#联系人名字xpath
        house_list=getRequet(zf_url,zf_xpath)
        for house  in house_list:

            print(\'正在下载：\',zf_url)#打印下载链接地址
            attrib = house.attrib
            house_name = attrib[\'title\']
            url =attrib[\'href\']
            username=getRequet(url,name_xpath)[0].text#取联系人名字的文本信息下标0
            # csv_write.witerow(house_name,username)
            # print(\'@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\')
            # print(name)
            # print(url)


            #
            f.write(house_name.encode())#encode编码
            f.write(\'\n\'.encode())
            f.write(username.encode())
            f.write(\'\n\'.encode())

            # print(house_name)
            # print(username)
            sleep(1)
            # print(\'@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\')
        print(\'下载完成\')
        f.close()

if __name__==\'__main__\':
    main()