爬取梨视频网页的视频

"""
目标:爬取梨视频世界板块的视频并下载到本地
分析:数据为动态加载的
"""
import requests
from lxml import etree
import re
import os

dirName = \'./videoLibs\'
if not os.path.exists(dirName):
    os.mkdir(dirName)

headers = {
    \'User-Agent\': \'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36\'
}
url = \'https://www.pearvideo.com/category_2\'
# 获取世界板块页面源码
page_text = requests.get(url, headers=headers).text
# 解析
tree = etree.HTML(page_text)
li_list = tree.xpath(\'//*[@id="categoryList"]/li\')

for li in li_list:
    video_url = \'https://www.pearvideo.com/\'+li.xpath(\'./div/a/@href\')[0]
    video_title = li.xpath(\'./div/a/@href\')[0] + \'.mp4\'    # 标题中多含特殊字符,不能作为文件命名,用视频编号代替一下...
    # 获取视频播放页面源码
    video_page_text = requests.get(video_url, headers=headers).text
    # 发现视频链接存放在js代码中,所以不能使用xpath解析,改用正则!
    ex = \'srcUrl="(.*?)",vdoUrl\'
    detail_url = re.findall(ex, video_page_text, re.S)[0]
    # 向视频链接发起get请求下载视频到本地
    video_data = requests.get(detail_url, headers=headers).content
    video_path = dirName + \'/\' + video_title
    with open(video_path, \'wb\') as fp:
        fp.write(video_data)