无法使用不同的代理来执行后续请求答案

【问题标题】：Can't make use of different proxies to perform subsequesnt requests无法使用不同的代理来执行后续请求
【发布时间】：2019-01-11 10:21:22
【问题描述】：

我在 python 中编写了一个脚本，使用代理从网页中抓取遍历不同页面的不同帖子的链接。 My goal here is to make two subsequesnt requests using different proxies from a list。

脚本从列表中获取随机代理并通过make_requests() 函数发送请求，然后通过使用make_ano_requests() 函数新填充的链接从列表中选择另一个代理再次发出另一个请求。

最后，get_title() 函数打印结果。

但是，如果任何代理不起作用，则它会被 make_requests() 或 make_ano_requests() 两个函数中的任何一个从列表中踢出。

当我运行脚本时，它似乎在工作，但在它执行的某个地方，脚本卡住了，永远不会完成任务。我怎样才能完成任务？

这是我到目前为止所写的（proxyVault 包含虚假代理）：

import random
import requests
from random import choice
from bs4 import BeautifulSoup
from urllib.parse import urljoin

base_url = 'https://stackoverflow.com/questions/tagged/web-scraping'
lead_urls = [f'https://stackoverflow.com/questions/tagged/web-scraping?sort='
            f'newest&page={page}&pagesize=50' for page in range(1, 5)]

linkList = []

proxyVault = ['103.110.37.244:36022', '180.254.218.229:8080', '110.74.197.207:50632', '1.20.101.95:49001', '200.10.193.90:8080', '173.164.26.117:3128', '103.228.118.66:43002', '178.128.231.201:3128', '1.2.169.54:55312', '181.52.85.249:31487', '97.64.135.4:8080', '190.96.214.123:53251', '52.144.107.142:31923', '45.5.224.145:52035', '89.218.22.178:8080', '192.241.143.186:80', '113.53.29.218:38310', '36.78.131.182:39243']

def make_requests(url):
    proxy_url = choice(proxyVault)
    proxy = {'https': f'http://{proxy_url}'}
    try:
        res = requests.get(url, proxies=proxy)
        soup = BeautifulSoup(res.text, "lxml")
        linkList.extend([urljoin(base_url, item.get("href")) for item in soup.select(".summary .question-hyperlink")])
    except requests.exceptions.ProxyError:
        if proxy_url in proxyVault:
            proxyVault.remove(proxy_url)
            print(f'kicked out bad proxy by first func: {proxy_url}')
        return make_requests(url)

def make_ano_requests(url):
    proxy_url = choice(proxyVault)
    proxy = {'https': f'http://{proxy_url}'}
    try:
        res = requests.get(url, proxies=proxy)
        get_title(res.text)
    except requests.exceptions.ProxyError:
        if proxy_url in proxyVault:
            proxyVault.remove(proxy_url)
            print(f'kicked out bad proxy by second func: {proxy_url}')
        return make_ano_requests(url)

def get_title(response):
    soup = BeautifulSoup(response, "lxml")
    print(soup.select_one("h1[itemprop='name'] a").text)

if __name__ == '__main__':
    for lead_url in lead_urls:
        make_requests(lead_url)

    for single_link in linkList:
        make_ano_requests(single_link)

【问题讨论】：

该代码似乎可以工作，但所有请求都非常慢（即使删除了代理）。补充说，您通过可能非常慢的代理会给人一种冻结的印象。尝试打印不同的阶段以了解我的意思

标签： python python-3.x web-scraping proxy

【解决方案1】：

很可能是您的requests.get 导致它“挂起”，因为它们没有超时。就像documentation 所说：

几乎所有的生产代码都应该使用这个参数要求。不这样做可能会导致您的程序挂起无限期

所以我建议把它改成res = requests.get(url, proxies=proxy, timeout=1)以防止它挂起。

然而，它真的很慢。为了加快速度，我建议删除第二个请求，而不是从请求 1 中获取链接，而是获取字符串 [item.string for item in soup.select(".summary .question-hyperlink")]，这通常与标题相同。

编辑，添加用于在request.get 中捕获超时的代码：

import random
import requests
from random import choice
from bs4 import BeautifulSoup
from urllib.parse import urljoin
import urllib3

base_url = 'https://stackoverflow.com/questions/tagged/web-scraping'
lead_urls = [f'https://stackoverflow.com/questions/tagged/web-scraping?sort='
            f'newest&page={page}&pagesize=50' for page in range(1, 5)]

linkList = []

proxyVault = ['103.110.37.244:36022', '180.254.218.229:8080', '110.74.197.207:50632', '1.20.101.95:49001', '200.10.193.90:8080', '173.164.26.117:3128', '103.228.118.66:43002', '178.128.231.201:3128', '1.2.169.54:55312', '181.52.85.249:31487', '97.64.135.4:8080', '190.96.214.123:53251', '52.144.107.142:31923', '45.5.224.145:52035', '89.218.22.178:8080', '192.241.143.186:80', '113.53.29.218:38310', '36.78.131.182:39243']

def make_requests(url):
    proxy_url = choice(proxyVault)
    proxy = {'https': f'http://{proxy_url}'}
    try:
        res = requests.get(url, proxies=proxy, timeout=1)
        soup = BeautifulSoup(res.text, "lxml")
        linkList.extend([urljoin(base_url, item.get("href")) for item in soup.select(".summary .question-hyperlink")])
    except (requests.exceptions.ProxyError,
            requests.exceptions.Timeout,
            requests.exceptions.ConnectionError,
            urllib3.exceptions.MaxRetryError):
        if proxy_url in proxyVault:
            proxyVault.remove(proxy_url)
            print(f'kicked out bad proxy by first func: {proxy_url}')
        return make_requests(url)

def make_ano_requests(url):
    proxy_url = choice(proxyVault)
    proxy = {'https': f'http://{proxy_url}'}
    try:
        res = requests.get(url, proxies=proxy, timeout=1)
        get_title(res.text)
    except (requests.exceptions.ProxyError,
            requests.exceptions.Timeout,
            requests.exceptions.ConnectionError,
            urllib3.exceptions.MaxRetryError):
        if proxy_url in proxyVault:
            proxyVault.remove(proxy_url)
            print(f'kicked out bad proxy by second func: {proxy_url}')
        return make_ano_requests(url)

def get_title(response):
    soup = BeautifulSoup(response, "lxml")
    print(soup.select_one("h1[itemprop='name'] a").text)

if __name__ == '__main__':
    for lead_url in lead_urls:
        make_requests(lead_url)

    for single_link in linkList:
        make_ano_requests(single_link)

【讨论】：

嗨@SimonF，我试图遵守你的第一个建议，但问题是脚本在几秒钟内崩溃而没有产生任何结果。至于您的第二种方法，我已经在我之前的帖子中从您那里了解到。我只是想学习如何使用我上面尝试的方式发出后续请求。谢谢。
那么崩溃的原因是什么？ Traceback... 如果超时，您需要像 proxyerror 一样捕获它
@robots.txt 我还在等待回溯...我很确定你只是没有捕捉到新的异常（超时）
如果您向我提供如何在requests.get(url, proxies=proxy, timeout=1) 中设置timeout=1 的演示，我愿意接受您的回答并奖励赏金，因为每次我尝试使用这种方法时都会得到@ 987654329@ 几乎立即出错，即使它们在try/except 块中。我只在上面脚本的两个函数中将res = requests.get(url, proxies=proxy) 更改为res = requests.get(url, proxies=proxy,timeout=1) 并运行了@SimonF。
@robots.txt 这被认为是一种很好的做法，在这种情况下，它是 None ，因为这是 python 中的默认值。但是考虑如果你想返回标题和链接而不是使用全局变量和打印，那么你只需要添加所说的东西的返回，而不是已经存在的返回

【解决方案2】：

您可以使用 asyncio 和 aiohttp 加快代理过滤过程。像这样的：

import aiohttp
import asyncio
import random
import requests
from random import choice
from bs4 import BeautifulSoup
from urllib.parse import urljoin


base_url = 'https://stackoverflow.com/questions/tagged/web-scraping'
lead_urls = [f'https://stackoverflow.com/questions/tagged/web-scraping?sort=' \
             f'newest&page={page}&pagesize=50' for page in range(1, 5)]
linkList = []

proxyVault = ['103.110.37.244:36022', '180.254.218.229:8080', '110.74.197.207:50632', '1.20.101.95:49001',
              '200.10.193.90:8080', '173.164.26.117:3128', '103.228.118.66:43002', '178.128.231.201:3128',
              '1.2.169.54:55312', '181.52.85.249:31487', '97.64.135.4:8080', '190.96.214.123:53251',
              '52.144.107.142:31923', '45.5.224.145:52035', '89.218.22.178:8080', '192.241.143.186:80',
              '113.53.29.218:38310', '36.78.131.182:39243']

def make_ano_requests(url):
    proxy_url = choice(proxyVault)
    proxy = {'https': f'http://{proxy_url}'}
    try:
        res = requests.get(url, proxies=proxy, timeout=5)
        get_title(res.text)
    except requests.exceptions.ProxyError:
        if proxy_url in proxyVault:
            proxyVault.remove(proxy_url)
            print(f'kicked out bad proxy by second func: {proxy_url}')
        return make_ano_requests(url)


def get_title(response):
    soup = BeautifulSoup(response, "lxml")
    print(soup.select_one("h1[itemprop='name'] a").text)


async def fetch(session, url, proxy_url):
    proxy = f'http://{proxy_url}'

    try:
        async with session.get(url, proxy=proxy) as response:
            return await response.text()
    except aiohttp.client_exceptions.ClientProxyConnectionError:
        print(f'kicked out bad proxy by first func: {proxy_url}')
        proxyVault.remove(proxy_url)


async def make_requests():
    tasks = []
    async with aiohttp.ClientSession() as session:
        for proxy in proxyVault:
            tasks.append(fetch(session, base_url, proxy))

        responses = await asyncio.gather(*tasks)

        print(f'Usefull proxies: {proxyVault}')
        for res in responses:
            if res:
                soup = BeautifulSoup(res.text, "lxml")
                linkList.extend(
                    [urljoin(base_url, item.get("href")) for item in soup.select(".summary .question-hyperlink")])


if __name__ == '__main__':
    loop = asyncio.get_event_loop()
    loop.run_until_complete(make_requests())

    for single_link in linkList:
        make_ano_requests(single_link)

【讨论】：