HTTPError：HTTP 错误 429：WIKI 页面的请求过多

Question

即使脚本访问的页面数量甚至不是 50，我也会收到 429 错误。如何调整脚本以避免错误。用户代理对我的 Chrome 浏览器来说是正确的。我试图在请求之后加上time.sleep(10)，但这并没有帮助。我在 Windows 10 上使用 jupyter notebook。

def get_wiki_list_italian_movies(year):
    import sys, bs4, requests, textwrap, , time, re
    from textblob import TextBlob
    
    p = r'https://en.wikipedia.org/wiki/List_of_Italian_films_of_' + str(year)
    list_wiki, list_wiki_links = [], []

    header = {'User-agent':
               'mozilla/5.0 (windows nt 10.0; win64; x64) applewebkit/537.36 (khtml, like gecko) chrome/86.0.4240.111 safari/537.36'}
   
    request = requests.get(p, headers = header)              
    time.sleep(10)

    soup = bs4.BeautifulSoup(request.text, 'lxml')
    table = soup.find_all('table', class_='wikitable')
    
    for el in table:
        td = el.find_all('td')
        for t in td:
            i = t.find('i')
            if i:
                for a in i.find_all('a', href=True):
                    result = re.sub(r'[()]','',a['href'])
                    if '/w/index' in result:
                        list_wiki.append(re.sub(r'\(.*','',a['title']).strip() + ' ' + str(year))
                    else:
                        list_wiki_links.append('https://en.wikipedia.org'+result)

    for link in list_wiki_links:
        request = requests.get(link, headers = header)
        time.sleep(10)
        soup = bs4.BeautifulSoup(request.text, 'lxml')
        i_list = soup.find_all('i')
        
        for i in i_list:
            b = i.find('b')
            if b:
                t= b.text
                if len(t) > 4 and TextBlob(t).detect_language() == 'it':
                    list_wiki.append(t.strip() + ' ' + str(year))
             
    return sorted(list(set(list_wiki)))

def movies_wiki_list(years_span):
    ll = []
    for year in years_span:
        ll += get_wiki_list_italian_movies(year)
        time.sleep(10)
    return ll

italian_movies_1932_1933 = movies_wiki_list(range(1932, 1934))
italian_movies_1932_1933

这是错误：

HTTPError                                 Traceback (most recent call last)
<ipython-input-12-6a4bc670faa6> in <module>
     53     return ll
     54 
---> 55 italian_movies_1932_1933 = movies_wiki_list(range(1932, 1934))
     56 italian_movies_1932_1933

<ipython-input-12-6a4bc670faa6> in movies_wiki_list(years_span)
     49     ll = []
     50     for year in years_span:
---> 51         ll += get_wiki_list_italian_movies(year)
     52         time.sleep(10)
     53     return ll

<ipython-input-12-6a4bc670faa6> in get_wiki_list_italian_movies(year)
     41             if b:
     42                 t= b.text
---> 43                 if len(t) > 4 and TextBlob(t).detect_language() == 'it':
     44                     list_wiki.append(t.strip() + ' ' + str(year))
     45 

~\anaconda3\lib\site-packages\textblob\blob.py in detect_language(self)
    566         :rtype: str
    567         """
--> 568         return self.translator.detect(self.raw)
    569 
    570     def correct(self):

~\anaconda3\lib\site-packages\textblob\translate.py in detect(self, source, host, type_)
     70         data = {"q": source}
     71         url = u'{url}&sl=auto&tk={tk}'.format(url=self.url, tk=_calculate_tk(source))
---> 72         response = self._request(url, host=host, type_=type_, data=data)
     73         result, language = json.loads(response)
     74         return language

~\anaconda3\lib\site-packages\textblob\translate.py in _request(self, url, host, type_, data)
     90         if host or type_:
     91             req.set_proxy(host=host, type=type_)
---> 92         resp = request.urlopen(req)
     93         content = resp.read()
     94         return content.decode('utf-8')

~\anaconda3\lib\urllib\request.py in urlopen(url, data, timeout, cafile, capath, cadefault, context)
    220     else:
    221         opener = _opener
--> 222     return opener.open(url, data, timeout)
    223 
    224 def install_opener(opener):

~\anaconda3\lib\urllib\request.py in open(self, fullurl, data, timeout)
    529         for processor in self.process_response.get(protocol, []):
    530             meth = getattr(processor, meth_name)
--> 531             response = meth(req, response)
    532 
    533         return response

~\anaconda3\lib\urllib\request.py in http_response(self, request, response)
    638         # request was successfully received, understood, and accepted.
    639         if not (200 <= code < 300):
--> 640             response = self.parent.error(
    641                 'http', request, response, code, msg, hdrs)
    642 

~\anaconda3\lib\urllib\request.py in error(self, proto, *args)
    561             http_err = 0
    562         args = (dict, proto, meth_name) + args
--> 563         result = self._call_chain(*args)
    564         if result:
    565             return result

~\anaconda3\lib\urllib\request.py in _call_chain(self, chain, kind, meth_name, *args)
    500         for handler in handlers:
    501             func = getattr(handler, meth_name)
--> 502             result = func(*args)
    503             if result is not None:
    504                 return result

~\anaconda3\lib\urllib\request.py in http_error_302(self, req, fp, code, msg, headers)
    753         fp.close()
    754 
--> 755         return self.parent.open(new, timeout=req.timeout)
    756 
    757     http_error_301 = http_error_303 = http_error_307 = http_error_302

~\anaconda3\lib\urllib\request.py in open(self, fullurl, data, timeout)
    529         for processor in self.process_response.get(protocol, []):
    530             meth = getattr(processor, meth_name)
--> 531             response = meth(req, response)
    532 
    533         return response

~\anaconda3\lib\urllib\request.py in http_response(self, request, response)
    638         # request was successfully received, understood, and accepted.
    639         if not (200 <= code < 300):
--> 640             response = self.parent.error(
    641                 'http', request, response, code, msg, hdrs)
    642 

~\anaconda3\lib\urllib\request.py in error(self, proto, *args)
    567         if http_err:
    568             args = (dict, 'default', 'http_error_default') + orig_args
--> 569             return self._call_chain(*args)
    570 
    571 # XXX probably also want an abstract factory that knows when it makes

~\anaconda3\lib\urllib\request.py in _call_chain(self, chain, kind, meth_name, *args)
    500         for handler in handlers:
    501             func = getattr(handler, meth_name)
--> 502             result = func(*args)
    503             if result is not None:
    504                 return result

~\anaconda3\lib\urllib\request.py in http_error_default(self, req, fp, code, msg, hdrs)
    647 class HTTPDefaultErrorHandler(BaseHandler):
    648     def http_error_default(self, req, fp, code, msg, hdrs):
--> 649         raise HTTPError(req.full_url, code, msg, hdrs, fp)
    650 
    651 class HTTPRedirectHandler(BaseHandler):

HTTPError: HTTP Error 429: Too Many Requests

Answer 1

根据维基百科的 API 指南：API:Etiquette 和 API:FAQ。

对读取请求没有硬性限制，但我们要求您 考虑周到，尽量不要关闭网站。大多数系统管理员保留 如果您确实危及 他们网站的稳定性。

如果您以串行方式而不是并行方式提出请求（即等待 在发送新请求之前完成一个请求，这样 你永远不会同时提出一个以上的请求），那么你 绝对应该没问题。还可以尝试将内容合并到一个请求中（例如，在一个标题参数中使用多个标题，而不是为每个标题发出一个新请求

这表明您可以为每个 API 请求获取 50 个页面。

如果您需要离线内容，您也可以使用Data Dumps（可能有点过时，我认为这对您来说不是问题）。

如果您达到任何限制，您可以使用these 状态消息处理 API 调用中的错误和警告。

Answer 2

HTTP 429 错误具有误导性。错误不是来自 Wiki 服务器，而是来自 谷歌。 Google 对 textblob 的每日收费请求有限制。但是谷歌翻译 请求是免费的。所以我不得不安装 googletrans 并使用它。

!pip install googletrans
from googletrans import Translator
translator = Translator()

我不得不改变

TextBlob(t).detect_language() == 'it'

到

 translator.detect(t).lang == 'it'

现在即使我使用更大的搜索，即 10 年而不是 2 年。 我没有收到 429 错误。

Answer 3

你可以使用：

try:
   urlpage=urllib.request.urlopen('url')

except:
   time.sleep(1)
   urlpage=urllib.request.urlopen('url')

这对我有用。在我们收到任何错误后等待一段时间，然后再次尝试相同的请求