使用 Python 请求库进行 Web 抓取，给出 429 客户端错误

Question

我正在尝试使用 Python 中的请求库来抓取网站数据，并以 429 Client Error: Too Many Requests for URL 结束，而我什至没有以编程方式访问过 URL。

需要帮助来克服这个错误，在此先感谢。

下面是代码：

import requests
import json

if __name__ == '__main__':
BASE_URL = f"https://groww.in/mutual-funds"
LISTING_URL = f"https://groww.in/slr/v1/search/derived/scheme"

HEADERS = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, '
                         'like Gecko) '
                         'Chrome/80.0.3987.149 Safari/537.36',
           'accept-language': 'en,gu;q=0.9,hi;q=0.8', 'accept-encoding': 'gzip, deflate, br'}

PARAMS = {'available_for_investment': 'true', 'doc_type': 'scheme', 'page': 0, 'plan_type': 'Direct',
          'size': 16, 'sort_by': 0}

try:
    session = requests.Session()

    print('FETCHING & SETTING COOKIES...')
    request = session.get(BASE_URL, headers=HEADERS, timeout=20)
    cookies = dict(request.cookies)

    response = session.get(url=LISTING_URL, headers=HEADERS, params=PARAMS, timeout=20,
                           cookies=cookies)

    response.raise_for_status()
except requests.exceptions.HTTPError as err:
    raise SystemExit(err)

dajs = json.loads(response.text)

Answer 1

这是可行的解决方案。

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from shutil import which
from scrapy.selector import Selector
from selenium_stealth import stealth
from time import sleep
 
 
class ListSpider(CrawlSpider):
    name = 'lists'
    allowed_domains = ['groww.in']
    start_urls = ['https://groww.in/mutual-funds/filter']
 
    rules = (
        Rule(LinkExtractor(
            restrict_xpaths='//*[@class="s11ResultSec"]/div/a'), callback='parse_item', follow=False),
    )
 
    def __init__(self):
        CrawlSpider.__init__(self)
        #options = Options()
        # options.add_argument("start-maximized")
        chrome_path = which("chromedriver")
        self.driver = webdriver.Chrome(executable_path=chrome_path)  # chrome_options=options,
 
        stealth(self.driver,
                user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
                languages=["en-US", "en"],
                vendor="Google Inc.",
                platform="Win32",
                webgl_vendor="Intel Inc.",
                renderer="Intel Iris OpenGL Engine",
                fix_hairline=False)
        self.driver.get('https://groww.in/mutual-funds/filter')
        self.driver.implicitly_wait(30)
        print(dir(self.driver))
        self.driver.maximize_window()
 
    def parse_item(self, response):
        self.driver.get(response.url)
        sleep(5)
 
        # Get scroll height after first time page load
        last_height = self.driver.execute_script(
            "return document.body.scrollHeight")
        while True:
            # Scroll down to bottom
            self.driver.execute_script(
                "window.scrollTo(0, document.body.scrollHeight);")
    # Wait to load page
            sleep(3)
    # Calculate new scroll height and compare with last scroll height
            new_height = self.driver.execute_script(
                "return document.body.scrollHeight")
            if new_height == last_height:
                break
            last_height = new_height
 
        sel = Selector(text=self.driver.page_source)
        yield{'Name': sel.xpath('//h1[@class="mh122FundName"]/text()').get() 
              }
        self.driver.close()



                     OUTPUT:

A part of complete output:
item_scraped_count': 15616

2021-07-11 19:15:14 [scrapy.core.scraper] DEBUG: Scraped from <200 https://groww.in/mutual-funds/filter>
{'name': 'ICICI Prudential Short Term Fund Direct Plan Growth'}
2021-07-11 19:15:14 [scrapy.core.scraper] DEBUG: Scraped from <200 https://groww.in/mutual-funds/filter>
{'name': 'Axis Midcap Direct Plan Growth'}
2021-07-11 19:15:14 [scrapy.core.scraper] DEBUG: Scraped from <200 https://groww.in/mutual-funds/filter>
{'name': 'ICICI Prudential Technology Direct Plan Growth'}
2021-07-11 19:15:14 [scrapy.core.scraper] DEBUG: Scraped from <200 https://groww.in/mutual-funds/filter>
{'name': 'Axis Small Cap Fund Direct Growth'}
2021-07-11 19:15:14 [scrapy.core.scraper] DEBUG: Scraped from <200 https://groww.in/mutual-funds/filter>
{'name': 'Tata Digital India Fund Direct Growth'}
2021-07-11 19:15:14 [scrapy.core.scraper] DEBUG: Scraped from <200 https://groww.in/mutual-funds/filter>
{'name': 'Parag Parikh Flexi Cap Fund Direct Growth'}
2021-07-11 19:15:14 [scrapy.core.scraper] DEBUG: Scraped from <200 https://groww.in/mutual-funds/filter>
{'name': 'Axis Bluechip Fund Direct Plan Growth'}
2021-07-11 19:15:14 [scrapy.core.scraper] DEBUG: Scraped from <200 https://groww.in/mutual-funds/filter>
{'name': 'IIFL Focused Equity Fund Direct Growth'}
2021-07-11 19:15:14 [scrapy.core.scraper] DEBUG: Scraped from <200 https://groww.in/mutual-funds/filter>
{'name': 'Nippon India Small Cap Fund Direct  Growth'}
2021-07-11 19:15:14 [scrapy.core.scraper] DEBUG: Scraped from <200 https://groww.in/mutual-funds/filter>
{'name': 'Mirae Asset Tax Saver Fund Direct Growth'}
2021-07-11 19:15:14 [scrapy.core.scraper] DEBUG: Scraped from <200 https://groww.in/mutual-funds/filter>
{'name': 'SBI Small Cap Fund Direct Growth'}
2021-07-11 19:15:14 [scrapy.core.scraper] DEBUG: Scraped from <200 https://groww.in/mutual-funds/filter>
{'name': 'UTI Flexi Cap Fund Direct Growth'}
2021-07-11 19:15:14 [scrapy.core.scraper] DEBUG: Scraped from <200 https://groww.in/mutual-funds/filter>
{'name': 'Nippon India Pharma Fund Direct Growth'}
2021-07-11 19:15:14 [scrapy.core.scraper] DEBUG: Scraped from <200 https://groww.in/mutual-funds/filter>