从股票筛选器中检索多个页面

Question

我正在从股票筛选器中抓取数据。我可以使用 get_screener('111&r=1') 获得前 20 个结果的第一页。下一页将是 get_screener('111&r=21') 以获取接下来的 20 个结果。我尝试添加额外的获取请求，但它给了我错误，所以我不知道如何编码，所以它会给我额外的行。有谁知道这是怎么做的吗？

谢谢

 import requests 
    import pandas as pd
    
    headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}
    
    
    def get_screener(version):
        screen = requests.get(f'https://finviz.com/screener.ashx?v={version}&f=all', headers = headers).text
            
        tables = pd.read_html(screen)
        tables = tables[-2]
        tables.columns = tables.iloc[0]
        tables = tables[1:]
    
        return tables
    
    tables111 = get_screener('111&r=1')
    tables161 = get_screener('161&r=1')
    tables121 = get_screener('121&r=1')
    
    consolidatedtables = pd.merge(tables111,tables161,how='outer',left_on='Ticker',right_on='Ticker')
    consolidatedtables = pd.merge(consolidatedtables,tables121,how='outer',left_on='Ticker',right_on='Ticker')
    
    consolidatedtables.to_csv('test.csv')
    
    print(consolidatedtables)

我正在使用的新代码

import pandas as pd
import requests
import bs4
import time
import random

headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}

def get_screener(version):
    url = 'https://finviz.com/screener.ashx?v={version}&r={page}&f=all'
    page = 1
   
    screen = requests.get(url.format(version=version, page=page), headers=headers)
    soup = bs4.BeautifulSoup(screen.text, features='lxml')
    pages = int(soup.find_all('a', {'class': 'screener-pages'})[-1].text)
      
    data = []
    for page in range(1, 20 * pages, 20):
        print(version, page)
        screen = requests.get(url.format(version=version, page=page), headers=headers).text
        tables = pd.read_html(screen)
        tables = tables[-2]
        tables.columns = tables.iloc[0]
        tables = tables[1:]
        data.append(tables)
        time.sleep(random.random())
    return pd.concat(data).reset_index(drop=True).rename_axis(columns=None)
       
df = get_screener('111')
df.info()

Output
111 1
111 21
111 41
111 61
111 81
111 101
111 121
111 141
111 161
111 181
111 201
111 221
111 241
111 8321
111 8341
111 8361
111 8381
111 8401
111 8421
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 8439 entries, 0 to 8438
Data columns (total 11 columns):
 #   Column      Non-Null Count  Dtype 
---  ------      --------------  ----- 
 0   No.         8439 non-null   object
 1   Ticker      8439 non-null   object
 2   Company     8439 non-null   object
 3   Sector      8439 non-null   object
 4   Industry    8439 non-null   object
 5   Country     8439 non-null   object
 6   Market Cap  8439 non-null   object
 7   P/E         8439 non-null   object
 8   Price       8439 non-null   object
 9   Change      8439 non-null   object
 10  Volume      8439 non-null   object
dtypes: object(11)
memory usage: 725.4+ KB

Answer 1

试试：

import pandas as pd
import requests
import bs4
import time
import random

headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}

def get_screener(version):
    url = 'https://finviz.com/screener.ashx?v={version}&r={page}&f=all'
    page = 1
   
    screen = requests.get(url.format(version=version, page=page), headers=headers)
    soup = bs4.BeautifulSoup(screen.text)
    pages = int(soup.find_all('a', {'class': 'screener-pages'})[-1].text)
      
    data = []
    for page in range(1, 20 * pages, 20):
        print(version, page)
        screen = requests.get(url.format(version=version, page=page), headers=headers).text
        tables = pd.read_html(screen)
        tables = tables[-2]
        tables.columns = tables.iloc[0]
        tables = tables[1:]
        data.append(tables)
        time.sleep(random.random())
    return pd.concat(data).reset_index(drop=True).rename_axis(columns=None)
       
df = get_screener('111')
print(df)

输出：

>>> df
       No. Ticker                           Company              Sector  ...    P/E   Price  Change   Volume
0        1      A        Agilent Technologies, Inc.          Healthcare  ...  39.97  157.80   0.65%  1409104
1        2     AA                 Alcoa Corporation     Basic Materials  ...  13.82   59.36   0.08%  7509699
2        3   AAAU   Goldman Sachs Physical Gold ETF           Financial  ...      -   17.99   0.33%   101432
3        4    AAC      Ares Acquisition Corporation           Financial  ...  43.53    9.75   0.00%    40324
4        5   AACG             ATA Creativity Global  Consumer Defensive  ...      -    1.24   1.64%    29131
...    ...    ...                               ...                 ...  ...    ...     ...     ...      ...
8434  8435    ZWS  Zurn Water Solutions Corporation           Utilities  ...  20.30   36.07   0.28%   461593
8435  8436     ZY                     Zymergen Inc.     Basic Materials  ...      -    7.61   4.10%   668322
8436  8437   ZYME                    Zymeworks Inc.          Healthcare  ...      -   16.57  -0.18%   269486
8437  8438   ZYNE     Zynerba Pharmaceuticals, Inc.          Healthcare  ...      -    3.23   0.62%   372210
8438  8439   ZYXI                       Zynex, Inc.          Healthcare  ...  39.00   10.96   3.69%   291003

[8439 rows x 11 columns]

更新

是否可以取回我原来拥有的其他 2 个筛选器并合并结果？

重用你的代码：

tables111 = get_screener('111')
tables161 = get_screener('161')
tables121 = get_screener('121')
    
consolidatedtables = pd.merge(tables111,tables161,how='outer',left_on='Ticker',right_on='Ticker')
consolidatedtables = pd.merge(consolidatedtables,tables121,how='outer',left_on='Ticker',right_on='Ticker')
    
consolidatedtables.to_csv('test.csv')