使用 BeautifulSoup 遍历 URL

Question

我编写了一些代码来从https://www.horseracing.net/racecards 收集每个赛道的 URL。我还编写了一些代码来从每个赛道页面中抓取数据。

每一段代码都可以正常工作，但我无法创建一个 for 循环来遍历所有赛道 URL。

以下是抓取课程网址的代码：

import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

todays_racecard_url = 'https://www.horseracing.net/racecards'
base_url = "https://www.horseracing.net"
reqs = requests.get(todays_racecard_url)
content = reqs.text
soup = BeautifulSoup(content, 'html.parser')
course_urls = []

for h in soup.findAll('h3'):
    a = h.find('a')

    try:
        if 'href' in a.attrs:
            card_url = urljoin(base_url, a.get('href'))
            course_urls.append(card_url)
    except:
        pass

for card_url in course_urls:
    print(card_url)

这是抓取页面的代码：

import requests
from requests import get
from bs4 import BeautifulSoup
import pandas as pd
import numpy as np

url = "https://www.horseracing.net/racecards/fontwell/13-05-21"

results = requests.get(url)

soup = BeautifulSoup(results.text, "html.parser")

date = []
course = []
time = []
runner = []
tips = []
tipsters = []

runner_div = soup.find_all('div', class_='row-cell-right')

for container in runner_div:

    runner_name = container.h5.a.text
    runner.append(runner_name)

    tips_no = container.find('span', class_='tip-text number-tip').text if container.find('span', class_='tip-text number-tip') else ''
    tips.append(tips_no)

    tipster_names = container.find('span', class_='pointers-text currency-text').text if container.find('span', class_='pointers-text currency-text') else ''
    tipsters.append(tipster_names)

newspaper_tips = pd.DataFrame({
'Runners': runner,
'Tips': tips,
'Tipsters': tipsters,
})

newspaper_tips['Tipsters'] = newspaper_tips['Tipsters'].str.replace(' - ', '')

newspaper_tips.to_csv('NewspaperTips.csv', mode='a', header=False, index=False)

我如何加入他们以获得我正在寻找的结果？

Answer 1

可以这样组合：

import pandas as pd
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

todays_racecard_url = 'https://www.horseracing.net/racecards'
base_url = "https://www.horseracing.net"

req = requests.get(todays_racecard_url)
soup_racecard = BeautifulSoup(req.content, 'html.parser')
df = pd.DataFrame(columns=['Runners', 'Tips', 'Tipsters'])

for h in soup_racecard.find_all('h3'):
    a = h.find('a', href=True)    # only find tags with href present
    
    if a:
        url = urljoin(base_url, a['href'])
        print(url)
        results = requests.get(url)
        soup_url = BeautifulSoup(results.text, "html.parser")

        for container in soup_url.find_all('div', class_='row-cell-right'):
            runner_name = container.h5.a.text
            tips_no = container.find('span', class_='tip-text number-tip').text if container.find('span', class_='tip-text number-tip') else ''
            tipster_names = container.find('span', class_='pointers-text currency-text').text if container.find('span', class_='pointers-text currency-text') else ''
            row = [runner_name, tips_no, tipster_names]
            df.loc[len(df)] = row       # append the new row

df['Tipsters'] = df['Tipsters'].str.replace(' - ', '')
df.to_csv('NewspaperTips.csv', index=False)    

给你一个CSV开始：

Runners,Tips,Tipsters
Ajrad,2,NEWMARKET
Royal Tribute,1,The Times
Time Interval,1,Daily Mirror
Hemsworth,1,Daily Express
Ancient Times,,
Final Watch,,
Hala Joud,,
May Night,1,The Star
Tell'Em Nowt,,