尝试爬行，但只有一个有效，其他无效

Question

import telegram
from telegram.ext import Updater, MessageHandler, Filters
import requests
from bs4 import BeautifulSoup

token=''
chat_id = 
bot = telegram.Bot(token)
updater = Updater(token=token)
dispatcher = updater.dispatcher
updater.start_polling()

bot.sendMessage(chat_id=chat_id, text='Welcome!')


def handler(update, context):
    text = update.message.text
    chat_id = update.message.chat_id

    if 'bot' in text:
        bot.send_message(chat_id=chat_id, text='Did you call me?')

    if text.startswith("hi"):
        bot.send_message(chat_id=chat_id, text='Hello')

    if text.startswith("eth"):
        html = requests.get("https://coinmarketcap.com/currencies/ethereum/")
        soup = BeautifulSoup(html.text, 'html.parser')
        price = soup.find('td').text
        change = soup.find('p',{'class': 'sc-1eb5slv-0 sc-1siv958-1 lfjNAJ'}).text
        bot.send_message(chat_id=chat_id, text='CURRENT ETH PRICE' + '\n' + price + '\n' + 'IN USD' + '\n' + 'LAST 24 HOURS' + '\n' + change)

    if text.startswith("DFM"):
        html = requests.get("https://etherscan.io/token/0x0ccD5DD52Dee42B171a623478e5261C1eaaE092A")
        soup = BeautifulSoup(html.text, 'html.parser')
        priced = soup.find('div', {'class': 'col-6'}).find_all('span', {'class':'d-block'}).text
        pricede = soup.find('span', {'class': 'small text-secondary text-nowrap'}).text
        circu = soup.find('button', {'class': 'u-label u-label--sm u-label--value u-label--text-dark u-label--secondary rounded'}).text
        bot.send_message(chat_id=chat_id, text='CURRENT DFM token PRICE' + '\n' + priced + '\n' + 'IN USD' + '\n' + pricede + '\n' + 'IN ETH' + '\n' + 'CIRCULATING SUPPLY MARKET CAP' + circu)
            
echo_handler = MessageHandler(Filters.text, handler)
dispatcher.add_handler(echo_handler)

第一个使用 eth 爬行确实有效，但另一个，DFM，不起作用。我无法找出问题所在。我试图从 https://ethereum.org/en/get-eth/ 但我也做不到。

##price = soup.find('div', {'class': 'col-6'}).text
##price = soup.find("div",attrs={"class":"col-6"}).find_all("span", attrs={"class":"d-block"}).text
##price = soup.select_one('div.col-6 span.d-block').text

这些是我以其他方式尝试的其他代码。我不知道需要什么信息来了解我的情况，我使用 Python Idle。

Answer 1

对于DFM，我需要进行两项更改。

首先：它需要find()而不是find_all()

priced = soup.find('div', {'class': 'col-6'}).find('span', {'class':'d-block'}).text

第二：它需要requests中的标题User-Agent。

它甚至可以是不完整的User-Agent，比如'Mozilla/5.0'

headers = {'User-Agent': 'Mozilla/5.0'}
#headers = {'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:85.0) Gecko/20100101 Firefox/85.0'}

url = "https://etherscan.io/token/0x0ccD5DD52Dee42B171a623478e5261C1eaaE092A"

html = requests.get(url, headers=headers)

---

if text.startswith("DFM"):
    headers = {'User-Agent': 'Mozilla/5.0'}# (X11; Ubuntu; Linux x86_64; rv:85.0) Gecko/20100101 Firefox/85.0'}
    url = "https://etherscan.io/token/0x0ccD5DD52Dee42B171a623478e5261C1eaaE092A"
    html = requests.get(url, headers=headers)
    soup = BeautifulSoup(html.text, 'html.parser')
    priced = soup.find('div', {'class': 'col-6'}).find('span', {'class':'d-block'}).text
    pricede = soup.find('span', {'class': 'small text-secondary text-nowrap'}).text
    circu = soup.find('button', {'class': 'u-label u-label--sm u-label--value u-label--text-dark u-label--secondary rounded'}).text
    bot.send_message(chat_id=chat_id, text='CURRENT DFM token PRICE' + '\n' + priced + '\n' + 'IN USD' + '\n' + pricede + '\n' + 'IN ETH' + '\n' + 'CIRCULATING SUPPLY MARKET CAP' + circu)

---

编辑：

最少的工作代码

import requests
from bs4 import BeautifulSoup

dfm_token = "0x0ccD5DD52Dee42B171a623478e5261C1eaaE092A"

headers = {'User-Agent': 'Mozilla/5.0'}# (X11; Ubuntu; Linux x86_64; rv:85.0) Gecko/20100101 Firefox/85.0'}
url = f"https://etherscan.io/token/{dfm_token}"
response = requests.get(url, headers=headers)

soup = BeautifulSoup(response.text, 'html.parser') # 'html5lib')

price = soup.find('div', {'class': 'col-6'}).find('span', {'class': 'd-block'}).text.strip()
price_USD, price_ETH = price.split('@')

price_USD = price_USD.strip()[1:]   # skip `$` at the beginning
price_ETH = price_ETH.strip()[:-4]  # skip `Eth` at the end

cap = soup.find('button', {'class': 'u-label u-label--sm u-label--value u-label--text-dark u-label--secondary rounded'}).text.strip()

print(f'TOKEN: {dfm_token}\nUSD: {price_USD}\nETH: {price_ETH}\nCAP: {cap}')

结果：

TOKEN: 0x0ccD5DD52Dee42B171a623478e5261C1eaaE092A
USD: 0.0716
ETH: 0.000045
CAP: $143,150,000.00

---

编辑：

页面https://ethereum.org/en/get-eth/ 使用JavaScript 向HTML 添加值，但requests、BeautifulSoup 不要运行JavaScript。

但是在Chrome/Firefox 中使用DevTools（标签：Network，过滤器：XHR）我发现JavaScript 使用的url 将这些值读取为JSON - 我可以将它与requests 获取它们 - 没有 BeautifulSoup 的事件

import requests

url = 'https://api.coingecko.com/api/v3/simple/price?ids=ethereum&vs_currencies=usd&include_24hr_change=true'

response = requests.get(url)
data = response.json() 

price = data['ethereum']['usd']
change = data['ethereum']['usd_24h_change']

print(f'price: {price}')
print(f'change: {change:.2f}%')

结果

price: 1578.36
change: -2.48%

---

https://www.coingecko.com/en/api 上似乎有免费 API，您可以使用它来获取价格而无需抓取 - 仅使用 requests 而不使用 BeautifulSoup