如何使用 Python、Selenium 和 BeautifulSoup 对 JSP 进行网络抓取？

Question

我是使用 Python 进行网络抓取的绝对初学者。 我正在尝试从此 URL 中提取 ATM 的位置：

https://www.visa.com/atmlocator/mobile/index.jsp#(page:results,params:(query:'Tokyo,%20Japan'))

使用以下代码。

#Script to scrape locations and addresses from VISA's ATM locator


# import the necessary libraries (to be installed if not available):

from selenium import webdriver
from bs4 import BeautifulSoup
import pandas as pd


#ChromeDriver
#(see https://chromedriver.chromium.org/getting-started as reference)

driver = webdriver.Chrome("C:/Users/DefaultUser/Local Settings/Application Data/Google/Chrome/Application/chromedriver.exe")

offices=[] #List to branches/ATM names
addresses=[] #List to branches/ATM locations
driver.get("https://www.visa.com/atmlocator/mobile/index.jsp#(page:results,params:(query:'Tokyo,%20Japan'))") 


content = driver.page_source
soup = BeautifulSoup(content, features = "lxml")


#the following code extracts all the content inside the tags displaying the information requested

for a in soup.findAll('li',attrs={'class':'visaATMResultListItem'}): 
    name=a.find('li', attrs={'class':'data-label'}) 
    address=a.find('li', attrs={'class':'data-label'}) 
    offices.append(name.text)
    addresses.append(address.text)


#next row defines the dataframe with the results of the extraction

df = pd.DataFrame({'Office':offices,'Address':addresses})


#next row displays dataframe content

print(df)


#export data to .CSV file named 'branches.csv'
with open('branches.csv', 'a') as f:
    df.to_csv(f, header=True)

该脚本一开始似乎可以正常工作，因为 Chromedriver 启动并在浏览器中显示所需的结果，但没有返回任何结果：

Empty DataFrame
Columns: [Office, Address]
Index: []
Process finished with exit code 0

也许我在选择选择器时犯了错误？

非常感谢您的帮助

Answer 1

问题出在定位器上，使用

for a in soup.findAll('li',attrs={'class':'visaATMResultListItem'}): 
    name = a.find('p', attrs={'class':'visaATMPlaceName '}) 
    address = a.find('p', attrs={'class':'visaATMAddress'}) 
    offices.append(name.text)
    addresses.append(address.text)

Answer 2

from selenium import webdriver
from selenium.webdriver.firefox.options import Options
import time
from bs4 import BeautifulSoup
import csv

options = Options()
options.add_argument('--headless')

driver = webdriver.Firefox(options=options)
driver.get("https://www.visa.com/atmlocator/mobile/index.jsp#(page:results,params:(query:'Tokyo,%20JAPAN'))")
time.sleep(2)

soup = BeautifulSoup(driver.page_source, 'html.parser')

na = []
addr = []
for name in soup.findAll("a", {'class': 'visaATMPlaceLink'}):
    na.append(name.text)
for add in soup.findAll("p", {'class': 'visaATMAddress'}):
    addr.append(add.get_text(strip=True, separator=" "))

with open('out.csv', 'w', newline="") as f:
    writer = csv.writer(f)
    writer.writerow(['Name', 'Address'])
    for _na, _addr in zip(na, addr):
        writer.writerow([_na, _addr])

driver.quit()

输出：Click-Here