用户定义的函数在运行 For 循环之前返回空数据帧

Question

我知道与我的问题非常相似的问题已经被问过很多次，但我几乎已经全部复习了，无法解决我自己的代码，所以我希望有人能给出答案。

我正在尝试遍历 csv 下载并将它们附加到一个调用 for 循环的用户定义函数中。我添加了打印行，以便我可以看到函数失败的地方。该函数打印 1、2、3 并返回空的数据框 df。

为什么 for 循环被跳过并且 df 返回空？当在用户设计的功能之外运行时，它可以完美运行。

提前致谢！

    # LoadPackages
from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.by import By
import pandas as pd

# ==================================================    download spp tie flows
# set directories
directory = r"E:/Data/sophie/line vision/sources/spp public/downloaded/"
driverPath = r"/Users/sophi/Downloads/chromedriver_win32/chromedriver.exe"

# define urls
page_url = "https://marketplace.spp.org/pages/historical-tie-flow"
prefix_download_url = ("https://marketplace.spp.org/file-browser-api/download/" +
                       "historical-tie-flow?path=%2F")

xpath = "//*[@id='main-content']/div/div/div[2]/div/div[3]/div/ul/li/a/span[@class='fname name' and contains(text(), '2021')]"

driver = webdriver.Chrome(ChromeDriverManager().install())

def download_and_append(page_url, prefix_download_url, xpath) :
    
    driver.get(page_url)
    print(1)
    
    # create empty dataframe to append to
    df = pd.DataFrame()
    print(2)
    
    # retrieve data from page
    elements = driver.find_elements(By.XPATH, xpath)
    print(3)
    
    for element in elements:
        
        index = element.text.split()[0]
        print(index)
        data = pd.read_csv(prefix_download_url + index)
        print(4)
        
        # clean dataframe and concatenate to df
        df = pd.concat([df, data])
        print(5)
        
    return df

hourly_tie_flows_2021 = download_and_append(page_url, prefix_download_url, xpath)
hourly_tie_flows_2021

# ========================================================== export data

hourly_tie_flows_2021.to_csv(directory + "/spp_tie_flows_by_ba_2021.csv")`

Answer 1

简短回答（添加一个睡眠让 javascript 完全加载）：

from time import sleep


sleep(2)
elements = driver.find_elements(By.XPATH, xpath)

更长的答案（仅使用 Selenium 获取 url。使用 requests 和 concurrent.futures 并发下载文件）：

import concurrent.futures
import os
from io import StringIO
from time import sleep

import pandas as pd
import requests
from selenium import webdriver
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager


class Scrape:
    def __init__(self, **kwargs):
        self.year = kwargs.get("year")
        self.urls = self.get_urls()
        self.output_file_path = r"E:/Data/sophie/line vision/sources/spp public/downloaded/"
        self.driver_path = r"/Users/sophi/Downloads/chromedriver_win32/chromedriver.exe"

    def get_urls(self) -> list:
        root_url = "https://marketplace.spp.org"
        page_url = f"{root_url}/pages/historical-tie-flow"
        download_url = f"{root_url}/file-browser-api/download/historical-tie-flow?path=%2F"

        driver = webdriver.Chrome(ChromeDriverManager().install())
        driver.get(page_url)

        sleep(2)
        elements = driver.find_elements(By.XPATH, "//*[contains(text(),'" + f"{self.year}" + "')]")

        return [f"{download_url}{x.text.split()[0]}" for x in elements]

    def download_files(self) -> None:
        with concurrent.futures.ProcessPoolExecutor(max_workers=os.cpu_count()) as executor:
            (pd
             .concat(executor.map(self.get_data, self.urls))
             .sort_values("GMTTIME")
             .reset_index(drop=True)
             ).to_csv(f"{self.output_file_path}/spp_tie_flows_by_ba_{self.year}.csv")

    @staticmethod
    def get_data(url: str) -> pd.DataFrame:
        with requests.Session() as request:
            response = request.get(url)
        if response.status_code != 200:
            print(response.raise_for_status())

        return pd.read_csv(StringIO(response.text), sep=",")


if __name__ == "__main__":
    Scrape(year=2021).download_files()