使用 Python Selenium 在内存中下载文件，而不是在磁盘中

Question

我有一堆脚本可以进行网页抓取、下载文件并使用 pandas 读取它们。这个过程必须部署在一个新的架构中，在这种架构中下载磁盘上的文件是不合适的，而是最好将文件保存在内存中并从那里用 pandas 读取它。出于演示目的，我在这里留下了一个从随机网站下载 excel 文件的网络抓取脚本：

import time
import pandas as pd
from io import StringIO, BytesIO
from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
from datetime import date, timedelta
from selenium.webdriver.common.keys import Keys
from selenium import webdriver
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By


pathDriver = #Path to chromedriver

driver  = webdriver.Chrome(executable_path=pathDriver)

url = 'https://file-examples.com/index.php/sample-documents-download/sample-xls-download/'

driver.get(url)
time.sleep(1)

file_link = driver.find_element_by_xpath('//*[@id="table-files"]/tbody/tr[1]/td[5]/a[1]')
file_link.click()

此脚本有效地将文件下载到我的下载文件夹中。我尝试的是在click() 方法之前和之后放置一个StringIO() 或BytesIO() 流，并读取与此类似的对象：

file_object = StringIO()
df = pd.read_excel(file_object.read())

但是file_object没有捕获文件，甚至文件仍然下载到我的磁盘中。

对此有什么建议吗？

Answer 1

您的问题可以通过添加selenium add_experimental_option来完成。 我还重新设计了您的代码以循环遍历表以提取 href 以将它们传递给 StringIO。没有使用此代码将文件下载到我的本地系统。

如果我遗漏了什么，请告诉我。

import pandas as pd
from time import sleep
from io import StringIO
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities

capabilities = DesiredCapabilities().CHROME

chrome_options = Options()
chrome_options.add_argument("--incognito")
chrome_options.add_argument("--disable-infobars")
chrome_options.add_argument("start-maximized")
chrome_options.add_argument("--disable-extensions")
chrome_options.add_argument("--disable-popup-blocking")

prefs = {
    'profile.default_content_setting_values':
     {
        'automatic_downloads': 0
  },

      'profile.content_settings.exceptions':
    {
        'automatic_downloads': 0
    }
  }

chrome_options.add_experimental_option('prefs', prefs)
capabilities.update(chrome_options.to_capabilities())

driver = webdriver.Chrome('/usr/local/bin/chromedriver', options=chrome_options)

url_main = 'https://file-examples.com/index.php/sample-documents-download/sample-xls-download/'

driver.get(url_main)

elements = driver.find_elements_by_xpath('//*[@id="table-files"]//td/a')
for element in elements:
   if str(element.get_attribute("href")).endswith('.xls'):
     file_object = StringIO(element.get_attribute("href"))
      xls_file = file_object.read()
      df = pd.read_excel(xls_file)
      print(df.to_string(index=False))

        First Name  Last Name  Gender        Country  Age        Date    Id
      1      Dulce      Abril  Female  United States   32  15/10/2017  1562
      2       Mara  Hashimoto  Female  Great Britain   25  16/08/2016  1582
      3     Philip       Gent    Male         France   36  21/05/2015  2587
      4   Kathleen     Hanner  Female  United States   25  15/10/2017  3549
      5    Nereida    Magwood  Female  United States   58  16/08/2016  2468
      6     Gaston      Brumm    Male  United States   24  21/05/2015  2554
      7       Etta       Hurn  Female  Great Britain   56  15/10/2017  3598
      8    Earlean     Melgar  Female  United States   27  16/08/2016  2456
      9   Vincenza    Weiland  Female  United States   40  21/05/2015  6548
      
      sleep(360)

这是一个使用 cmets 中提到的 RAMDISK 的示例。此选项不使用 selenium add_experimental_option 或 StringIO。

import fs
import pandas as pd
from time import sleep
from selenium import webdriver
from selenium.webdriver.chrome.options import Options

chrome_options = Options()
chrome_options.add_argument("--incognito")
chrome_options.add_argument("--disable-infobars")
chrome_options.add_argument("start-maximized")
chrome_options.add_argument("--disable-extensions")
chrome_options.add_argument("--disable-popup-blocking")

driver = webdriver.Chrome('/usr/local/bin/chromedriver', options=chrome_options)

url_main = 'https://file-examples.com/index.php/sample-documents-download/sample-xls-download/'

driver.get(url_main)

urls_to_process = []
elements = driver.find_elements_by_xpath('//*[@id="table-files"]//td/a')

# Create RAMDISK
mem_fs = fs.open_fs('mem://')
mem_fs.makedir('hidden_dir')

for element in elements:
  if str(element.get_attribute("href")).endswith('.xls'):
     with mem_fs.open('hidden_dir/file1.csv', 'w') as in_file:
        in_file.write(element.get_attribute("href"))
        in_file.close()
     with mem_fs.open('hidden_dir/file1.csv', 'r') as out_file:
        df = pd.read_excel(out_file.read())
        print(df.to_string(index=False))
        # same output as above
        sleep(360)

Answer 2

IMO，显然没有必要使用 selenium。仅使用 requests + beautifulsoup + pandas 即可。（这比使用 selenium 快得多，并且需要更少的代码）。

代码如下：

from io import BytesIO
import requests
from bs4 import BeautifulSoup
import pandas as pd

response = requests.get("https://file-examples.com/index.php/sample-documents-download/sample-xls-download/")
soup = BeautifulSoup(response.text, "html.parser")
# get the download link
file_link = soup.select_one(".file-link > a").get("href")

# download it in memory
bytes_of_file = requests.get(file_link).content
df = pd.read_excel(BytesIO(bytes_of_file))
print(df)

结果：

   0 First Name  Last Name  Gender        Country  Age        Date    Id
0  1      Dulce      Abril  Female  United States   32  15/10/2017  1562
1  2       Mara  Hashimoto  Female  Great Britain   25  16/08/2016  1582
2  3     Philip       Gent    Male         France   36  21/05/2015  2587
3  4   Kathleen     Hanner  Female  United States   25  15/10/2017  3549
4  5    Nereida    Magwood  Female  United States   58  16/08/2016  2468
5  6     Gaston      Brumm    Male  United States   24  21/05/2015  2554
6  7       Etta       Hurn  Female  Great Britain   56  15/10/2017  3598
7  8    Earlean     Melgar  Female  United States   27  16/08/2016  2456
8  9   Vincenza    Weiland  Female  United States   40  21/05/2015  6548

而且这不会下载任何 excel 文件。