在 python 中使用 src 下载图像会产生空图像

Question

我的脚本有点工作，但它保存的文件是空的。有任何想法吗？请原谅我在顶部所有未使用的导入！我尝试了很多不同的方法来做到这一点。在这里，我正在使用 selenium 拉动 img。然后 SRC 通过循环迭代并转换为字节，以便可以使用 os.path 写入它们。我怀疑该网站可能正在保护自己免受此类抓取？

from bs4 import BeautifulSoup
from urllib.request import Request, urlopen
import os
import urllib
import urllib3
import time
from selenium.webdriver.support.ui import WebDriverWait 
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
import requests


driver = webdriver.Firefox()
options = Options()
options.headless = True
driver = webdriver.Firefox(options=options)
driver.get("https://superrare.com/features/the-intersection-of-machine-and-artist")
time.sleep(2)                                                                                                            

#the element with longest height on page
ele=driver.find_element("xpath", '//div[@id="root"]')
total_height = ele.size["height"]+8000
time.sleep(2)  
driver.set_window_size(1920, total_height) 
time.sleep(2)



imgsrc2 = WebDriverWait(driver,50).until(EC.presence_of_all_elements_located((By.XPATH, "//img")))

time.sleep(5)
download_folder = "/Users/rcastong/Desktop/imgs"
if not os.path.exists(download_folder):
    os.makedirs(download_folder)

for i in imgsrc2:
    imgsrc = i.get_attribute("src")
    str_img = str.encode(imgsrc)
    with open(os.path.join(download_folder, os.path.basename(imgsrc)), "wb") as f:
        f.write(str_img)
     

Answer 1

您忘记使用requests 从服务器获取数据

    response = requests.get(img_src)
    data = response.content
    
    with open(fullname, "wb") as f:
        f.write(data)

---

最小的工作示例。

它适用于我的一些第一张图片。也许其他图片需要更长的sleep() 或者它需要滚动到底部以通过 JavaScript 加载所有src。

import os
import time
import requests
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.support.ui import WebDriverWait 
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

options = Options()
options.headless = True
driver = webdriver.Firefox(options=options)

driver.get("https://superrare.com/features/the-intersection-of-machine-and-artist")
time.sleep(2)                                                                                                            

#the element with longest height on page
root = driver.find_element("xpath", '//div[@id="root"]')
total_height = root.size["height"] + 8000
print('total_height:', total_height)
time.sleep(2)

driver.set_window_size(1920, total_height) 
time.sleep(2)

imgs = WebDriverWait(driver, 50).until(EC.presence_of_all_elements_located((By.XPATH, "//img")))
time.sleep(5)

print('len(imgs):', len(imgs))

download_folder = "/Users/rcastong/Desktop/imgs"

# it will create only if not exists
os.makedirs(download_folder, exist_ok=True)

for number, item in enumerate(imgs, 1):
    print('---', number, '---')

    img_src = item.get_attribute("src")
    print('from:', img_src)

    fullname = os.path.join(download_folder, os.path.basename(img_src))
    print('  to:', fullname)
    
    response = requests.get(img_src)
    data = response.content
    
    with open(fullname, "wb") as f:
        f.write(data)