bs4 再次从网站保存到文本文件

Question

我现在正在学习如何从网站中提取数据，并设法获得了大量信息。但是，对于我的下一个网站，由于某些未知原因，我失败了，因为没有任何内容保存到文本文件中，也没有得到任何打印输出。这是我的一段代码：

import json
import urllib.request
from bs4 import BeautifulSoup
import requests


url = 'https://www.jaffari.org/'
request = urllib.request.Request(url,headers={'User-Agent': 'Mozilla/5.0'})
response = urllib.request.urlopen(request)
html = response.read()
soup = BeautifulSoup(html.decode("utf-8"), "html.parser")

table = soup.find('div', attrs={"class":"textwidget"})
name = table.text.encode('utf-8').strip()

with open('/home/pi/test.txt', 'w') as outfile:
    json.dump(name, outfile)
print (name)

有人可以帮忙吗？

Answer 1

祈祷时间是由java-scripts渲染的，所以你需要使用selenium之类的浏览器工具来加载页面，然后使用漂亮的汤来获取数据。

您需要从此链接下载兼容的ChromeDriver，并按照我提供的方式传递 chrome 驱动程序路径。

此处的代码用于获取name 和prayer times 并保存在text 文件中。

from selenium.webdriver.chrome.options import Options
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup
import re

options = Options()
# Runs Chrome in headless mode.
options.add_argument("--headless")
#path of the chrome driver
driver=webdriver.Chrome(executable_path="D:\Software\chromedriver.exe", chrome_options=options)
driver.headless=True
driver.get('https://www.jaffari.org/')
WebDriverWait(driver,20).until(EC.visibility_of_element_located((By.CSS_SELECTOR,'div.sidebar-widget.widget_text>div>table')))
print("Data rendered successfully!!!")
#Get the page source
html=driver.page_source
soup=BeautifulSoup(html,'html.parser')
#Close the driver
driver.close()

with open('testPrayers.txt', 'w') as outfile:
     for row in soup.select("div.sidebar-widget.widget_text>div>table tr"):
         name=row.select("td")[0].text.strip()
         time=re.findall('(\d{1,2}:?\d{1,2}\W[A|P]M$)',row.select("td")[1].text.strip())
         outfile.write(name + " " + time[0] + "\n")
         print(name + " " + time[0])

outfile.close()
print('Done')

---

用不同的文件名更新数据。

---

from selenium.webdriver.chrome.options import Options
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup
import re

options = Options()
# Runs Chrome in headless mode.
options.add_argument("--headless")
#path of the chrome driver
driver=webdriver.Chrome(executable_path="D:\Software\chromedriver.exe", chrome_options=options)
driver.headless=True
driver.get('https://www.jaffari.org/')
WebDriverWait(driver,20).until(EC.visibility_of_element_located((By.CSS_SELECTOR,'div.sidebar-widget.widget_text>div>table')))
print("Data rendered successfully!!!")
#Get the page source
html=driver.page_source
soup=BeautifulSoup(html,'html.parser')
#Close the driver
driver.close()

for row in soup.select("div.sidebar-widget.widget_text>div>table tr"):
    name=row.select("td")[0].text.strip()
    time=re.findall('(\d{1,2}:?\d{1,2}\W[A|P]M$)',row.select("td")[1].text.strip())

    print(name + " " + time[0])
    with open(name+'.txt', 'w') as outfile:
        outfile.write(time[0])
        outfile.close()


print('Done')

Answer 2

name 变量必须是字符串而不是字节对象。试试

with open('/home/pi/test.txt', 'w') as outfile:
    json.dump(name.decode(), outfile)
print (name.decode())

希望对您有所帮助。