BeautifulSoup 连接两个表（行）以生成 csv 文件

Question

from urllib.request import urlopen
from bs4 import BeautifulSoup
import csv

html = urlopen(
    "https://www.accessdata.fda.gov/scripts/drugshortages/default.cfm")
bsObj = BeautifulSoup(html, "lxml")
table = bsObj.find('table', id="cont")
rows = table.findAll("tr")


links =  [a['href'] for a in table.find_all('a', href=True) if a.text]
new_links = []
for link in links:
    new_links.append(("https://www.accessdata.fda.gov/scripts/drugshortages/"+link).replace(" ", "%20"))
href_rows = []
for link in new_links:   
    link = link.replace("®", "%C2%AE")
    html = urlopen(link)
    bsObj_href = BeautifulSoup(html, "lxml")
    #bsObj_href = BeautifulSoup (html.decode('utf-8', 'ignore'))
    div_href = bsObj_href.find("div",{"id":"accordion"})
    href_rows.append(div_href.findAll("tr"))


csvFile = open("drug_shortage.csv", 'wt', newline='')
writer = csv.writer(csvFile)
try:
    for row in rows:
        csvRow = []
        for cell in row.findAll(['td', 'th']):
            csvRow.append(cell.get_text())
        writer.writerow(csvRow)

finally:
    csvFile.close()

你好，所以我创建了两个这样的行。如果你去这个网站https://www.accessdata.fda.gov/scripts/drugshortages/default.cfm 他们有药物名称和状态列，当您单击药物名称时，您可以找到另外四个列。我喜欢按顺序组合在一起（基于药物名称） 所以这将是药物名称、状态、介绍、可用性和估计的短缺持续时间、相关信息、短缺原因（根据 FDASIA）。 但目前的代码只生成第一个（药物名称，状态）。 我试过了

for row in rows,rows_href:

然后我得到 AttributeError: ResultSet object has no attribute 'findAll'。我得到同样的错误

for row in rows_href:

任何建议我如何生成我想要的？

Answer 1

你的代码太乱了。

您获取所有行，接下来是所有链接，然后您尝试获取所有其他信息，但这样您无法控制要加入行的值。最大的问题是当某些行在子页面上没有数据并且您的所有数据将移动到上一行时。

您应该从主页上的表格中获取所有行，然后使用for-loop 分别处理每一行以仅获取该单行的其他元素 - 仅读取该行的链接，仅从子页面获取该行的数据等，并将该行的所有数据作为子列表[name, status, link, presentation, availability, related, reason] 放在列表中。之后，您将获得下一项工作，并且只使用下一行的数据。

顺便说一句：因为子页面可能有很多行，所以我在data 中创建了很多行，名称、状态相同，但其他值不同

 [name, status, values from first row on subpage]
 [name, status, values from second row on subpage]
 [name, status, values from string row on subpage]

---

from urllib.request import urlopen
from bs4 import BeautifulSoup
import csv

html = urlopen("https://www.accessdata.fda.gov/scripts/drugshortages/default.cfm")
bsObj = BeautifulSoup(html, "lxml")

# list for all rows with all values
data = []

# get table on main page
table = bsObj.find('table', {'id': 'cont'})

# work with every row separatelly
for row in table.find_all("tr")[1:]:  # use `[1:]` to skip header
    # get columns only in this row
    cols = row.find_all('td')

    # get name and url from first column
    link = cols[0].find('a', href=True)
    name = link.text.strip()
    url  = link['href']
    url = "https://www.accessdata.fda.gov/scripts/drugshortages/" + url
    url = url.replace(" ", "%20").replace("®", "%C2%AE")
    print('name:', name)
    print('url:', url)

    # get status from second column
    status = cols[1].text.strip()
    print('status:', status)

    # subpage 
    html = urlopen(url)
    bsObj_href = BeautifulSoup(html, "lxml")
    subtable = bsObj_href.find("table")
    if not subtable:
        data.append([name, status, link, '', '', '', ''])
        print('---')
    else:
        for subrows in subtable.find_all('tr')[1:]:   # use `[1:]` to skip header
            #print(subrows)
            subcols = subrows.find_all('td')
            presentation = subcols[0].text.strip()    
            availability = subcols[1].text.strip()    
            related = subcols[2].text.strip()    
            reason = subcols[3].text.strip()    
            data.append([name, status, link, presentation, availability, related, reason])
            print(presentation, availability, related, reason)
            print('---')

    print('----------')


with open("drug_shortage.csv", 'wt', newline='') as csvfile:
    writer = csv.writer(csvFile)

    # write header - one row - using `writerow` without `s` at the end
    #writer.writerow(['Name', 'Status', 'Link', 'Presentation', 'Availability', 'Related', 'Reason'])

    # write data - many rowr - using `writerows` with `s` at the end
    writer.writerows(data)

# no need to close because it use `with`