新闻网站抓取不起作用？

Question

您好，我是python新手，很抱歉在我不知道出了什么问题的时候问了这么一个具体的问题..

我正在尝试从一个韩国新网站抓取新闻文章。 当我运行这段代码时

    import sys
    from bs4 import BeautifulSoup
    import urllib.request
    from urllib.parse import quote

    target_url_b4_pn="http://news.donga.com/search?p="
    target_url_b4_keyword='&query='

target_url_rest="&check_news1&more=1&sorting1&search_date1&v1=&v2=&range=1"



    def get_text(URL, output_file):
        source_code_from_URL=urllib.request.urlopen(URL)
        soup=BeautifulSoup(source_code_from_URL, 'lxml', from_encoding='UTF-8')
        content_of_article=soup.select('div.article')
        for item in content_of_article:
            string_item=str(item.find_all(text=True))
            output_file.write(string_item)

    def get_link_from_news_title(page_num, URL, output_file):
        for i in range(page_num):
            current_page_num=1+i*15
            position=URL.index('=')
                    URL_with_page_num=URL[:position+1]+str(current_page_num)+URL[position+1:]
            source_code_from_URL=urllib.request.urlopen(URL_with_page_num)
            soup=BeautifulSoup(source_code_from_URL, 'lxml',from_encoding='UTF-8')

            for title in soup.find_all('p','tit'):
                title_link=title.select('a')
                article_URL=title_link[0]['href']
                get_text(article_URL, output_file)

    def main():
        keyword="노무현"
        page_num=1
        output_file_name="output.txt"
        target_url=target_url_b4_pn+target_url_b4_keyword+quote(keyword)+target_url_rest
        output_file=open(output_file_name, "w", -1, "utf-8")
        get_link_from_news_title(page_num, target_url, output_file)
        output_file.close()


    if __name__=='__main__':
        main()
    print(target_url)
    print(11111)

jupyter notebook 不响应输入，甚至在底部不吐出任何简单的命令（不打印任何东西）

认为代码以某种方式冻结了它，请告诉我哪里可能出错了？

the picture where it's not responding

Answer 1

1. get_text 函数的第一行，urllib.request.urlopen(URL) 表示您打开 url，但就像打开文件一样，您必须 read 它。
   所以在它后面加上一个read()。
   urllib.request.urlopen(URL).read() 否则beautifulsoup 将无法识别它。

2. 在你的css选择器soup.select('div.article')中，页面中没有这个元素，我猜你想要的是soup.select('div.article_txt')，它与文章的段落相匹配。

3. 你的 print(target_url) 应该移动到你的 main 函数中，因为 target_url 只在 main 中定义。

代码

import sys
from bs4 import BeautifulSoup
import urllib.request
from urllib.parse import quote

target_url_b4_pn="http://news.donga.com/search?p="
target_url_b4_keyword='&query='

target_url_rest="&check_news1&more=1&sorting1&search_date1&v1=&v2=&range=1"



def get_text(URL, output_file):
    source_code_from_URL=urllib.request.urlopen(URL)
    soup=BeautifulSoup(source_code_from_URL, 'lxml', from_encoding='UTF-8')
    # change your css selector so it match some element
    content_of_article=soup.select('div.article_txt')
    for item in content_of_article:
        string_item=item.find_all(text=True)
        #write string to file
        output_file.write(" ".join(string_item))

def get_link_from_news_title(page_num, URL, output_file):
    for i in range(page_num):
        current_page_num=1+i*15
        position=URL.index('=')
        URL_with_page_num=URL[:position+1]+str(current_page_num)+URL[position+1:]
        source_code_from_URL=urllib.request.urlopen(URL_with_page_num)
        soup=BeautifulSoup(source_code_from_URL, 'lxml',from_encoding='UTF-8')

        for title in soup.find_all('p','tit'):
            title_link=title.select('a')
            article_URL=title_link[0]['href']
            get_text(article_URL, output_file)

def main():
    keyword="노무현"
    page_num=1
    output_file_name="output.txt"
    target_url=target_url_b4_pn+target_url_b4_keyword+quote(keyword)+target_url_rest
    # move `target_url` here
    print(target_url)

    output_file=open(output_file_name, "w", -1, "utf-8")
    get_link_from_news_title(page_num, target_url, output_file)
    output_file.close()


if __name__=='__main__':
    main()
    print(11111)