使用 BeautifulSoup 检索 Google Scholar 结果的问题

Question

我将继续从我的previous question 开始的分析。我在一个由四列组成的数据框中获得了有关特定工作论文出版物的信息：出版年份、出版顺序（每年的出版顺序，在这种情况下完全没用）、标题和作者。因此，我想使用这个数据框来抓取 Google Scholar 并检索有关引用次数的信息。 因为有些论文的标题有点笼统，在某些情况下，谷歌学者的第一个结果实际上并不是我感兴趣的。因此，为了进行更量身定制的研究，在创建链接来执行我的研究包括每篇论文的标题和作者。我在编写代码时遵循了this 线程。

注意：因为执行此抓取需要真实姓名，所以我宁愿不创建示例数据框。我已将 .csv 文件上传到我的 GitHub 上。

import pandas as pd
import requests
from bs4 import BeautifulSoup as bs
from random import randint
from time import sleep

url = 'https://raw.githubusercontent.com/nicolacaravaggio/working_paper_roma3/master/rm3_working_paper_list.csv'
df = pd.read_csv(url, error_bad_lines = False)

papers = [] 

for index, rows in df.iterrows():  
    list_paper = rows.title + ' ' + rows.author

    papers.append(list_paper)

title_list_gs = []
citations_list_gs = []

with requests.Session() as s:

    for paper in papers: 

            sleep(randint(1,3))

            url = 'https://scholar.google.com/scholar?q=' + paper + '&ie=UTF-8&oe=UTF-8&hl=en&btnG=Search'
            r = s.get(url)
            soup = bs(r.content, 'html.parser')

            title_gs = soup.select_one('h3.gs_rt a').text if soup.select_one('h3.gs_rt a') is not None else 'No title'
            title_list_gs.append(title_gs)

            citations_gs = soup.select_one('a:contains("Cited by")').text if soup.select_one('a:contains("Cited by")') is not None else 'No citation count'
            citations_list_gs.append(citations_gs)

            print('Title:', title_gs, '; Citations:', citations_gs)  

但是，我从这个脚本得到的结果只是一个列表：

Title: No title ; Citations: No Citation count

我不确定问题出在我笨拙的脚本（可能）还是因为 Google 阻止我从 Scholar 中抓取太多内容。事实上，即使是我在这个线程中用作起点的script，它也并不总是以预期的结果返回。我希望有人能给我一些建议。提前谢谢你。

Answer 1

听起来您正在触发学者机器人检测。从个人抓取 Google Scholar 的经验来看，45 秒足以避免 CAPTCHA 和 bot 检测。我的刮刀已经运行了 3 天以上而没有被发现。如果您确实被标记，等待大约 2 小时就足以重新开始。 Here is an extract from my code.。

class ScholarScrape():
    def __init__(self):
        self.page = None
        self.last_url = None
        self.last_time = time.time()
        self.min_time_between_scrape = int(ConfigFile.instance().config.get('scholar','bot_avoidance_time'))
        self.header = {'User-Agent':ConfigFile.instance().config.get('scholar','user_agent')}
        self.session = requests.Session()
        pass

    def search(self, query=None, year_lo=None, year_hi=None, title_only=False, publication_string=None, author_string=None, include_citations=True, include_patents=True):
        url = self.get_url(query, year_lo, year_hi, title_only, publication_string, author_string, include_citations, include_patents)
        while True:
            wait_time = self.min_time_between_scrape - (time.time() - self.last_time)
            if wait_time > 0:
                logger.info("Delaying search by {} seconds to avoid bot detection.".format(wait_time))
                time.sleep(wait_time)
            self.last_time = time.time()
            logger.info("SCHOLARSCRAPE: " + url)
            self.page = BeautifulSoup(self.session.get(url, headers=self.header).text, 'html.parser')
            self.last_url = url

            if "Our systems have detected unusual traffic from your computer network" in str(self.page):
                raise BotDetectionException("Google has blocked this computer for a short time because it has detected this scraping script.")

            return

    def get_url(self, query=None, year_lo=None, year_hi=None, title_only=False, publication_string=None, author_string=None, include_citations=True, include_patents=True):
        base_url = "https://scholar.google.com.au/scholar?"
        url = base_url + "as_q=" + urllib.parse.quote(query)

        if year_lo is not None and bool(re.match(r'.*([1-3][0-9]{3})', str(year_lo))):
            url += "&as_ylo=" + str(year_lo)

        if year_hi is not None and bool(re.match(r'.*([1-3][0-9]{3})', str(year_hi))):
            url += "&as_yhi=" + str(year_hi)

        if title_only:
            url += "&as_yhi=title"
        else:
            url += "&as_yhi=any"

        if publication_string is not None:
            url += "&as_publication=" + urllib.parse.quote('"' + str(publication_string) + '"')

        if author_string is not None:
            url += "&as_sauthors=" + urllib.parse.quote('"' + str(author_string) + '"')

        if include_citations:
            url += "&as_vis=0"
        else:
            url += "&as_vis=1"

        if include_patents:
            url += "&as_sdt=0"
        else:
            url += "&as_sdt=1"

        return url

    def get_results_count(self):
        e = self.page.findAll("div", {"class": "gs_ab_mdw"})
        try:
            item = e[1].text.strip()
        except IndexError as ex:
            if "Our systems have detected unusual traffic from your computer network" in str(self.page):
                raise BotDetectionException("Google has blocked this computer for a short time because it has detected this scraping script.")
            else:
                raise ex

        if self.has_numbers(item):
            return self.get_results_count_from_soup_string(item)
        for item in e:
            item = item.text.strip()
            if self.has_numbers(item):
                return self.get_results_count_from_soup_string(item)
        return 0

    @staticmethod
    def get_results_count_from_soup_string(element):
        if "About" in element:
            num = element.split(" ")[1].strip().replace(",","")
        else:
            num = element.split(" ")[0].strip().replace(",","")
        return num

    @staticmethod
    def has_numbers(input_string):
        return any(char.isdigit() for char in input_string)


class BotDetectionException(Exception):
    pass

if __name__ == "__main__":
    s = ScholarScrape()
    s.search(**{
        "query":"\"policy shaping\"",
        # "publication_string":"JMLR",
        "author_string": "gilboa",
        "year_lo": "1995",
        "year_hi": "2005",

    })
    x = s.get_results_count()
    print(x)

This question might have more information to help you

Answer 2

您也可以使用第三方解决方案，例如 SerpApi。这是一个免费试用的付费 API。我们为您处理代理、解决验证码并解析所有丰富的结构化数据。

示例 python 代码（其他库中也有）：

from serpapi import GoogleSearch

params = {
  "api_key": "secret_api_key",
  "engine": "google_scholar",
  "q": "Francesco Crespi",
  "hl": "en"
}

search = GoogleSearch(params)
results = search.get_dict()

示例 JSON 输出：

{
  "position": 1,
  "title": "Demand and innovation in productivity growth",
  "result_id": "6vfKIRtQWRQJ",
  "link": "https://www.tandfonline.com/doi/abs/10.1080/02692170802407429",
  "snippet": "The labour productivity impact of demand and innovation is investigated in this paper combining insights from the Kaldorian and Schumpeterian traditions. After a review of studies in such traditions, a general model is proposed for explaining productivity growth in …",
  "publication_info": {
    "summary": "F Crespi, M Pianta - International Review of Applied Economics, 2008 - Taylor & Francis",
    "authors": [
      {
        "name": "F Crespi",
        "link": "https://scholar.google.com/citations?user=gKdC5-0AAAAJ&hl=en&oi=sra",
        "serpapi_scholar_link": "https://serpapi.com/search.json?author_id=gKdC5-0AAAAJ&engine=google_scholar_author&hl=en",
        "author_id": "gKdC5-0AAAAJ"
      },
      {
        "name": "M Pianta",
        "link": "https://scholar.google.com/citations?user=knrxY9EAAAAJ&hl=en&oi=sra",
        "serpapi_scholar_link": "https://serpapi.com/search.json?author_id=knrxY9EAAAAJ&engine=google_scholar_author&hl=en",
        "author_id": "knrxY9EAAAAJ"
      }
    ]
  },
  "inline_links": {
    "serpapi_cite_link": "https://serpapi.com/search.json?engine=google_scholar_cite&q=6vfKIRtQWRQJ",
    "cited_by": {
      "total": 88,
      "link": "https://scholar.google.com/scholar?cites=1466291231147096042&as_sdt=2005&sciodt=0,5&hl=en",
      "cites_id": "1466291231147096042",
      "serpapi_scholar_link": "https://serpapi.com/search.json?as_sdt=2005&cites=1466291231147096042&engine=google_scholar&hl=en"
    },
    "related_pages_link": "https://scholar.google.com/scholar?q=related:6vfKIRtQWRQJ:scholar.google.com/&scioq=Francesco+Crespi&hl=en&as_sdt=0,5",
    "versions": {
      "total": 7,
      "link": "https://scholar.google.com/scholar?cluster=1466291231147096042&hl=en&as_sdt=0,5",
      "cluster_id": "1466291231147096042",
      "serpapi_scholar_link": "https://serpapi.com/search.json?as_sdt=0%2C5&cluster=1466291231147096042&engine=google_scholar&hl=en"
    }
  }
},
...

查看documentation了解更多详情。

免责声明：我在 SerpApi 工作。