如何使用 python 将 Steam 中的游戏评论中的所有 Steam id、评论内容、profile_url 抓取到 excel 文件中？答案

【问题标题】：How to scrape all Steam id, review content, profile_url from reviews of a game in steam into excel file using python?如何使用 python 将 Steam 中的游戏评论中的所有 Steam id、评论内容、profile_url 抓取到 excel 文件中？
【发布时间】：2020-10-08 10:57:17
【问题描述】：

#错误是它只打印前 11 条评论（当使用 n

from msedge.selenium_tools import Edge, EdgeOptions
from selenium.webdriver.common.keys import Keys
import re
from time import sleep
from datetime import datetime
from openpyxl import Workbook

game_id= 1097150
url = 'https://steamcommunity.com/app/1097150/positivereviews/?p=1&browsefilter=trendweek&filterLanguage=english'

options = EdgeOptions()
options.use_chromium = True
driver = Edge(options=options)
driver.get(url)

#页面不断滚动，开始抓取

last_position = driver.execute_script("return window.pageYOffset;")
reviews = []
review_ids = set()

while True:
  cards = driver.find_elements_by_class_name('apphub_Card')
  for card in cards[-20:]:
    profile_url = card.find_element_by_xpath('.//div[@class="apphub_friend_block"]/div/a[2]').get_attribute('href')
    steam_id = profile_url.split('/')[-2]
    date_posted = card.find_element_by_xpath('.//div[@class="apphub_CardTextContent"]/div').text
    review_content = card.find_element_by_xpath('.//div[@class="apphub_CardTextContent"]').text.replace(date_posted,'').strip()  
     
    review = (steam_id, profile_url, review_content)
    reviews.append(review) 
  
  attempt_count = 0
  while attempt_count < 3:
       driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")    
       curr_position = driver.execute_script("return window.pageYOffset;")
       
       if curr_position == last_position:
             attempt_count += 1
             sleep(0.5)
         else:
             break
driver.close()

#保存结果

wb = Workbook()
ws = wb.worksheets[0]
ws.append(['SteamId', 'ProfileURL', 'ReviewText'])
for row in reviews:
    ws.append(row)
    
today = datetime.today().strftime('%Y%m%d')    
wb.save(f'Steam_Reviews_{game_id}_{today}.xlsx')    
wb.close()

【问题讨论】：

为什么不向下滚动直到 len(cards)>=500 然后进行计算。

标签： python-3.x selenium web-scraping beautifulsoup openpyxl

【解决方案1】：

在您的情况下，以下是无限向下滚动或直到 500 个元素的方法。

while True:
  cards = driver.find_elements_by_class_name('apphub_Card')
  if(len(cards)>=500):
      break
  last_position = driver.execute_script("return window.pageYOffset;")
  driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
  time.sleep(1)
  curr_position = driver.execute_script("return window.pageYOffset;")
  if(last_position==curr_position):
      break
    
for card in cards[:500]:
    profile_url = card.find_element_by_xpath('.//div[@class="apphub_friend_block"]/div/a[2]').get_attribute('href')
    steam_id = profile_url.split('/')[-2]
    date_posted = card.find_element_by_xpath('.//div[@class="apphub_CardTextContent"]/div').text
    review_content = card.find_element_by_xpath('.//div[@class="apphub_CardTextContent"]').text.replace(date_posted,'').strip()  
    review = (steam_id, profile_url, review_content)
    reviews.append(review)

【讨论】：