【发布时间】：2021-07-15 02:45:03
【问题描述】：
我正在尝试进行网络抓取。到目前为止，我有代码可以从一页中提取值并更改为下一页。但是，当我循环该过程以对所有其他页面执行相同操作时，它会返回错误。到目前为止，我有这个代码：
import time
import requests
import pandas
import pandas as pd
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
import json
driver = webdriver.Chrome('C:\DRIVERS\chromedriver.exe')
driver.get('https://www.remax.pt/comprar?searchQueryState={%22regionName%22:%22%22,%22businessType%22:1,%22listingClass%22:1,%22page%22:1,%22sort%22:{%22fieldToSort%22:%22ContractDate%22,%22order%22:1},%22mapIsOpen%22:false}')
driver.maximize_window()
driver.implicitly_wait(5)
wait = WebDriverWait(driver, 10)
cookies = driver.find_element_by_id('rcc-decline-button')
cookies.click()
element_list = []
for j in range (1,2569):
try:
for i in range(1,40,2):
link = driver.find_element_by_xpath("(//div[@class='listing-search-searchdetails-component'])[{0}]".format(i))
link.click()
try:
detalhes = driver.find_element_by_id('details')
preço = driver.find_element_by_id('listing-price')
tipo = driver.find_element_by_id('listing-title')
freguesia = driver.find_element_by_xpath('//h5[@class="listing-address"]')
imoveis = [detalhes.text, preço.text, tipo.text, freguesia.text]
element_list.append(imoveis)
finally:
driver.back()
finally:
wait.until(EC.element_to_be_clickable((By.XPATH,"//a[@class='page-link'][.//span[.='Next']]"))).click()
所有值都在第一个页面中被抓取，但是当它改变页面时出现此错误：
错误：
---------------------------------------------------------------------------
StaleElementReferenceException Traceback (most recent call last)
<ipython-input-7-052f5032275d> in <module>
12 for i in range(1,40,2):
13 link = driver.find_element_by_xpath("(//div[@class='listing-search-searchdetails-component'])[{0}]".format(i))
---> 14 link.click()
15 try:
16 detalhes = driver.find_element_by_id('details')
~\anaconda3\lib\site-packages\selenium\webdriver\remote\webelement.py in click(self)
78 def click(self):
79 """Clicks the element."""
---> 80 self._execute(Command.CLICK_ELEMENT)
81
82 def submit(self):
~\anaconda3\lib\site-packages\selenium\webdriver\remote\webelement.py in _execute(self, command, params)
631 params = {}
632 params['id'] = self._id
--> 633 return self._parent.execute(command, params)
634
635 def find_element(self, by=By.ID, value=None):
~\anaconda3\lib\site-packages\selenium\webdriver\remote\webdriver.py in execute(self, driver_command, params)
319 response = self.command_executor.execute(driver_command, params)
320 if response:
--> 321 self.error_handler.check_response(response)
322 response['value'] = self._unwrap_value(
323 response.get('value', None))
~\anaconda3\lib\site-packages\selenium\webdriver\remote\errorhandler.py in check_response(self, response)
240 alert_text = value['alert'].get('text')
241 raise exception_class(message, screen, stacktrace, alert_text)
--> 242 raise exception_class(message, screen, stacktrace)
243
244 def _value_or_default(self, obj, key, default):
StaleElementReferenceException: Message: stale element reference: element is not attached to the page document
(Session info: chrome=90.0.4430.72)
这是什么元素？
【问题讨论】：
标签： python pandas selenium selenium-chromedriver