使用“scrapy”进行 Python 网络抓取：不从 span 中提取文本答案

【问题标题】：Python web scraping with 'scrapy': not extracting text from span使用“scrapy”进行 Python 网络抓取：不从 span 中提取文本
【发布时间】：2019-09-24 17:43:11
【问题描述】：

问题出在第 54 行

port = data.xpath('./td[3]/span[@class="port"]/text()').extract_first()

完整代码

import json
import re
import urllib
from html.parser import HTMLParser
from urllib.parse import urljoin

from scrapy import Field, Item, Selector
from scrapy.http import FormRequest, HtmlResponse, Request
from scrapy.spiders import CrawlSpider

from requests import Session


class ProxyServersPro(Item):
    ip = Field()
    port = Field()
    country = Field()
    speed = Field()
    protocol = Field()
    anon = Field()
    lastcheck = Field()
    speed = Field()


class ProxyServers(CrawlSpider):
    name = "ProxyServersProCrawler"

    allowed_domains = ["proxyservers.pro"]
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        "Accept-Language": "en-US,en;q=0.5",
        "Accept-Encoding": "gzip, deflate",
        "Connection": "keep-alive",
    }
    start_url = [
        "https://es.proxyservers.pro/proxy/list/speed/2/anonymity/elite/order/duration/order_dir/asc/page/1",
        "https://es.proxyservers.pro/proxy/list/speed/2/anonymity/elite/order/duration/order_dir/asc/page/2",
        "https://es.proxyservers.pro/proxy/list/speed/2/anonymity/elite/order/duration/order_dir/asc/page/3",
        "https://es.proxyservers.pro/proxy/list/speed/2/anonymity/elite/order/duration/order_dir/asc/page/4",
        "https://es.proxyservers.pro/proxy/list/speed/2/anonymity/elite/order/duration/order_dir/asc/page/5",
    ]

    def __init__(self):
        super(ProxyServers, self).__init__()

    def start_requests(self):
        for url in self.start_url:
            yield Request(url, callback=self.parse_companies, headers=self.headers)

    def parse_companies(self, response):
        table = response.xpath('//table[@class="table table-hover"]/tbody/tr')
        for data in table:
            ip = data.xpath("./td[2]/a/text()").extract_first()
            country = data.xpath("./td[4]/text()").extract_first()
            protocol = data.xpath("./td[7]/text()").extract_first()
            anon = data.xpath("./td[8]/text()").extract_first()
            port = data.xpath('./td[3]/span[@class="port"]/text()').extract_first()

            item = ProxyServersPro()
            item["ip"] = ip
            item["country"] = country
            item["protocol"] = protocol
            item["anon"] = anon
            item["port"] = port
            yield item

【问题讨论】：

标签： python html scrapy

【解决方案1】：

Scrapy 没有提取值，因为跨度中没有文本。

试试这个：

(Pdb) data.xpath('./td[3]/span/@data-port').extract_first() '050F040E'

【讨论】：

这是在跨度内获取类值，但我需要文本
这里没有 TEXT td[3]/span[@class="port"]/text() 。在从scrapy请求模块给出的响应中。