在网站上查找单词并获取其页面链接

Question

我想抓取一些网站，看看那里是否存在“katalog”一词。如果是，我想检索该单词所在的所有选项卡/子页面的链接。有可能吗？

我尝试按照本教程进行操作，但最后得到的 wordlist.csv 是空的，即使网站上确实存在单词 catalog。

https://www.phooky.com/blog/find-specific-words-on-web-pages-with-scrapy/

        wordlist = [
            "katalog",
            "downloads",
            "download"
            ]

def find_all_substrings(string, sub):
    starts = [match.start() for match in re.finditer(re.escape(sub), string)]
    return starts

class WebsiteSpider(CrawlSpider):

    name = "webcrawler"
    allowed_domains = ["www.reichelt.com/"]
    start_urls = ["https://www.reichelt.com/"]
    rules = [Rule(LinkExtractor(), follow=True, callback="check_buzzwords")]

    crawl_count = 0
    words_found = 0                                 

    def check_buzzwords(self, response):

        self.__class__.crawl_count += 1

        crawl_count = self.__class__.crawl_count

        url = response.url
        contenttype = response.headers.get("content-type", "").decode('utf-8').lower()
        data = response.body.decode('utf-8')

        for word in wordlist:
                substrings = find_all_substrings(data, word)
                print("substrings", substrings)
                for pos in substrings:
                        ok = False
                        if not ok:
                                self.__class__.words_found += 1
                                print(word + ";" + url + ";")
        return Item()

    def _requests_to_follow(self, response):
        if getattr(response, "encoding", None) != None:
                return CrawlSpider._requests_to_follow(self, response)
        else:
                return []

如何在网站上找到一个单词的所有实例并获取该单词所在页面的链接？

Answer 1

主要问题是错误allowed_domain - 它必须没有路径/

    allowed_domains = ["www.reichelt.com"]

其他问题可能是本教程已有 3 年历史了（有 Scarpy 文档的链接1.5，但最新版本是2.5.0）。

它还使用了一些无用的代码行。

它得到contenttype，但从不将它用于decoderequest.body。您的网址使用iso8859-1 表示原始语言，utf-8 表示?LANGUAGE=PL - 但您可以简单地使用request.text，它会自动解码。

它也使用ok = False，后来检查它，但它完全没用。

---

最少的工作代码 - 您可以将其复制到单个文件并以 python script.py 运行，而无需创建项目。

import scrapy
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
import re

wordlist = [
    "katalog",
    "catalog",
    "downloads",
    "download",
]

def find_all_substrings(string, sub):
    return [match.start() for match in re.finditer(re.escape(sub), string)]

class WebsiteSpider(CrawlSpider):

    name = "webcrawler"
    
    allowed_domains = ["www.reichelt.com"]
    start_urls = ["https://www.reichelt.com/"]
    #start_urls = ["https://www.reichelt.com/?LANGUAGE=PL"]
    
    rules = [Rule(LinkExtractor(), follow=True, callback="check_buzzwords")]

    #crawl_count = 0
    #words_found = 0                                 

    def check_buzzwords(self, response):
        print('[check_buzzwords] url:', response.url)
        
        #self.crawl_count += 1

        #content_type = response.headers.get("content-type", "").decode('utf-8').lower()
        #print('content_type:', content_type)
        #data = response.body.decode('utf-8')
        
        data = response.text

        for word in wordlist:
            print('[check_buzzwords] check word:', word)
            substrings = find_all_substrings(data, word)
            print('[check_buzzwords] substrings:', substrings)
            
            for pos in substrings:
                #self.words_found += 1
                # only display
                print('[check_buzzwords] word: {} | pos: {} | sub: {} | url: {}'.format(word, pos, data[pos-20:pos+20], response.url))
                # send to file
                yield {'word': word, 'pos': pos, 'sub': data[pos-20:pos+20], 'url': response.url}

# --- run without project and save in `output.csv` ---

from scrapy.crawler import CrawlerProcess

c = CrawlerProcess({
    'USER_AGENT': 'Mozilla/5.0',
    # save in file CSV, JSON or XML
    'FEEDS': {'output.csv': {'format': 'csv'}},  # new in 2.1
})
c.crawl(WebsiteSpider)
c.start() 

---

编辑：

我在生成的数据中添加了data[pos-20:pos+20]，以查看子字符串在哪里，有时它位于.../elements/adw_2018/catalog/... 之类的URL 或<img alt=""catalog"" 之类的其他地方 - 所以使用regex 不一定是个好主意。也许更好的是使用xpath 或css selector 仅在某些地方或链接中搜索文本。

---

编辑：

使用列表中的单词搜索链接的版本。它使用response.xpath 搜索所有 linsk，然后检查href 中是否有单词 - 所以它不需要regex。

问题可能是它将与-downloads-（与s）的链接视为与单词download和downloads的链接，因此需要更复杂的方法来检查（即使用regex）来对待它仅作为与单词downloads的链接

import scrapy
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor

wordlist = [
    "katalog",
    "catalog",
    "downloads",
    "download",
]

class WebsiteSpider(CrawlSpider):

    name = "webcrawler"
    
    allowed_domains = ["www.reichelt.com"]
    start_urls = ["https://www.reichelt.com/"]
    
    rules = [Rule(LinkExtractor(), follow=True, callback="check_buzzwords")]

    def check_buzzwords(self, response):
        print('[check_buzzwords] url:', response.url)
        
        links = response.xpath('//a[@href]')
        
        for word in wordlist:
            
            for link in links:
                url = link.attrib.get('href')
                if word in url:
                    print('[check_buzzwords] word: {} | url: {} | page: {}'.format(word, url, response.url))
                    # send to file
                    yield {'word': word, 'url': url, 'page': response.url}

# --- run without project and save in `output.csv` ---

from scrapy.crawler import CrawlerProcess

c = CrawlerProcess({
    'USER_AGENT': 'Mozilla/5.0',
    # save in file CSV, JSON or XML
    'FEEDS': {'output.csv': {'format': 'csv'}},  # new in 2.1
})
c.crawl(WebsiteSpider)
c.start() 

Answer 2

您可以使用 requests-html 并呈现页面：

from requests_html import HTMLSession

session = HTMLSession()
url = "https://www.reichelt.com/"

r = session.get(url)
r.html.render(sleep=2)

if "your_word" in r.html.text: #or r.html.html if you want it in raw html
    print([link for link in r.html.absolute_links if "your_word" in link])