从用python编写的蜘蛛获取csv中的重复项[关闭]

Question

我创建了一个蜘蛛，它按照我的预期收集数据。我现在面临的唯一问题是结果有很多重复。但是，我想在 csv 中写入结果时去掉重复项：

代码如下：

import csv
import requests
from lxml import html

def Startpoint():
    global writer
    outfile=open('Data.csv','w',newline='')
    writer=csv.writer(outfile)
    writer.writerow(["Name","Price"])
    address = "https://www.sephora.ae/en/stores/"
    page = requests.get(address)
    tree = html.fromstring(page.text)
    titles=tree.xpath('//li[contains(@class,"level0")]')
    for title in titles:
        href = title.xpath('.//a[contains(@class,"level0")]/@href')[0]
        Layer2(href)

def Layer2(address):
    global writer
    page = requests.get(address)
    tree = html.fromstring(page.text)
    titles=tree.xpath('//li[contains(@class,"amshopby-cat")]')
    for title in titles:
        href = title.xpath('.//a/@href')[0]
        Endpoint(href)

def Endpoint(address):
    global writer
    page = requests.get(address)
    tree = html.fromstring(page.text)
    titles=tree.xpath('//div[@class="product-info"]')
    for title in titles:
        Name = title.xpath('.//div[contains(@class,"h3")]/a[@title]/text()')[0]
        Price = title.xpath('.//span[@class="price"]/text()')[0]
        metco=(Name,Price)
        print(metco)
        writer.writerow(metco)

Startpoint()

Answer 1

您不需要 csv 模块来写入 csv 文件。指定扩展名就足够了。因此，把你的代码变成

import requests
from lxml import html

delimiter = ";"
file_name = 'data.csv'

def Startpoint():
    address = "https://www.sephora.ae/en/stores/"
    page = requests.get(address)
    tree = html.fromstring(page.text)
    titles=tree.xpath('//li[contains(@class,"level0")]')
    for title in titles:
        href = title.xpath('.//a[contains(@class,"level0")]/@href')[0]
        Layer2(href)

def Layer2(address):
    page = requests.get(address)
    tree = html.fromstring(page.text)
    titles=tree.xpath('//li[contains(@class,"amshopby-cat")]')
    for title in titles:
        href = title.xpath('.//a/@href')[0]
        Endpoint(href)

def Endpoint(address):
    page = requests.get(address)
    tree = html.fromstring(page.text)
    titles=tree.xpath('//div[@class="product-info"]')
    for title in titles:
        Name = title.xpath('.//div[contains(@class,"h3")]/a[@title]/text()')[0]
        Price = title.xpath('.//span[@class="price"]/text()')[0]
        metco=(Name,Price)
        print(metco)
        with open(file_name,'a') as outfile:
            outfile.write(delimiter.join(metco).encode('utf8') + '\n')

with open(file_name,'w') as outfile:
    outfile.write(delimiter.join(["Product Name", "Price"])+'\n')
Startpoint()

应该可以解决问题。请注意.encode('utf8') 部分，它会阻止您的写作过程来自UnicodeEncodeError。此外，请注意open 函数中使用的参数'w' 和'a'。第一个意思是“写”，第二个意思是“追加”。然而，即使从启发式的角度来看，这段代码是有效的，但它远不是“很好”的想法。