vcf 文件中用于 vcard 重复删除的 python 代码适用于 vobject，但仅适用于“精确重复”

Question

#!/usr/bin/env python2.7 

import vobject

abinfile='/foo/bar/dir/infile.vcf' #ab stands for address book  

aboutfile='/foo/bar/dir/outfile.vcf'  

def eliminate_vcard_duplicates (abinfile, aboutfile):

    #we first convert the Adrees Book IN FILE into a list

    with open(abinfile) as source_file:
        ablist = list(vobject.readComponents(source_file))

    #then add each vcard from that list in a new list unless it's already there

    ablist_norepeats=[]
    ablist_norepeats.append(ablist[0])

    for i in range(1, len(ablist)):
        jay=len(ablist_norepeats)
        for j in reversed(range(0, jay)): #we do reversed because usually cards have duplicates nearby
            if ablist_norepeats[j].serialize() == ablist[i].serialize():
                break
            else:
                jay += -1
        if jay == 0:
            ablist_norepeats.append(ablist[i])

    #and finally write the singularized list to the Adrees Book OUT FILE

    with open(aboutfile, 'w') as destination_file:
        for j in range(0, len(ablist_norepeats)):
            destination_file.write(ablist_norepeats[j].serialize)

eliminate_vcard_duplicates(abinfile, aboutfile)

上面的代码有效并创建了一个新文件，其中没有完全相同的重复项（具有相同单数化的重复项）。我知道代码有一些效率问题：它是 n 平方，当它可能是 n*log n 时；我们只能将每个 vacard 序列化一次； for 等的使用效率低下。在这里，我想提供一个简短的代码来说明我不知道如何解决的一个问题。

我不知道如何优雅地解决这个问题：如果一些卡片中的字段被打乱，它不会检测到它们是否相等。有没有办法使用 vobject、re 或其他方法检测此类重复项？

测试中使用的文件内容，有四个相等的电子名片（电话打乱了代码 - 不是电子邮件打乱的想法），是这个：

BEGIN:VCARD
VERSION:3.0
FN:Foo_bar1
N:;Foo_bar1;;;
EMAIL;TYPE=INTERNET:foobar1@foo.bar.com
TEL;TYPE=CELL:123456789
TEL;TYPE=CELL:987654321
END:VCARD
BEGIN:VCARD
VERSION:3.0
FN:Foo_bar1
N:;Foo_bar1;;;
EMAIL;TYPE=INTERNET:foobar1@foo.bar.com
TEL;TYPE=CELL:123456789
TEL;TYPE=CELL:987654321
END:VCARD
BEGIN:VCARD
VERSION:3.0
FN:Foo_bar1
N:;Foo_bar1;;;
TEL;TYPE=CELL:123456789
TEL;TYPE=CELL:987654321
EMAIL;TYPE=INTERNET:foobar1@foo.bar.com
END:VCARD
BEGIN:VCARD
VERSION:3.0
FN:Foo_bar1
N:;Foo_bar1;;;
TEL;TYPE=CELL:987654321
TEL;TYPE=CELL:123456789
EMAIL;TYPE=INTERNET:foobar1@foo.bar.com
END:VCARD

上面的代码不会检测到四个都是一样的，因为最后一个的电话号码被打乱了。

作为奖励积分，如果有人有更快的算法，如果可以共享它会很棒。上面的一个 30.000 Vcard 文件需要几天时间......

Answer 1

您可能已经注意到的一件事是，如果您调用 .serialize() 方法然后 EMAIL 在 FN 之前排序。但 不幸的是，电话号码没有排序。如果他们是，你 可以将序列化的单个组件添加到一个集合中，并让 唯一的散列会整理出多次出现的情况。

如果你调查你从生成器中得到了什么 vobject.readComponents()（例如使用type()），你会看到 是来自模块vobject.base 的Component，并使用dir() on 你看到一个方法getSortedChildren()的实例。如果你这样看 在源代码中，您会发现：

def getSortedChildren(self):
    return [obj for k in self.sortChildKeys() for obj in self.contents[k]]

和sortChildKeys() 直接在其上方：

def sortChildKeys(self):
    try:
        first = [s for s in self.behavior.sortFirst if s in self.contents]
    except Exception:
        first = []
    return first + sorted(k for k in self.contents.keys() if k not in first)

在您的示例实例上调用 sortChildKeys() 会得到 ['version', 'email', 'fn', 'n', 'tel']，从而得出两个结论：

• sortFirst 导致 version 位于最前面
• for obj in self.contents[k] 未排序，因此您的 TEL 条目未排序。

解决方案似乎是你将getSortedChildren()重新定义为：

    return [obj for k in self.sortChildKeys() for obj in sorted(self.contents[k])]

但这会导致：

TypeError：“ContentLine”和“ContentLine”实例之间不支持“

所以你需要提供一些基本的比较操作 ContentLine 也在 vobject.base 中定义：

import vobject

from vobject.base import Component, ContentLine

def gsc(self):
    return [obj for k in self.sortChildKeys() for obj in sorted(self.contents[k])]

Component.getSortedChildren = gsc

def ltContentLine(self, other):
    return str(self) < str(other)

def eqContentLine(self, other):
    return str(self) == str(other)

ContentLine.__lt__ = ltContentLine
ContentLine.__eq__ = eqContentLine


addresses = set()
with open('infile.vcf') as fp:
  for vcard in vobject.readComponents(fp):
     # print(type(vcard))
     # print(dir(vcard))
     # print(vcard.sortChildKeys())
     # print(vcard.contents.keys())
     addresses.add(vcard.serialize())

with open('outfile.vcf', 'w') as fp:
    for a in addresses:
        fp.write(a)

# and check
with open('outfile.vcf') as fp:
    print(fp.read(), end="")

给出：

BEGIN:VCARD
VERSION:3.0
EMAIL;TYPE=INTERNET:foobar1@foo.bar.com
FN:Foo_bar1
N:;Foo_bar1;;;
TEL;TYPE=CELL:123456789
TEL;TYPE=CELL:987654321
END:VCARD

Answer 2

以下是更快的代码（大约三个数量级），但仍然只删除完全重复的...

    #!/usr/bin/env python2.7 

    import vobject
    import datetime

    abinfile='/foo/bar/dir/infile.vcf' #ab stands for address book  

    aboutfile='/foo/bar/dir/outfile.vcf' 

    def eliminate_vcard_duplicatesv2(abinfile, aboutfile):

        #we first convert the Adrees Book IN FILE into a list
        ablist=[]
        with open(abinfile) as source_file:
            ablist = list(vobject.readComponents(source_file))

        #we then serialize the list to expedite comparison process
        ablist_serial=[]
        for i in range(0, len(ablist)):
            ablist_serial.append(ablist[i].serialize())

        #then add each unique vcard's position from that list in a new list unless it's already there
        ablist_singletons=[]
        duplicates=0
        for i in range(1, len(ablist_serial)):
            if i % 1000 == 0:
                print "COMPUTED CARD:", i, "Number of duplicates: ", duplicates, "Current time:", datetime.datetime.now().time()
            jay=len(ablist_singletons)
            for j in reversed(range(0, jay)): #we do reversed because usually cards have duplicates nearby
                if ablist_serial[ablist_singletons[j]] == ablist_serial[i]:
                    duplicates += 1
                    break
                else:
                    jay += -1
            if jay == 0:
                ablist_singletons.append(i)

        print "Length of Original Vcard File: ", len(ablist)
        print "Length of Singleton Vcard File: ", len(ablist_singletons)
        print "Generating Singleton Vcard file and storing it in: ", aboutfile

        #and finally write the singularized list to the Adrees Book OUT FILE
        with open(aboutfile, 'w') as destination_file:
            for k in range(0, len(ablist_singletons)):
                destination_file.write(ablist_serial[ablist_singletons[k]])

    eliminate_vcard_duplicatesv2(abinfile, aboutfile)

Answer 3

Anthon's answer 的变体，使用类装饰器。

import vobject
from vobject.base import Component, ContentLine


def sortedContents(cls):
    def getSortedChildren(self):
        return [obj for k in self.sortChildKeys() for obj in sorted(self.contents[k])]

    cls.getSortedChildren = getSortedChildren
    return cls


def sortableContent(cls):
    def __lt__(self, other):
        return str(self) < str(other)

    def __eq__(self, other):
        return str(self) == str(other)

    cls.__lt__ = __lt__
    cls.__eq__ = __eq__
    return cls


Component = sortedContents(Component)
ContentLine = sortableContent(ContentLine)


addresses = set()

with open('infile.vcf') as infile:
    for vcard in vobject.readComponents(infile):
        addresses.add(vcard.serialize())

with open('outfile.vcf', 'wb') as outfile:
    for address in addresses:
        outfile.write(bytes(address, 'UTF-8'))