2 
#!/usr/bin/env python2.7 

import vobject

abinfile='/foo/bar/dir/infile.vcf' #ab stands for address book  

aboutfile='/foo/bar/dir/outfile.vcf'  

def eliminate_vcard_duplicates (abinfile, aboutfile):

    #we first convert the Adrees Book IN FILE into a list

    with open(abinfile) as source_file:
        ablist = list(vobject.readComponents(source_file))

    #then add each vcard from that list in a new list unless it's already there

    ablist_norepeats=[]
    ablist_norepeats.append(ablist[0])

    for i in range(1, len(ablist)):
        jay=len(ablist_norepeats)
        for j in reversed(range(0, jay)): #we do reversed because usually cards have duplicates nearby
            if ablist_norepeats[j].serialize() == ablist[i].serialize():
                break
            else:
                jay += -1
        if jay == 0:
            ablist_norepeats.append(ablist[i])

    #and finally write the singularized list to the Adrees Book OUT FILE

    with open(aboutfile, 'w') as destination_file:
        for j in range(0, len(ablist_norepeats)):
            destination_file.write(ablist_norepeats[j].serialize)

eliminate_vcard_duplicates(abinfile, aboutfile)

上面的代码有效并创建了一个新文件，其中没有完全相同的重复项（具有相同单数化的重复项）。我知道代码有一些效率问题：它是 n 平方，当它可能是 n*log n 时；我们只能将每个 vacard 序列化一次；for 等的使用效率低下。在这里，我想提供一个简短的代码来说明我不知道如何解决的一个问题。

我不知道如何优雅地解决这个问题：如果卡片中的某些字段被打乱，它不会检测到它们是否相等。有没有办法使用 vobject、re 或其他方法检测此类重复项？

测试中使用的文件内容，有四个相同的电子名片（电话乱码乱码 - 不是电子邮件乱码的想法），是这个： 

BEGIN:VCARD
VERSION:3.0
FN:Foo_bar1
N:;Foo_bar1;;;
EMAIL;TYPE=INTERNET:foobar1@foo.bar.com
TEL;TYPE=CELL:123456789
TEL;TYPE=CELL:987654321
END:VCARD
BEGIN:VCARD
VERSION:3.0
FN:Foo_bar1
N:;Foo_bar1;;;
EMAIL;TYPE=INTERNET:foobar1@foo.bar.com
TEL;TYPE=CELL:123456789
TEL;TYPE=CELL:987654321
END:VCARD
BEGIN:VCARD
VERSION:3.0
FN:Foo_bar1
N:;Foo_bar1;;;
TEL;TYPE=CELL:123456789
TEL;TYPE=CELL:987654321
EMAIL;TYPE=INTERNET:foobar1@foo.bar.com
END:VCARD
BEGIN:VCARD
VERSION:3.0
FN:Foo_bar1
N:;Foo_bar1;;;
TEL;TYPE=CELL:987654321
TEL;TYPE=CELL:123456789
EMAIL;TYPE=INTERNET:foobar1@foo.bar.com
END:VCARD

上面的代码不会检测到这四个都是一样的，因为最后一个的电话号码被打乱了。

作为奖励积分，如果有人有更快的算法，如果可以共享它会很棒。上述一个需要几天时间才能处理一个 30.000 Vcard 文件......

4

3 回答 3

2

您可能已经注意到的一件事是，如果您调用该 .serialize()方法，则EMAIL排序在FN. 但不幸的是，电话号码没有排序。如果是这样，您可以将序列化的单个组件添加到一个集合中，并让唯一的哈希对多次出现的情况进行排序。

如果你调查你从生成器中得到的东西 vobject.readComponents()（例如 using type()），你会看到那是Component来自 module vobject.base，并且dir()在实例上使用你会看到 method getSortedChildren()。如果你在源代码中查找它，你会发现：

def getSortedChildren(self):
    return [obj for k in self.sortChildKeys() for obj in self.contents[k]]

和sortChildKeys()直接高于：

def sortChildKeys(self):
    try:
        first = [s for s in self.behavior.sortFirst if s in self.contents]
    except Exception:
        first = []
    return first + sorted(k for k in self.contents.keys() if k not in first)

调用sortChildKeys()您的示例实例给出['version', 'email', 'fn', 'n', 'tel']了两个结论：

  • sortFirst导致version在前面
  • for obj in self.contents[k]未排序，因此您的电话条目未排序。

解决方案似乎是您重新定义getSortedChildren()为：

    return [obj for k in self.sortChildKeys() for obj in sorted(self.contents[k])]

但这会导致：

TypeError：“ContentLine”和“ContentLine”实例之间不支持“<”

所以你需要提供一些基本的比较操作， ContentLine这些操作也定义在vobject.base：

import vobject

from vobject.base import Component, ContentLine

def gsc(self):
    return [obj for k in self.sortChildKeys() for obj in sorted(self.contents[k])]

Component.getSortedChildren = gsc

def ltContentLine(self, other):
    return str(self) < str(other)

def eqContentLine(self, other):
    return str(self) == str(other)

ContentLine.__lt__ = ltContentLine
ContentLine.__eq__ = eqContentLine


addresses = set()
with open('infile.vcf') as fp:
  for vcard in vobject.readComponents(fp):
     # print(type(vcard))
     # print(dir(vcard))
     # print(vcard.sortChildKeys())
     # print(vcard.contents.keys())
     addresses.add(vcard.serialize())

with open('outfile.vcf', 'w') as fp:
    for a in addresses:
        fp.write(a)

# and check
with open('outfile.vcf') as fp:
    print(fp.read(), end="")

这使：

BEGIN:VCARD
VERSION:3.0
EMAIL;TYPE=INTERNET:foobar1@foo.bar.com
FN:Foo_bar1
N:;Foo_bar1;;;
TEL;TYPE=CELL:123456789
TEL;TYPE=CELL:987654321
END:VCARD
于 2019-03-28T17:27:25.563 回答
1

以下是一个更快的代码（大约三个数量级），但仍然只删除完全重复的...

    #!/usr/bin/env python2.7 

    import vobject
    import datetime

    abinfile='/foo/bar/dir/infile.vcf' #ab stands for address book  

    aboutfile='/foo/bar/dir/outfile.vcf' 

    def eliminate_vcard_duplicatesv2(abinfile, aboutfile):

        #we first convert the Adrees Book IN FILE into a list
        ablist=[]
        with open(abinfile) as source_file:
            ablist = list(vobject.readComponents(source_file))

        #we then serialize the list to expedite comparison process
        ablist_serial=[]
        for i in range(0, len(ablist)):
            ablist_serial.append(ablist[i].serialize())

        #then add each unique vcard's position from that list in a new list unless it's already there
        ablist_singletons=[]
        duplicates=0
        for i in range(1, len(ablist_serial)):
            if i % 1000 == 0:
                print "COMPUTED CARD:", i, "Number of duplicates: ", duplicates, "Current time:", datetime.datetime.now().time()
            jay=len(ablist_singletons)
            for j in reversed(range(0, jay)): #we do reversed because usually cards have duplicates nearby
                if ablist_serial[ablist_singletons[j]] == ablist_serial[i]:
                    duplicates += 1
                    break
                else:
                    jay += -1
            if jay == 0:
                ablist_singletons.append(i)

        print "Length of Original Vcard File: ", len(ablist)
        print "Length of Singleton Vcard File: ", len(ablist_singletons)
        print "Generating Singleton Vcard file and storing it in: ", aboutfile

        #and finally write the singularized list to the Adrees Book OUT FILE
        with open(aboutfile, 'w') as destination_file:
            for k in range(0, len(ablist_singletons)):
                destination_file.write(ablist_serial[ablist_singletons[k]])

    eliminate_vcard_duplicatesv2(abinfile, aboutfile)
于 2017-01-07T01:27:44.457 回答
1

Anthon 的答案的一个变体，使用类装饰器。

import vobject
from vobject.base import Component, ContentLine


def sortedContents(cls):
    def getSortedChildren(self):
        return [obj for k in self.sortChildKeys() for obj in sorted(self.contents[k])]

    cls.getSortedChildren = getSortedChildren
    return cls


def sortableContent(cls):
    def __lt__(self, other):
        return str(self) < str(other)

    def __eq__(self, other):
        return str(self) == str(other)

    cls.__lt__ = __lt__
    cls.__eq__ = __eq__
    return cls


Component = sortedContents(Component)
ContentLine = sortableContent(ContentLine)


addresses = set()

with open('infile.vcf') as infile:
    for vcard in vobject.readComponents(infile):
        addresses.add(vcard.serialize())

with open('outfile.vcf', 'wb') as outfile:
    for address in addresses:
        outfile.write(bytes(address, 'UTF-8'))
于 2021-02-10T02:35:55.287 回答