0

我正在使用 requests 和 BeautifulSoup 来解析 Wikidata 以构建 Person 对象。我能够成功地做到这一点，但是在迭代地做这件事时，我在创建 ~3,000 个 Person 对象后遇到了下面的 MemoryError。

MemoryError

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "TreeBuilder.py", line 11, in <module>
    ancestor = Person(next['id'])
  File "/home/ec2-user/Person.py", line 14, in __init__
    html = soup (data , 'lxml')
  File "/usr/local/lib/python3.7/site-packages/bs4/__init__.py", line 325, in __init__
    self._feed()
  File "/usr/local/lib/python3.7/site-packages/bs4/__init__.py", line 399, in _feed
    self.builder.feed(self.markup)
  File "/usr/local/lib/python3.7/site-packages/bs4/builder/_lxml.py", line 324, in feed
    self.parser.feed(markup)
  File "src/lxml/parser.pxi", line 1242, in lxml.etree._FeedParser.feed
  File "src/lxml/parser.pxi", line 1285, in lxml.etree._FeedParser.feed
  File "src/lxml/parser.pxi", line 855, in lxml.etree._BaseParser._getPushParserContext
  File "src/lxml/parser.pxi", line 871, in lxml.etree._BaseParser._createContext
  File "src/lxml/parser.pxi", line 528, in lxml.etree._ParserContext.__cinit__
SystemError: <class 'lxml.etree._ErrorLog'> returned a result with an error set

我尝试使用以下方法捕获不起作用的异常；

try:
  data = requests.get (url).text
  html = soup(data, 'lxml')
except MemoryError:
  return None

在 Pycharm 中运行程序时，我的本地计算机上不会发生此错误，只是在我的 AWS EC2 服务器上。

更新

请参阅下面的代码。我gc.collect()在每 100 次迭代后添加，这似乎没有帮助。

人物.py

import requests
from bs4 import BeautifulSoup as soup

class Person:

    def __init__(self, id):
        url = 'https://www.wikidata.org/wiki/' + id
        data = requests.get (url).text
        html = soup (data , 'lxml')

        ### id ###
        self.id = id

        ### Name ###
        if html.find ("span" , {"class": "wikibase-title-label"}) != None:
            self.name = html.find ("span" , {"class": "wikibase-title-label"}).string
        else:
            self.name = ""

        ### Born ###
        self.birth = ""

        birth = html.find ("div" , {"id": "P569"})
        if birth != None:
            birth = birth.findAll ("div" , {"class": "wikibase-snakview-variation-valuesnak"})
            if len(birth) > 0:
                self.birth = birth[0].string

        ### Death ###
        self.death = ""

        death = html.find ("div" , {"id": "P570"})
        if death != None:
            death = death.findAll ("div" , {"class": "wikibase-snakview-variation-valuesnak"})
            if len(death) > 0:
                self.death = death[0].string

        #### Sex ####
        sex = html.find ("div" , {"id": "P21"})

        if sex != None:
            for item in sex.strings:
                if item == 'male' or item == 'female':
                    self.sex = item

        ### Mother ###
        self.mother = ""

        mother = html.find ("div" , {"id": "P25"})
        if mother != None:
            mother = mother.findAll ("div" , {"class": "wikibase-snakview-variation-valuesnak"})
            if len(mother) > 0:
                self.mother = {"name": mother[0].string , "id": mother[0].find ('a')['title']}

        ### Father ###
        self.father = ""

        father = html.find ("div" , {"id": "P22"})
        if father != None:
            father = father.findAll ("div" , {"class": "wikibase-snakview-variation-valuesnak"})
            if len(father) > 0:
                self.father = {"name": father[0].string , "id": father[0].find ('a')['title']}

        ### Children ###
        self.children = []
        x = html.find("div" , {"id": "P40"})
        if x != None:
            x = x.findAll("div" , {"class": "wikibase-statementview"})

            for i in x:
                a = i.find ('a')
                if a != None and a['title'][0] == 'Q':
                    self.children.append ({'name': a.string , 'id': a['title']})

    def __str__(self):
        return self.name + "\n\tBirth: " + self.birth + "\n\tDeath: " + self.death + "\n\n\tMother: " + \
               self.mother['name'] + "\n\tFather: " + self.father['name'] + "\n\n\tNumber of Children: " + \
               str(len(self.children))

树生成器.py

from Person import Person
import gc, sys

file = open('ancestors.txt', 'w+')

ancestors = [{'name':'Charlemange', 'id':'Q3044'}]
all = [ancestors[0]['id']]
i = 1

while ancestors != []:
    next = ancestors.pop(0)
    ancestor = Person(next['id'])

    for child in ancestor.children:
        if child['id'] not in all:
            all.append(child['id'])
            ancestors.append(child)

    if ancestor.mother != "" and ancestor.mother['id'] not in all:
        all.append(ancestor.mother['id'])
        ancestors.append(ancestor.mother)

    if ancestor.father != "" and ancestor.father['id'] not in all:
        all.append(ancestor.father['id'])
        ancestors.append(ancestor.father)


    file.write(ancestor.id + "*" + ancestor.name + "*" + "https://www.wikidata.org/wiki/" + ancestor.id + "*" + str(ancestor.birth) + "*" + str(ancestor.death) + "\n")

    if i % 100 == 0:
        print (ancestor.name + " (" + ancestor.id + ")" + " - " + str(len(all)) + " - " + str (len(ancestors)) + " - " + str (sys.getsizeof(all)))
        gc.collect()

    i += 1

file.close()
print("\nDone!")
4

1 回答 1

0

解决方案，无论多么烦人，是将所有对 BS 对象的引用包装在一个str()调用中。虽然看起来你在做的时候只是存储了一个对裸字符串birth[0].string的引用，但事实证明，当你在循环中这样做时，对 BS 对象的引用只是继续在内存中构建。试一试，我刚刚用你的代码试了一下，内存占用很低。

只要确保你抓住了所有的参考资料，例如：

self.children.append({'name': str(a.string), 'id': str(a['title'])})

编辑：请参阅此答案以了解为什么 BS 最终会保留这些参考资料。

于 2020-01-08T23:12:11.957 回答