python - 调试泡菜

Question

我试图腌制一个相当复杂的对象层次结构并得到异常：

pickle.PicklingError: Can't pickle <class 'function'>: attribute lookup builtins.function failed

是否有任何合理的方法可以用来测试对象层次结构的可挑选性？我的目标是找到违规功能的位置

Answer 1

为此，我会使用dill，它可以在 python 中序列化几乎任何东西。Dill 还提供了一些很好的工具，可以帮助您了解在代码失败时导致酸洗失败的原因。

>>> import dill
>>> dill.loads(dill.dumps(your_bad_object))
>>> ...
>>> # if you get a pickling error, use dill's tools to figure out a workaround
>>> dill.detect.badobjects(your_bad_object, depth=0)
>>> dill.detect.badobjects(your_bad_object, depth=1)
>>> ...

如果您绝对想要，您可以使用 dill badobjects（或其他检测函数之一）递归地潜入对象的引用链，并弹出不可腌制的对象，而不是像上面那样在每个深度调用它。

此外，objgraph也是对测试套件的一个非常方便的补充。

>>> # visualize the references in your bad objects
>>> objgraph.show_refs(your_bad_object, filename='your_bad_object.png')

Answer 2

我这样做了，它在很多时候都对我有用......一旦我发现一些完全万无一失的东西，我会更新这个

它会进行一堆打印，然后如果有一个要引发异常则引发异常，这样您就可以看到对象层次结构的哪一部分导致了问题。

def test_pickle(xThing,lTested = []):
    import pickle
    if id(xThing) in lTested:
        return lTested
    sType = type(xThing).__name__
    print('Testing {0}...'.format(sType))

    if sType in ['type','int','str']:
        print('...too easy')
        return lTested
    if sType == 'dict':
        print('...testing members')
        for k in xThing:
            lTested = Pickalable.test_pickle(xThing[k],lTested)
        print('...tested members')
        return lTested
    if sType == 'list':
        print('...testing members')
        for x in xThing:
            lTested = Pickalable.test_pickle(x)
        print('...tested members')
        return lTested

    lTested.append(id(xThing))
    oClass = type(xThing)

    for s in dir(xThing):
        if s.startswith('_'):
            print('...skipping *private* thingy')
            continue
        #if it is an attribute: Skip it
        try:
            xClassAttribute = oClass.__getattribute__(oClass,s)
        except AttributeError:
            pass
        else:
            if type(xClassAttribute).__name__ == 'property':
                print('...skipping property')
                continue

        xAttribute = xThing.__getattribute__(s)
        print('Testing {0}.{1} of type {2}'.format(sType,s,type(xAttribute).__name__))
        #if it is a function make sure it is stuck to the class...
        if type(xAttribute).__name__ == 'function':
            raise Exception('ERROR: found a function')
        if type(xAttribute).__name__ == 'method':
            print('...skipping method')
            continue
        if type(xAttribute).__name__ == 'HtmlElement':
            continue
        if type(xAttribute) == dict:
            print('...testing dict values for {0}.{1}'.format(sType,s))
            for k in xAttribute:
                lTested = Pickalable.test_pickle(xAttribute[k])
                continue
            print('...finished testing dict values for {0}.{1}'.format(sType,s))

        try:
            oIter = xAttribute.__iter__()
        except AttributeError:
            pass
        except AssertionError:
            pass #lxml elements do this
        else:
            print('...testing iter values for {0}.{1} of type {2}'.format(sType,s,type(xAttribute).__name__))
            for x in xAttribute:   
                lTested = Pickalable.test_pickle(x,lTested)
            print('...finished testing iter values for {0}.{1}'.format(sType,s))

        try:
            xAttribute.__dict__
        except AttributeError:
            pass
        else:
            #this attribute should be explored seperately...
            lTested = Pickalable.test_pickle(xAttribute,lTested)
            continue
        pickle.dumps(xAttribute)


    print('Testing {0} as complete object'.format(sType))
    pickle.dumps(xThing)
    return lTested   

Answer 3

这是@Sheena 代码的稍微调整的版本，它也适用于 python 2 和一些其他类型：

def test_pickle(xThing, lTested = []):
    import pickle
    if id(xThing) in lTested:
        return lTested
    sType = type(xThing).__name__
    print('Testing {0}...'.format(sType))

    if sType in ['type','int','str', 'bool', 'NoneType', 'unicode']:
        print('...too easy')
        return lTested
    if sType == 'dict':
        print('...testing members')
        for k in xThing:
            lTested = test_pickle(xThing[k],lTested)
        print('...tested members')
        return lTested
    if sType == 'list':
        print('...testing members')
        for x in xThing:
            lTested = test_pickle(x)
        print('...tested members')
        return lTested

    lTested.append(id(xThing))
    oClass = type(xThing)

    for s in dir(xThing):
        if s.startswith('_'):
            print('...skipping *private* thingy')
            continue
        #if it is an attribute: Skip it
        try:
            xClassAttribute = oClass.__getattribute__(oClass,s)
        except (AttributeError, TypeError):
            pass
        else:
            if type(xClassAttribute).__name__ == 'property':
                print('...skipping property')
                continue

        xAttribute = xThing.__getattribute__(s)
        print('Testing {0}.{1} of type {2}'.format(sType,s,type(xAttribute).__name__))
        if type(xAttribute).__name__ == 'function':
            print("...skipping function")
            continue
        if type(xAttribute).__name__ in ['method', 'instancemethod']:
            print('...skipping method')
            continue
        if type(xAttribute).__name__ == 'HtmlElement':
            continue
        if type(xAttribute) == dict:
            print('...testing dict values for {0}.{1}'.format(sType,s))
            for k in xAttribute:
                lTested = test_pickle(xAttribute[k])
                continue
            print('...finished testing dict values for {0}.{1}'.format(sType,s))

        try:
            oIter = xAttribute.__iter__()
        except (AttributeError, TypeError):
            pass
        except AssertionError:
            pass #lxml elements do this
        else:
            print('...testing iter values for {0}.{1} of type {2}'.format(sType,s,type(xAttribute).__name__))
            for x in xAttribute:
                lTested = test_pickle(x,lTested)
            print('...finished testing iter values for {0}.{1}'.format(sType,s))

        try:
            xAttribute.__dict__
        except AttributeError:
            pass
        else:
            #this attribute should be explored seperately...
            lTested = test_pickle(xAttribute,lTested)
            continue
        print(0, xThing, xAttribute)
        pickle.dumps(xAttribute)

    print('Testing {0} as complete object'.format(sType))
    pickle.dumps(xThing)
    return lTested

dill我发现这是最有用的选择（在没有泡菜的地方也更宽容）。你可以简单地运行它

test_pickle(my_complex_object)
print("Done!")