2

我有这个代码：

from twisted.internet import reactor
from twisted.internet import defer, task
from pysnmp.entity import engine, config
from pysnmp.carrier.twisted import dispatch
from pysnmp.carrier.twisted.dgram import udp
from pysnmp.entity.rfc3413.twisted import cmdgen


import __webimport__
import tools.config
from tools.database import makedsn
import psycopg2


def cmp_varBinds(varBind, varName):
    if varName[0] in str(varBind[0]):
        return True


def cbFun(cbCtx, ip, varNames):
    (errorIndication, errorStatus, errorIndex, varBinds) = cbCtx
    if varBinds and any(map(cmp_varBinds, varBinds[0], varNames)):
        print ip, [str(x[1]) for x in varBinds[0]]
        df = defer.Deferred()
        df.addCallback(cbFun, ip=ip, varNames=varNames)
        return df  # This also indicates that we wish to continue walkin


def parallel(iterable, count, callable, *args, **named):
    coop = task.Cooperator()
    work = (callable(elem, *args, **named) for elem in iterable)
    return defer.DeferredList([coop.coiterate(work) for i in xrange(count)])





def fetch(host):
    id, ip, community, hc = host
    snmpEngine = engine.SnmpEngine()
    snmpEngine.registerTransportDispatcher(dispatch.TwistedDispatcher())
    config.addV1System(snmpEngine, 'test-agent', community)
    config.addTargetParams(snmpEngine, 'myParams', 'test-agent', 'noAuthNoPriv', 1)

    config.addTargetAddr(
                     snmpEngine, 'myRouter', config.snmpUDPDomain,
                     (ip, 161), 'myParams', timeout=1
    )

    # Transport
    config.addSocketTransport(
                          snmpEngine,
                          udp.domainName,
                          udp.UdpTwistedTransport().openClientMode()
    )

    getCmdGen = cmdgen.NextCommandGenerator()
    varNames = [('1.3.6.1.2.1.2.2.1.11', None),
            ('1.3.6.1.2.1.2.2.1.12', None),
            ('1.3.6.1.2.1.2.2.1.13', None),
            ('1.3.6.1.2.1.2.2.1.14', None)]
    df = getCmdGen.sendReq(snmpEngine, 'myRouter', varNames)
    df.addCallback(cbFun, ip=ip, varNames=varNames)
    return df


dsn = makedsn(tools.config.main_db)
connection = psycopg2.connect(dsn)
cursor = connection.cursor()
cursor.execute("""SELECT e.id, e.ip, e.snmpcomm, e.hccnt
           FROM snmp_ports sp, equipment e
           WHERE e.snmp = 'Y' and sp.equipment = e.id
           GROUP BY e.id,e.ip,e.snmpcomm,e.hccnt
           ORDER BY e.id""")
hosts = cursor.fetchall()


finished = parallel(hosts, len(hosts), fetch)
finished.addErrback(log.err)
finished.addCallback(lambda ign: reactor.stop())
reactor.run()

我从数据库中获取 4000 台主机并询问每台主机。如果我在 sql 查询中设置限制 1000 它工作正常。但是当主机超过 1000 时，我得到一个错误：

Unhandled Error
Traceback (most recent call last):
  File "crawler.py", line 98, in <module>

  File "/home/kalombo/.virtualenvs/dev/local/lib/python2.7/site-packages/twisted/internet/base.py", line 1192, in run

  File "/home/kalombo/.virtualenvs/dev/local/lib/python2.7/site-packages/twisted/internet/base.py", line 1201, in mainLoop

--- <exception caught here> ---
  File "/home/kalombo/.virtualenvs/dev/local/lib/python2.7/site-packages/twisted/internet/base.py", line 824, in runUntilCurrent

  File "/home/kalombo/.virtualenvs/dev/local/lib/python2.7/site-packages/pysnmp/carrier/base.py", line 52, in _cbFun

  File "/home/kalombo/.virtualenvs/dev/local/lib/python2.7/site-packages/pysnmp/entity/engine.py", line 64, in __receiveMessageCbFun

  File "/home/kalombo/.virtualenvs/dev/local/lib/python2.7/site-packages/pysnmp/proto/rfc3412.py", line 274, in receiveMessage

  File "/home/kalombo/.virtualenvs/dev/local/lib/python2.7/site-packages/pysnmp/smi/builder.py", line 299, in importSymbols

  File "/home/kalombo/.virtualenvs/dev/local/lib/python2.7/site-packages/pysnmp/smi/builder.py", line 270, in loadModules

pysnmp.smi.error.SmiError: MIB file "__SNMPv2-MIB.py[co]" not found in search path

然后脚本停止。为什么会这样？

4

2 回答 2

2

你用的是什么pysnmp？确保您使用的是最新的 pysnmp 版本。

作为旁注 - 您的方法效率极低，因为您似乎在每个 GET 操作上重新初始化 [heavy] SnmpEngine。更好的方法是为每个进程/线程保留一个持久的 SnmpEngine 实例。

于 2013-10-18T07:56:07.440 回答
2

如果这仅在您提高并发级别时发生，那么一个可能的解释是您已经遇到平台允许您在任何时候拥有的打开文件数量的限制。每个打开的套接字都计入此限制，打开“常规”文件（来自文件系统）也是如此。

如果你用完了所有允许的文件，那么 Python 就无法从磁盘读取模块的源代码，因为该平台不会让它打开它们。

在这种情况下发生的情况并不完全明显，因为（如果是的话）pysnmp 正在处理真正的异常并重新引发隐藏细节的新异常。

如果这是问题，那么您可以通过提高打开文件限制来解决它。大多数情况下，您可以通过运行：

$ ulimit -Sn 2048

阅读有关 ulimit 的更多信息（help ulimit在您的 shell 中，在互联网上），了解有关控制该限制的更多详细信息。

于 2013-10-18T11:34:51.570 回答