0

我在我的项目中使用 pgdb(PyGreSQL) 连接到工作数据库。我编写了生成器来从巨大的表格中获取带有分页的行。

# DbHelper.py

...

def GetMultipleRowsGenerator(self, query, page_size=None):
    """Execute a query and return the generator
    for a fetching rows
    """
    if self._condition:
        self._condition.acquire()
    try:
        self.VerifyConnection()
        return pgsqlUtils.dbGetMultipleRowsGenerator(self._dbConnection, query, page_size)
    finally:
        if self._condition:
            self._condition.release()

...

# pgsqlUtils.py

...

def dbGetMultipleRowsGenerator(db, query, page_size=None):
    if not query.strip().lower().startswith('select '):
        raise ValueError("Bad query '%s'. Query must starts with 'SELECT ...' for a generator." % query)

    if page_size is not None:
        try:
            page_size = int(page_size)
        except TypeError:
            raise ValueError("Expected int type for 'page_size' (%s given)." % page_size)

        if page_size <= 0:
            raise ValueError("The 'page_size' must be greater than 0 (but %s given)." % page_size)

    def get_generator(q):
        c = None
        try:
            c = db.cursor()
        except SyntaxError:
            pass

        try:
            c.execute(q)

            while True:
                r = c.fetchone()
                if r is None:
                    break
                yield r

        except SyntaxError:
            pass
        finally:
            if c is not None:
                c.close()
            db.commit()

    orig_query = query
    page = -1

    while True:
        query = orig_query
        if page_size is not None:
            page += 1
            offset = page * page_size
            query = "%s OFFSET %s LIMIT %s" % (orig_query, offset, page_size)

        generator = get_generator(query)

        row = None
        while True:
            try:
                row = next(generator)
            except StopIteration:
                break

            yield row

        if row is None:
            break  # no rows - end of table

...

使用生成器：

curr_data_generator = \
    self.conn.GetMultipleRowsGenerator("SELECT * FROM %s" % temp_tablename, page_size=100000)

while True: 
    try:
        data = next(curr_data_generator)
    except StopIteration:
        break

    # do something with data ...

它运行良好，但有时我会遇到以下异常：

   File "pgsqlUtils.py", line 111, in dbGetMultipleRowsGenerator
   File "pgsqlUtils.py", line 84, in get_generator
   File "/usr/myproject/depot/Python-3.4.3/lib/python3.4/site-packages/PyGreSQL-5.0-py3.4-linux-x86_64.egg/pgdb.py", line 958, in fetchone
    res = self.fetchmany(1, False)
   File "/usr/myproject/depot/Python-3.4.3/lib/python3.4/site-packages/PyGreSQL-5.0-py3.4-linux-x86_64.egg/pgdb.py", line 981, in fetchmany
    result = self._src.fetch(size)
 pg.DatabaseError: Last query did not return tuples

环境细节

  • 蟒蛇版本：3.4.3
  • 操作系统：CentOS 6.6 版
  • 我在多线程模式下使用此代码。

为什么会这样？我该如何解决？

4

0 回答 0