1

我的最简单形式的代码如下所示。我正在尝试从 jupyter 笔记本连接到配置单元。当我查询较少数量的行时，我的代码工作正常，说“从表限制 200 中选择 *”，但是当我执行“从表中选择 *”之类的操作时抛出此错误，表大约 180MB，可以轻松加载到内存中。我不确定为什么会这样。任何帮助深表感谢！我查看了其他类似的问题，但它们没有帮助

package versions -
    python - 3.7
    pandas - 0.24.2
    pyhive 0.6.1

import pandas
import os
from pyhive import hive
def hiveconnection(query):
    conn = hive.Connection(host=HOST, port=10000, username=USER, 
                       password=PASSWORD, auth='LDAP')
    cur = conn.cursor()
    cur.execute(query)
    result = cur.fetchall()
    conn.close()

    return result
query = """select * from table"""
df_new = hiveconnection(query)

查询运行良好，我得到较小结果集的结果说 -select * from table limit 200 但是当我查询大量行时出现以下错误。

-------------------------------------------------- ------------------------- TTransportException Traceback（最近一次调用最后一次）在 1 个查询 = """select * from sample_table""" --- -> 2 df_new = hiveconnection(查询)

在 hiveconnection(query) 7 cur = conn.cursor() 8 cur.execute(query) ----> 9 result = cur.fetchall() 10 conn.close() 11

/usr/local/anaconda/lib/python3.7/site-packages/pyhive/common.py in fetchall(self) 134 :py:meth:execute未产生任何结果集或尚未发出调用。135 """ --> 136 返回列表(iter(self.fetchone, None)) 137 138 @property

/usr/local/anaconda/lib/python3.7/site-packages/pyhive/common.py in fetchone(self) 103 104 # 休眠直到我们完成或者我们有一些数据要返回 --> 105 self._fetch_while (lambda: not self._data and self._state != self._STATE_FINISHED) 106 107 if not self._data:

/usr/local/anaconda/lib/python3.7/site-packages/pyhive/common.py in _fetch_while(self, fn) 43 def _fetch_while(self, fn): 44 while fn(): ---> 45 self ._fetch_more() 46 if fn(): 47 time.sleep(self._poll_interval)

/usr/local/anaconda/lib/python3.7/site-packages/pyhive/hive.py in _fetch_more(self) 384 maxRows=self.arraysize, 385) --> 386 response = self._connection.client.FetchResults(请求）387 _check_status（响应）388 架构 = self.description

/usr/local/anaconda/lib/python3.7/site-packages/TCLIService/TCLIService.py 在 FetchResults(self, req) 712 """ 713 self.send_FetchResults(req) --> 714 return self.recv_FetchResults() 715 716 def send_FetchResults（自我，请求）：

/usr/local/anaconda/lib/python3.7/site-packages/TCLIService/TCLIService.py in recv_FetchResults(self) 731 raise x 732 result = FetchResults_result() --> 733 result.read(iprot) 734 iprot.readMessageEnd () 735 如果 result.success 不是无：

/usr/local/anaconda/lib/python3.7/site-packages/TCLIService/TCLIService.py in read(self, iprot) 3468 if ftype == TType.STRUCT: 3469 self.success = TFetchResultsResp() -> 3470 self .success.read(iprot) 3471 其他：3472 iprot.skip(ftype)

/usr/local/anaconda/lib/python3.7/site-packages/TCLIService/ttypes.py in read(self, iprot) 6581 if ftype == TType.STRUCT: 6582 self.results = TRowSet() -> 6583 self .results.read(iprot) 6584 其他：6585 iprot.skip(ftype)

/usr/local/anaconda/lib/python3.7/site-packages/TCLIService/ttypes.py in read(self, iprot) 2865 for _i114 in range(_size110): 2866 _elem115 = TColumn() -> 2867 _elem115.read (iprot) 2868 self.columns.append(_elem115) 2869
iprot.readListEnd()

/usr/local/anaconda/lib/python3.7/site-packages/TCLIService/ttypes.py in read(self, iprot) 2727 if ftype == TType.STRUCT: 2728 self.i64Val = TI64Column() -> 2729 self .i64Val.read(iprot) 2730 其他：2731 iprot.skip(ftype)

/usr/local/anaconda/lib/python3.7/site-packages/TCLIService/ttypes.py in read(self, iprot) 2351 (_etype79, _size76) = iprot.readListBegin() 2352 for _i80 in range(_size76): -> 2353 _elem81 = iprot.readI64() 2354 self.values.append(_elem81) 2355
iprot.readListEnd()

/usr/local/anaconda/lib/python3.7/site-packages/thrift/protocol/TBinaryProtocol.py in readI64(self) 220 221 def readI64(self): --> 222 buff = self.trans.readAll(8 ) 223 val, = unpack('!q', buff) 224 返回值

/usr/local/anaconda/lib/python3.7/site-packages/thrift/transport/TTransport.py in readAll(self, sz) 60 have = 0 61 while (have < sz): ---> 62 chunk = self.read(sz - have) 63 chunkLen = len(chunk) 64 have += chunkLen

/usr/local/anaconda/lib/python3.7/site-packages/thrift_sasl/ init .py in read(self, sz) 171 return ret 172 --> 173 self._read_frame() 174 return ret + self.__rbuf。读取（sz - len（ret））175

/usr/local/anaconda/lib/python3.7/site-packages/thrift_sasl/ init .py in _read_frame(self) 188 else: 189 # 如果帧没有编码，直接通过 --> 190 decoded = self ._trans.read(length) 191 self.__rbuf = BufferIO(decoded) 192

/usr/local/anaconda/lib/python3.7/site-packages/thrift/transport/TSocket.py in read(self, sz) 141 if len(buff) == 0: 142 raise TTransportException(type=TTransportException.END_OF_FILE , --> 143 message='TSocket read 0 bytes') 144 return buff 145

TTransportException：TSocket 读取 0 个字节

4

0 回答 0