我无法tf.data为腌制数据帧列表构建管道（在 python 3.7.7 和 Windows 10 上使用 Tensorflow 2.1.0）。要开始使用以下代码：
import numpy as np
import pandas as pd
import os, pickle
import tensorflow as tf
from tensorflow.keras.preprocessing.sequence import pad_sequences
from sklearn.preprocessing import MultiLabelBinarizer
MAX_LEN = 8
NULL_VALUE = -1
# creating the sample dataframe
df = pd.DataFrame({'SOURCE': [[0, 0, 0, 0], [0, 0, 0, 0, 1, 1, 1, 2, 2], [0, 0, 0], [0, 0, 0, 1, 1, 1, 0], [0, 0, 0, 1, 1, 1, 0, 1, 2, 2, 2], [0, 0, 0, 0, 1, 1, 1], [-1, -1, -1, -1, -1, -1, -1, -1], [0, 0, 0, 1, 1, 1, 0, 1]], 'TIME': [[0, 0, 0, 0], [0, 0, 0, 0, 1, 1, 1, 2, 2], [0, 0, 0], [0, 0, 0, 1, 1, 1, 2], [0, 0, 0, 1, 1, 1, 2, 7, 2, 2, 2], [0, 0, 0, 0, 1, 1, 1], [-1, -1, -1, -1, -1, -1, -1, -1], [0, 0, 0, 1, 1, 1, 2, 7]], 'DATE': [[0, 0, 0, 0], [0, 0, 0, 0, 1, 1, 1, 2, 2], [0, 0, 0], [0, 0, 0, 1, 1, 1, 2], [0, 0, 0, 1, 1, 1, 2, 7, 2, 2, 2], [0, 0, 0, 0, 1, 1, 1], [-1, -1, -1, -1, -1, -1, -1, -1], [0, 0, 0, 1, 1, 1, 2, 7]], 'CAT_AF1': [[1, 2, 3, 4], [1, 2, 3, 4, 1, 2, 3, 1, 2], [0, 0, 0], [0, 0, 0, 1, 1, 1, 2], [0, 0, 0, 1, 1, 1, 2, 7, 2, 2, 2], [1, 2, 3, 4, 1, 2, 3], [-1, -1, -1, -1, -1, -1, -1, -1], [0, 0, 0, 1, 1, 1, 2, 7]], 'CAT_AF2': [[1, 2, 3, 4], [1, 2, 3, 4, 1, 2, 3, 1, 2], [0, 0, 0], [0, 0, 0, 1, 1, 1, 2], [0, 0, 0, 1, 1, 1, 2, 7, 2, 2, 2], [1, 2, 3, 4, 1, 2, 3], [-1, -1, -1, -1, -1, -1, -1, -1], [0, 0, 0, 1, 1, 1, 2, 7]], 'PROD': [[1, 2, 3, 4], [1, 2, 3, 4, 1, 2, 3, 1, 2], [0, 0, 0], [0, 0, 0, 1, 1, 1, 2], [0, 0, 0, 1, 1, 1, 2, 7, 2, 2, 2], [1, 2, 3, 4, 1, 2, 3], [-1, -1, -1, -1, -1, -1, -1, -1], [0, 0, 0, 1, 1, 1, 2, 7]], 'PROD_Y': [[1, 2, 3], [1, 2, 3, 4], [1], [7], [3], [1, 2], [0], [2]]})
FEATURES = [col for col in df.columns if col != 'PROD_Y']
# saving the same dataframe as multiple pickle files
os.mkdir('data')
os.mkdir(os.path.join('data', 'pickles'))
for n in range(10):
with open(os.path.join('data', 'pickles', 'pickle-n-{n}.pkl'.format(n=n)), 'wb') as f:
pickle.dump(df, f, pickle.HIGHEST_PROTOCOL)
def read_pickles(filename):
with open(filename, 'rb') as f:
df_ = pickle.load(f)
return df_
def pad_(x):
return pad_sequences([x], dtype=np.int32, maxlen=MAX_LEN, padding='pre', truncating='pre', value=NULL_VALUE)[0]
def parse_files(filename):
df_ = read_pickles(filename)
MLB = MultiLabelBinarizer(classes=list(set([j for i in df_['PROD_Y'].tolist() for j in i])))
MLB.fit(y=df_['PROD_Y'])
for col in FEATURES:
df_[col] = df_[col].apply(pad_)
df_['PROD_Y'] = MLB.transform(df_['PROD_Y']).tolist()
return dict(df_)
# return tf.data.Dataset.from_tensor_slices(dict(df_))
# return df_.to_dict('list')
def parse_(filename):
out = tf.py_function(parse_files, inp=[filename], Tout=[tf.float32 for k in FEATURES]) # not sure how to write the dict returned into Tout argument
return out
list_of_files = [os.path.join('data', 'pickles', f) for f in os.listdir(os.path.join('data', 'pickles'))]
dataset = tf.data.Dataset.from_tensor_slices(list_of_files)
dataset_2 = dataset.map(parse_)
a = [d for d in dataset_2]
在最后一行，它在 内引发以下错误read_pickles：
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb0 in position 0: invalid start byte
我尝试使用filename.decode('utf-8')withinread_pickles但这也会导致另一个错误：
AttributeError: 'tensorflow.python.framework.ops.EagerTensor' object has no attribute 'decode'
当我搜索它时，我发现了这些，但它们对这个问题没有帮助：
完整追溯：
2021-05-16 19:59:01.568800: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library cudart64_101.dll
2021-05-16 19:59:04.926696: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library nvcuda.dll
2021-05-16 19:59:06.100711: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties:
pciBusID: 0000:01:00.0 name: GeForce GTX 1660 Ti computeCapability: 7.5
coreClock: 1.59GHz coreCount: 24 deviceMemorySize: 6.00GiB deviceMemoryBandwidth: 268.26GiB/s
2021-05-16 19:59:06.100787: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library cudart64_101.dll
2021-05-16 19:59:06.109139: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library cublas64_10.dll
2021-05-16 19:59:06.114618: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library cufft64_10.dll
2021-05-16 19:59:06.117511: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library curand64_10.dll
2021-05-16 19:59:06.124338: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library cusolver64_10.dll
2021-05-16 19:59:06.129912: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library cusparse64_10.dll
2021-05-16 19:59:06.142492: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library cudnn64_7.dll
2021-05-16 19:59:06.142600: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 0
2021-05-16 19:59:06.143031: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2
2021-05-16 19:59:06.144206: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties:
pciBusID: 0000:01:00.0 name: GeForce GTX 1660 Ti computeCapability: 7.5
coreClock: 1.59GHz coreCount: 24 deviceMemorySize: 6.00GiB deviceMemoryBandwidth: 268.26GiB/s
2021-05-16 19:59:06.144245: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library cudart64_101.dll
2021-05-16 19:59:06.144265: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library cublas64_10.dll
2021-05-16 19:59:06.144281: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library cufft64_10.dll
2021-05-16 19:59:06.144299: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library curand64_10.dll
2021-05-16 19:59:06.144314: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library cusolver64_10.dll
2021-05-16 19:59:06.144328: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library cusparse64_10.dll
2021-05-16 19:59:06.144343: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library cudnn64_7.dll
2021-05-16 19:59:06.144390: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 0
2021-05-16 19:59:06.866245: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix:
2021-05-16 19:59:06.866285: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 0
2021-05-16 19:59:06.866300: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 0: N
2021-05-16 19:59:06.866531: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 4755 MB memory) -> physical GPU (device: 0, name: GeForce GTX 1660 Ti, pci bus id: 0000:01:00.0, compute capability: 7.5)
2021-05-16 19:59:07.063641: W tensorflow/core/framework/op_kernel.cc:1643] Invalid argument: UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb0 in position 0: invalid start byte
Traceback (most recent call last):
File "C:\Users\D\Miniconda3\envs\Dev\lib\site-packages\tensorflow_core\python\ops\script_ops.py", line 234, in __call__
return func(device, token, args)
File "C:\Users\D\Miniconda3\envs\Dev\lib\site-packages\tensorflow_core\python\ops\script_ops.py", line 123, in __call__
ret = self._func(*args)
File "stack-overflow-tf-data-question.py", line 33, in parse_files
df_ = read_pickles(filename)
File "stack-overflow-tf-data-question.py", line 25, in read_pickles
with open(filename, 'rb') as f:
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb0 in position 0: invalid start byte
2021-05-16 19:59:07.063812: W tensorflow/core/framework/op_kernel.cc:1655] OP_REQUIRES failed at iterator_ops.cc:941 : Invalid argument: UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb0 in position 0: invalid start byte
Traceback (most recent call last):
File "C:\Users\D\Miniconda3\envs\Dev\lib\site-packages\tensorflow_core\python\ops\script_ops.py", line 234, in __call__
return func(device, token, args)
File "C:\Users\D\Miniconda3\envs\Dev\lib\site-packages\tensorflow_core\python\ops\script_ops.py", line 123, in __call__
ret = self._func(*args)
File "stack-overflow-tf-data-question.py", line 33, in parse_files
df_ = read_pickles(filename)
File "stack-overflow-tf-data-question.py", line 25, in read_pickles
with open(filename, 'rb') as f:
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb0 in position 0: invalid start byte
[[{{node EagerPyFunc}}]]
stack-overflow-tf-data-question.py:25: DeprecationWarning: path should be string, bytes, or os.PathLike, not tensorflow.python.framework.ops.EagerTensor
with open(filename, 'rb') as f:
Traceback (most recent call last):
File "C:\Users\D\Miniconda3\envs\Dev\lib\site-packages\tensorflow_core\python\eager\context.py", line 1897, in execution_mode
yield
File "C:\Users\D\Miniconda3\envs\Dev\lib\site-packages\tensorflow_core\python\data\ops\iterator_ops.py", line 659, in _next_internal
output_shapes=self._flat_output_shapes)
File "C:\Users\D\Miniconda3\envs\Dev\lib\site-packages\tensorflow_core\python\ops\gen_dataset_ops.py", line 2479, in iterator_get_next_sync
_ops.raise_from_not_ok_status(e, name)
File "C:\Users\D\Miniconda3\envs\Dev\lib\site-packages\tensorflow_core\python\framework\ops.py", line 6606, in raise_from_not_ok_status
six.raise_from(core._status_to_exception(e.code, message), None)
File "<string>", line 3, in raise_from
tensorflow.python.framework.errors_impl.InvalidArgumentError: UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb0 in position 0: invalid start byte
Traceback (most recent call last):
File "C:\Users\D\Miniconda3\envs\Dev\lib\site-packages\tensorflow_core\python\ops\script_ops.py", line 234, in __call__
return func(device, token, args)
File "C:\Users\D\Miniconda3\envs\Dev\lib\site-packages\tensorflow_core\python\ops\script_ops.py", line 123, in __call__
ret = self._func(*args)
File "stack-overflow-tf-data-question.py", line 33, in parse_files
df_ = read_pickles(filename)
File "stack-overflow-tf-data-question.py", line 25, in read_pickles
with open(filename, 'rb') as f:
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb0 in position 0: invalid start byte
[[{{node EagerPyFunc}}]] [Op:IteratorGetNextSync]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "stack-overflow-tf-data-question.py", line 52, in <module>
a = [d for d in dataset_2]
File "stack-overflow-tf-data-question.py", line 52, in <listcomp>
a = [d for d in dataset_2]
File "C:\Users\D\Miniconda3\envs\Dev\lib\site-packages\tensorflow_core\python\data\ops\iterator_ops.py", line 630, in __next__
return self.next()
File "C:\Users\D\Miniconda3\envs\Dev\lib\site-packages\tensorflow_core\python\data\ops\iterator_ops.py", line 674, in next
return self._next_internal()
File "C:\Users\D\Miniconda3\envs\Dev\lib\site-packages\tensorflow_core\python\data\ops\iterator_ops.py", line 665, in _next_internal
return structure.from_compatible_tensor_list(self._element_spec, ret)
File "C:\Users\D\Miniconda3\envs\Dev\lib\contextlib.py", line 130, in __exit__
self.gen.throw(type, value, traceback)
File "C:\Users\D\Miniconda3\envs\Dev\lib\site-packages\tensorflow_core\python\eager\context.py", line 1900, in execution_mode
executor_new.wait()
File "C:\Users\D\Miniconda3\envs\Dev\lib\site-packages\tensorflow_core\python\eager\executor.py", line 67, in wait
pywrap_tensorflow.TFE_ExecutorWaitForAllPendingNodes(self._handle)
tensorflow.python.framework.errors_impl.InvalidArgumentError: UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb0 in position 0: invalid start byte
Traceback (most recent call last):
File "C:\Users\D\Miniconda3\envs\Dev\lib\site-packages\tensorflow_core\python\ops\script_ops.py", line 234, in __call__
return func(device, token, args)
File "C:\Users\D\Miniconda3\envs\Dev\lib\site-packages\tensorflow_core\python\ops\script_ops.py", line 123, in __call__
ret = self._func(*args)
File "stack-overflow-tf-data-question.py", line 33, in parse_files
df_ = read_pickles(filename)
File "stack-overflow-tf-data-question.py", line 25, in read_pickles
with open(filename, 'rb') as f:
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb0 in position 0: invalid start byte
[[{{node EagerPyFunc}}]]
感谢你的帮助！蒂亚！