4

我已向 cloud ml 提交了培训作业。但是，它找不到 csv 文件。它在桶里。这是代码。

# Use scikit-learn to grid search the batch size and epochs
import numpy
from sklearn.model_selection import GridSearchCV
from keras.models import Sequential
from keras.layers import Dense
from keras.wrappers.scikit_learn import KerasClassifier

def create_model():
    model = Sequential()
    model.add(Dense(12, input_dim=11, activation='relu'))
    model.add(Dense(1, activation='sigmoid'))
    model.compile(loss='binary_crossentropy', optimizer='nadam', metrics=['accuracy'])
    return model
seed = 7
numpy.random.seed(seed)

FIL = "gs://bubbly-hexagon-112008-ml/dataset/mixed.csv"
dataset = numpy.loadtxt(FIL, delimiter=",")
X = dataset[:,0:11]
Y = dataset[:,11]

model = KerasClassifier(build_fn=create_model, verbose=1)
batch_size = [10, 20, 40, 60, 80, 100]
epochs = [10, 50, 100, 500, 1000]
param_grid = dict(batch_size=batch_size, nb_epoch=epochs)
grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1)
grid_result = grid.fit(X, Y)
print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']
for mean, stdev, param in zip(means, stds, params):
    print("%f (%f) with: %r" % (mean, stdev, param))

提交作业后，我收到此错误。

Traceback (most recent call last): File "/usr/lib/python2.7/runpy.py", line 162, in _run_module_as_main "__main__", fname, loader, pkg_name) File "/usr/lib/python2.7/runpy.py", line 72, in _run_code exec code in
run_globals File "/root/.local/lib/python2.7/
site-packages/trainer/task.py", line 18, in <module> dataset = numpy.loadtxt(FIL, delimiter=",") File "/root/.local/lib/python2.7/
site-packages/numpy/lib/npyio.py", line 803, in loadtxt fh = iter(open(fname, 'U')) IOError: [Errno 2] No such file or directory: 
'gs://bubbly-hexagon-112008-ml/dataset/mixed.csv'

-文件在指定的bucket中，其权限包括cloud ml作为阅读器。

-我也用来gcloud beta ml init-project初始化项目。

- 我创建了一个新的存储桶并将文件放在那里，但得到了同样的错误。

-我的存储桶与我提交的作业位于同一区域。

谢谢

4

3 回答 3

0

file_io来自 tensorflow 的效果很好：

from tensorflow.python.lib.io import file_io
import numpy as np
import json

要读取一个 numpy 数组： 

with file_io.FileIO(path_npx, 'rb') as f:
    np_arr = np.load( BytesIO(f.read()) )
    print(np_arr)

读取 json 文件：

with file_io.FileIO(path_json, 'r') as f:
    print(json.loads(f.read()))
于 2019-04-08T13:40:14.883 回答
0

你不能直接从 gfs 中读取，你需要使用某种 io 库。

from io import BytesIO
import tensorflow as tf
import numpy as np
from tensorflow.python.lib.io import file_io

FIL = "gs://bubbly-hexagon-112008-ml/dataset/mixed.csv"
f = BytesIO(file_io.read_file_to_string(FIL, binary_mode=True))
data = np.load(f)
于 2018-12-19T15:26:09.513 回答
-1

我不认为你可以用 numpy 直接读取 gcs 文件。

于 2017-01-07T20:17:25.757 回答