1.问题：我使用Keras（后端：Theano）设计了一个简单的模型来训练股票数据集。配置 Theano 使用 cpu 时，没有出现问题。但是在使用gpu的时候，就会出现如下图的问题。
In file included from /home/haichao3/anaconda2/include/python2.7/Python.h:8:0,
from mod.cu:1:
/home/haichao3/anaconda2/include/python2.7/pyconfig.h:1193:0: warning: "_POSIX_C_SOURCE" redefined [enabled by default]
In file included from /usr/local/cuda/bin/..//include/host_config.h:178:0,
from /usr/local/cuda/bin/..//include/cuda_runtime.h:78,
from <command-line>:0:
/usr/include/features.h:230:0: note: this is the location of the previous definition
In file included from /home/haichao3/anaconda2/include/python2.7/Python.h:8:0,
from mod.cu:1:
/home/haichao3/anaconda2/include/python2.7/pyconfig.h:1215:0: warning: "_XOPEN_SOURCE" redefined [enabled by default]
In file included from /usr/local/cuda/bin/..//include/host_config.h:178:0,
from /usr/local/cuda/bin/..//include/cuda_runtime.h:78,
from <command-line>:0:
/usr/include/features.h:162:0: note: this is the location of the previous definition
mod.cu(298): error: identifier "callkernel_node_a6d034c0a2ad758dff9965b4853b6097_0" is undefined
1 error detected in the compilation of "/tmp/tmpxft_000002ab_00000000-9_mod.cpp1.ii".
(None, 82, 32)
(None, 78, 16)
(None, 1248)
(None, 256)
(None, 32)
(None, 1)
['nvcc', '-shared', '-O3', '--maxrregcount=32', '-arch=sm_61', '-m64', '-Xcompiler', '-fno-math-errno,-Wno-unused-label,-Wno-unused-variable,-Wno-write-strings,-DCUDA_NDARRAY_CUH=c72d035fdf91890f3b36710688069b2e,-DNPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION,-fPIC,-fvisibility=hidden', '-Xlinker', '-rpath,/home/haichao3/.theano/compiledir_Linux-4.4--generic-x86_64-with-debian-jessie-sid-x86_64-2.7.12-64/cuda_ndarray', '-I/home/haichao3/.theano/compiledir_Linux-4.4--generic-x86_64-with-debian-jessie-sid-x86_64-2.7.12-64/cuda_ndarray', '-I/usr/local/cuda/root/include', '-I/home/haichao3/anaconda2/lib/python2.7/site-packages/numpy/core/include', '-I/home/haichao3/anaconda2/include/python2.7', '-I/home/haichao3/Theano/theano/gof', '-I/home/haichao3/Theano/theano/sandbox/cuda', '-L/home/haichao3/.theano/compiledir_Linux-4.4--generic-x86_64-with-debian-jessie-sid-x86_64-2.7.12-64/cuda_ndarray', '-L/home/haichao3/anaconda2/lib', '-o', '/home/haichao3/.theano/compiledir_Linux-4.4--generic-x86_64-with-debian-jessie-sid-x86_64-2.7.12-64/tmppSrSeT/a6d034c0a2ad758dff9965b4853b6097.so', 'mod.cu', '-lcudart', '-lcublas', '-lcuda_ndarray', '-lpython2.7']
Traceback (most recent call last):
File "model_conv_trend.py", line 44, in <module>
history = model.fit(tr_input, tr_output, nb_epoch=epoch, batch_size=train_batch_size, verbose=1,validation_data=(ts_input,ts_output))
File "/home/haichao3/anaconda2/lib/python2.7/site-packages/Keras-1.2.1-py2.7.egg/keras/models.py", line 672, in fit
initial_epoch=initial_epoch)
File "/home/haichao3/anaconda2/lib/python2.7/site-packages/Keras-1.2.1-py2.7.egg/keras/engine/training.py", line 1133, in fit
self._make_test_function()
File "/home/haichao3/anaconda2/lib/python2.7/site-packages/Keras-1.2.1-py2.7.egg/keras/engine/training.py", line 783, in _make_test_function
**self._function_kwargs)
File "/home/haichao3/anaconda2/lib/python2.7/site-packages/Keras-1.2.1-py2.7.egg/keras/backend/theano_backend.py", line 969, in function
return Function(inputs, outputs, updates=updates, **kwargs)
File "/home/haichao3/anaconda2/lib/python2.7/site-packages/Keras-1.2.1-py2.7.egg/keras/backend/theano_backend.py", line 955, in __init__
**kwargs)
File "/home/haichao3/Theano/theano/compile/function.py", line 326, in function
output_keys=output_keys)
File "/home/haichao3/Theano/theano/compile/pfunc.py", line 486, in pfunc
output_keys=output_keys)
File "/home/haichao3/Theano/theano/compile/function_module.py", line 1784, in orig_function
defaults)
File "/home/haichao3/Theano/theano/compile/function_module.py", line 1651, in create
input_storage=input_storage_lists, storage_map=storage_map)
File "/home/haichao3/Theano/theano/gof/link.py", line 699, in make_thunk
storage_map=storage_map)[:3]
File "/home/haichao3/Theano/theano/gof/vm.py", line 1063, in make_all
impl=impl))
File "/home/haichao3/Theano/theano/gof/op.py", line 924, in make_thunk
no_recycling)
File "/home/haichao3/Theano/theano/gof/op.py", line 828, in make_c_thunk
output_storage=node_output_storage)
File "/home/haichao3/Theano/theano/gof/cc.py", line 1190, in make_thunk
keep_lock=keep_lock)
File "/home/haichao3/Theano/theano/gof/cc.py", line 1131, in __compile__
keep_lock=keep_lock)
File "/home/haichao3/Theano/theano/gof/cc.py", line 1589, in cthunk_factory
key=key, lnk=self, keep_lock=keep_lock)
File "/home/haichao3/Theano/theano/gof/cmodule.py", line 1155, in module_from_key
module = lnk.compile_cmodule(location)
File "/home/haichao3/Theano/theano/gof/cc.py", line 1492, in compile_cmodule
preargs=preargs)
File "/home/haichao3/Theano/theano/sandbox/cuda/nvcc_compiler.py", line 390, in compile_str
'for cmd', ' '.join(cmd))
Exception: ('The following error happened while compiling the node', GpuElemwise{RoundHalfToEven}[(0, 0)](GpuElemwise{Composite{scalar_sigmoid((i0 + i1))}}[(0, 0)].0), '\n', 'nvcc return status', 2, 'for cmd', 'nvcc -shared -O3 --maxrregcount=32 -arch=sm_61 -m64 -Xcompiler -fno-math-errno,-Wno-unused-label,-Wno-unused-variable,-Wno-write-strings,-DCUDA_NDARRAY_CUH=c72d035fdf91890f3b36710688069b2e,-DNPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION,-fPIC,-fvisibility=hidden -Xlinker -rpath,/home/haichao3/.theano/compiledir_Linux-4.4--generic-x86_64-with-debian-jessie-sid-x86_64-2.7.12-64/cuda_ndarray -I/home/haichao3/.theano/compiledir_Linux-4.4--generic-x86_64-with-debian-jessie-sid-x86_64-2.7.12-64/cuda_ndarray -I/usr/local/cuda/root/include -I/home/haichao3/anaconda2/lib/python2.7/site-packages/numpy/core/include -I/home/haichao3/anaconda2/include/python2.7 -I/home/haichao3/Theano/theano/gof -I/home/haichao3/Theano/theano/sandbox/cuda -L/home/haichao3/.theano/compiledir_Linux-4.4--generic-x86_64-with-debian-jessie-sid-x86_64-2.7.12-64/cuda_ndarray -L/home/haichao3/anaconda2/lib -o /home/haichao3/.theano/compiledir_Linux-4.4--generic-x86_64-with-debian-jessie-sid-x86_64-2.7.12-64/tmppSrSeT/a6d034c0a2ad758dff9965b4853b6097.so mod.cu -lcudart -lcublas -lcuda_ndarray -lpython2.7', '[GpuElemwise{RoundHalfToEven}[(0, 0)](<CudaNdarrayType(float32, matrix)>)]')
2.硬件：Titan X Maxwell，驱动版本367.48，cuda V8.0.44。cuDNN 5. 我注意到我在安装时被警告我的 cuDNN 版本比 Theano 版本太新。
3.这是源代码：
#!/home/xxx/anaconda2/bin/python
import numpy as np
import scipy.io as sio
from keras.models import Sequential
from keras.layers import Dense, Activation, Convolution1D, Flatten
from keras.optimizers import SGD
from data_provider import DataProvider
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from time import sleep
past_len = 90
predict_len = 5
train_batch_size = 128
test_batch_size = 64
epoch = 10
lr = 1e-3 #important
decay = 1e-5
init = 'he_normal'
folders = ['./data/table1', './data/table2', './data/table3']
provider = DataProvider(folders, kind='trend', load=True, data_path = './data/data_conv_trend.h5')
tr_input, tr_output, ts_input, ts_output = provider.prepare(20091231, past_len, predict_len)
tr_output = tr_output.reshape(tr_output.shape[0], 1)
ts_output = ts_output.reshape(ts_output.shape[0], 1)
model = Sequential()
model.add(Convolution1D(64, 5, activation = 'relu', init=init, border_mode='valid', input_shape=(past_len, 5)))
model.add(Convolution1D(32, 5, activation = 'relu', init=init, border_mode='valid'))
model.add(Convolution1D(16, 5, activation = 'relu', init=init, border_mode='valid'))
model.add(Flatten())
model.add(Dense(256, activation = 'sigmoid', init=init))
model.add(Dense(32, activation = 'sigmoid', init=init))
model.add(Dense(1, activation='sigmoid', init=init))
for i in range(-6, 0):
print(model.layers[i].output_shape)
#model.load_weights('./mdl/weights.h5');
model.compile(loss='binary_crossentropy', optimizer=SGD(lr=lr,decay=decay,momentum=0.9,nesterov=False), metrics=['accuracy'])
history = model.fit(tr_input, tr_output, nb_epoch=epoch, batch_size=train_batch_size, verbose=1,validation_data=(ts_input,ts_output))
model.save('./mdl/model_' + str(lr) + '_' + str(decay) + '_' + init + '_' + '.h5')