新设置：2x2080ti Nvidia 驱动程序：430 Cuda 10.0 Cudnn 7.6 Tensorflow 1.13.1
旧设置：2x1080ti Nvidia 驱动程序：410 Cuda 9.0 Tensorflow 1.10
我实现了一个分割模型，它可以在 FP32 或混合精度下进行训练（按照此处的说明http://on-demand.gputechconf.com/gtc-taiwan/2018/pdf/5-1_Internal%20Speaker_Michael%20Carilli_PDF%20For% 20 共享.pdf )。
它适用于旧设置，但 1080ti 不完全支持 float16，这就是我切换到新设置的原因。
在新设置中，FP32 工作正常，但混合精度总是有错误： tensorflow.python.framework.errors_impl.InternalError: RET_CHECK failure (tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc:3171) ShapeUtil::Equal(first_reduce ->形状（），inst->形状（））
模型结构：
with tf.name_scope('Inputs'):
is_training_tensor = tf.placeholder(dtype=tf.bool, shape=(), name='is_training')
input_tensor = tf.placeholder(dtype=tf.float32, shape=set_shape(hypes, hypes['arch']['num_channels']),
name='inputs')
if hypes['arch']['half_precision']:
input_tensor = tf.cast(input_tensor, tf.float16)
binary_label_tensors = []
for label in hypes['data']['predict_labels']:
binary_label_tensor = tf.placeholder(dtype=tf.int64, shape=set_shape(hypes, 1, is_input=False), name=label)
binary_label_tensors.append(binary_label_tensor)
tower_grads = []
loss_dicts = []
eval_dicts = []
with tf.name_scope('Optimizer'):
opt, step = create_optimizer_wrapper(hypes)
with tf.variable_scope('ModelCrossGPUs', reuse=tf.AUTO_REUSE, custom_getter=float32_variable_storage_getter
if hypes['arch']['half_precision'] else None):
for i in range(gpus):
with tf.device('/device:GPU:{}'.format(i)):
with tf.name_scope('GPU_{}'.format(i)):
# restructure input
input_tensor_gpu = input_tensor[i * batch_size: (i + 1) * batch_size]
binary_label_tensors_gpu = []
for tensor in binary_label_tensors:
binary_label_tensors_gpu.append(tensor[i * batch_size: (i + 1) * batch_size])
# instantiate the network
net_module = getattr(importlib.import_module('ml.projects.xxx.nets.' +
hypes['arch']['net']), 'inference')
inference_net = net_module(hypes,
input_tensor=input_tensor_gpu,
is_training_tensor=is_training_tensor)
if hypes['arch']['half_precision']:
logitss = [tf.cast(logits, tf.float32) for logits in inference_net['logitss']]
else:
logitss = inference_net['logitss']
binary_seg_rets = inference_net['binary_seg_rets']
with tf.name_scope('Loss'):
loss_dict = loss.multi_binary_segmentation_loss(hypes, input_tensor_gpu,
binary_label_tensors_gpu, logitss)
loss_dict.update({'total_loss': loss.consolidation_loss(loss_dict['binary_seg_loss'])})
loss_dicts.append(loss_dict)
with tf.name_scope('Evaluation'):
evaluator = eval.Evaluator()
eval_dict = evaluator.eval_logits(hypes, input_tensor_gpu, binary_label_tensors_gpu, logitss)
eval_dicts.append(eval_dict)
with tf.name_scope('Gradients'):
grads = single_gradients(hypes, loss_dict['total_loss'], opt)
tower_grads.append(grads)
with tf.name_scope('Summary_Train/'):
with tf.name_scope('Summary_Train_{}'.format(i)):
add_tensor_to_summary(hypes, input_tensor_gpu, binary_label_tensors_gpu, inference_net)
for grad in grads:
tf.summary.histogram("Gradient/" + grad.name.split(':')[0], grad)
with tf.name_scope('Summary_Eval/'):
with tf.name_scope('Summary_Eval_{}'.format(i)):
add_tensor_to_summary(hypes, input_tensor_gpu, binary_label_tensors_gpu, inference_net)
with tf.name_scope('Optimizer'):
grads = average_gradients(tower_grads)
train_op = global_optimizer(grads, opt, step)
错误发生在这里：
binary_label = tf.multiply(binary_label, mask)
is_binary_label_one = tf.equal(binary_label, 1)
is_out_one = tf.equal(out, 1)
# Ground truth
t = tf.count_nonzero(binary_label, dtype=tf.int64)
# Prediction
p = tf.count_nonzero(out, dtype=tf.int64)
# Union
u = tf.count_nonzero(tf.logical_or(is_binary_label_one, is_out_one))
# Intersection
i = tf.count_nonzero(tf.logical_and(is_binary_label_one, is_out_one))
# Valid mask region
m = tf.count_nonzero(mask)
# correct prediction including both positive and negative prediction
c = tf.count_nonzero(tf.logical_and(tf.equal(binary_label, out), tf.equal(mask, 1)))
one = tf.constant(1.0, dtype=tf.float64)
accuracy = tf.cond(tf.equal(m, 0), lambda: one, lambda: c / m)
precision = tf.cond(tf.equal(p, 0), lambda: one, lambda: i / p)
recall = tf.cond(tf.equal(t, 0), lambda: one, lambda: i / t)
iou = tf.cond(tf.equal(u, 0), lambda: one, lambda: i / u)
f1 = tf.cond(tf.equal(precision + recall, 0), lambda: one, lambda: 2 * precision * recall /
(precision + recall))
错误：
* Begin stack trace
tensorflow::Status xla::HloInstruction::Visit<xla::HloInstruction*>(xla::DfsHloVisitorBase<xla::HloInstruction*>*)
tensorflow::Status xla::HloInstruction::Accept<xla::HloInstruction*>(xla::DfsHloVisitorBase<xla::HloInstruction*>*, bool, bool)
tensorflow::Status xla::HloComputation::Accept<xla::HloInstruction*>(xla::DfsHloVisitorBase<xla::HloInstruction*>*) const
xla::gpu::NVPTXCompiler::RunBackend(std::unique_ptr<xla::HloModule, std::default_delete<xla::HloModule> >, stream_executor::StreamExecutor*, xla::DeviceMemoryAllocator*)
xla::Service::BuildExecutable(xla::HloModuleProto const&, std::unique_ptr<xla::HloModuleConfig, std::default_delete<xla::HloModuleConfig> >, xla::Backend*, stream_executor::StreamExecutor*, xla::DeviceMemoryAllocator*
tensorflow::XlaCompilationCache::BuildExecutable(tensorflow::XlaCompiler::Options const&, tensorflow::XlaCompiler::CompilationResult const&, std::unique_ptr<xla::LocalExecutable, std::default_delete<xla::LocalExecutable> >*)
tensorflow::XlaCompilationCache::CompileImpl(tensorflow::XlaCompiler::Options const&, tensorflow::NameAttrList const&, absl::Span<tensorflow::XlaCompiler::Argument const>, std::function<tensorflow::Status (tensorflow::XlaCompiler*, tensorflow::XlaCompiler::CompilationResult*)> const&, absl::optional<long long>, tensorflow::XlaCompiler::CompilationResult const**, xla::LocalExecutable**)
tensorflow::XlaCompilationCache::Compile(tensorflow::XlaCompiler::Options const&, tensorflow::NameAttrList const&, absl::Span<tensorflow::XlaCompiler::Argument const>, tensorflow::XlaCompiler::CompileOptions const&, tensorflow::XlaCompilationCache::CompileMode, tensorflow::XlaCompiler::CompilationResult const**, xla::LocalExecutable**)
tensorflow::XlaCompileOp::Compute(tensorflow::OpKernelContext*)
tensorflow::BaseGPUDevice::ComputeHelper(tensorflow::OpKernel*, tensorflow::OpKernelContext*)
tensorflow::BaseGPUDevice::Compute(tensorflow::OpKernel*, tensorflow::OpKernelContext*)
Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&)
clone
*End stack trace
2019-06-03 21:16:54.599314: W tensorflow/core/framework/op_kernel.cc:1401]
OP_REQUIRES failed at xla_ops.cc:429 : Internal: RET_CHECK failure (tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc:3171) ShapeUtil::Equal(first_reduce->shape(), inst->shape())
Traceback (most recent call last):
File "/home/usr/workspace/virtualenvs/xxx/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1334, in _do_call
return fn(*args)
File "/home/usr/workspace/virtualenvs/xxx/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1319, in _run_fn
options, feed_dict, fetch_list, target_list, run_metadata)
File "/home/usr/workspace/virtualenvs/xxx/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1407, in _call_tf_sessionrun
run_metadata)
tensorflow.python.framework.errors_impl.InternalError: RET_CHECK failure (tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc:3171) ShapeUtil::Equal(first_reduce->shape(), inst->shape())
[[{{node cluster_26_1/xla_compile}}]]
[[{{node ModelCrossGPUs/GPU_0/Evaluation/cond_2/Merge}}]]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/usr/pycharm/pycharm-community-2018.3.5/helpers/pydev/pydevd.py", line 1741, in <module>
main()
File "/home/usr/pycharm/pycharm-community-2018.3.5/helpers/pydev/pydevd.py", line 1735, in main
globals = debugger.run(setup['file'], None, None, is_module)
File "/home/usr/pycharm/pycharm-community-2018.3.5/helpers/pydev/pydevd.py", line 1135, in run
pydev_imports.execfile(file, globals, locals) # execute the script
File "/home/usr/pycharm/pycharm-community-2018.3.5/helpers/pydev/_pydev_imps/_pydev_execfile.py", line 18, in execfile
exec(compile(contents+"\n", file, 'exec'), glob, loc)
File "/home/usr/workspace/projects/xxx/train.py", line 201, in <module>
tf.app.run()
File "/home/usr/workspace/virtualenvs/xxx/lib/python3.6/site-packages/tensorflow/python/platform/app.py", line 125, in run
_sys.exit(main(argv))
File "/home/usr/workspace/projects/xxx/train.py", line 197, in main
train_net(hypes, graph, session, run_options, itr_init)
File "/home/usr/workspace/projects/xxx/train.py", line 107, in train_net
run_metadata=run_options['metadata'])
File "/home/usr/workspace/virtualenvs/xxx/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 929, in run
run_metadata_ptr)
File "/home/usr/workspace/virtualenvs/xxx/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1152, in _run
feed_dict_tensor, options, run_metadata)
File "/home/usr/workspace/virtualenvs/xxx/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1328, in _do_run
run_metadata)
File "/home/usr/workspace/virtualenvs/xxx/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1348, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InternalError: RET_CHECK failure (tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc:3171) ShapeUtil::Equal(first_reduce->shape(), inst->shape())
[[{{node cluster_26_1/xla_compile}}]]
[[node ModelCrossGPUs/GPU_0/Evaluation/cond_2/Merge (defined at /home/usr/workspace/projects/xxx/utils/eval.py:84) ]]