1

我正在实现一个指针网络的变体,我想在编码/解码步骤中将 dropout 应用于隐藏状态。

这是我的辍学功能:

def dropout_layer(proj, use_noise):
    trng = RandomStreams()
    proj = tensor.switch(use_noise,(proj *trng.binomial(proj.shape,
                         p=0.5, n=1,dtype=proj.dtype)), proj * 0.5)
    return proj

我选择在RandomStreams每个函数调用时初始化一个实例,因为这似乎是 keras 所做的......

这是函数扫描调用:

def _lstm_e(m_, x_, h_, c_):
    preact = tensor.dot(x_, tparams[_p('lstm_en', 'W')]) + tparams[_p('lstm_en', 'b')]
    preact += tensor.dot(h_, tparams[_p('lstm_en', 'U')])

    i = tensor.nnet.sigmoid(_slice(preact, 0, options['dim_proj']))
    f = tensor.nnet.sigmoid(_slice(preact, 1, options['dim_proj']))
    o = tensor.nnet.sigmoid(_slice(preact, 2, options['dim_proj']))
    c = tensor.tanh(_slice(preact, 3, options['dim_proj']))

    c = f * c_ + i * c
    c = m_[:, None] * c + (1. - m_)[:, None] * c_
    h = o * tensor.tanh(c)
    h = m_[:, None] * h + (1. - m_)[:, None] * h_
    h = dropout_layer(h, use_noise)#, trng)

    u = tensor.dot(h, tparams['fc_arg_W']) + tparams['fc_arg_b']


    prob = tensor.nnet.nnet.softmax(u)
    #prob = softmax(at_mask, u)
    return h, c, prob

这是扫描的调用:

ones = tensor.ones((n_samples,), dtype=p.dtype)
h0 = tensor.outer(ones, tparams['lstm_hterm'])  # n_samples * dim_proj; T.tile doesn't work on non-constant reps
c0 = tensor.alloc(numpy_floatX(0.), n_samples, options['dim_proj'])
at_mask = tensor.set_subtensor(p_mask[0, :], tensor.constant(1, dtype=config.floatX))
rval, updates = theano.scan(_lstm_e,
                      sequences=[p_mask, p],
                      outputs_info=[h0, c0, None],
                      #non_sequences=trng,
                      name='encoding',
                      n_steps=n_sizes)
hiddens, cells, argt_probs = rval  # hiddens: n_sizes * n_samples * dim_proj
# hiddens = tensor.concatenate([tensor.shape_padleft(h0), hiddens], axis=0)
f_encode = theano.function([p_mask, p], hiddens, updates=updates)

f_argt_probs = theano.function([p_mask, p], argt_probs, updates=updates)

请注意我是如何提供对 theano 函数的更新的,正如人们所说的那样,可以解决类似的问题。

最后,这是错误:

Traceback (most recent call last):
  File "ptrnets_j1_d.py", line 973, in <module>
    reload_model=args.reload,
  File "ptrnets_j1_d.py", line 760, in train_lstm
    (use_noise, p, p_mask, x, x_mask, y, y_mask, at, at_mask, preds, cost, f_encode, f_decode, f_probi, f_argt_preds, anneal) = build_model(tparams, model_options)
  File "ptrnets_j1_d.py", line 690, in build_model
    preds, f_encode, f_decode, f_probi, argt_preds, f_argt_preds = ptr_network(tparams, p, p_mask, x, x_mask, xi, xi_mask, hidi, celi, hids, use_noise, options)
  File "ptrnets_j1_d.py", line 368, in ptr_network
    f_decode = theano.function([p_mask, p, x_mask, x], preds, updates=updates)
  File "/home/ppotash/virtual-env/local/lib/python2.7/site-packages/theano/compile/function.py", line 320, in function
    output_keys=output_keys)
  File "/home/ppotash/virtual-env/local/lib/python2.7/site-packages/theano/compile/pfunc.py", line 479, in pfunc
    output_keys=output_keys)
  File "/home/ppotash/virtual-env/local/lib/python2.7/site-packages/theano/compile/function_module.py", line 1776, in orig_function
    output_keys=output_keys).create(
  File "/home/ppotash/virtual-env/local/lib/python2.7/site-packages/theano/compile/function_module.py", line 1428, in __init__
    accept_inplace)
  File "/home/ppotash/virtual-env/local/lib/python2.7/site-packages/theano/compile/function_module.py", line 177, in std_fgraph
    update_mapping=update_mapping)
  File "/home/ppotash/virtual-env/local/lib/python2.7/site-packages/theano/gof/fg.py", line 171, in __init__
    self.__import_r__(output, reason="init")
  File "/home/ppotash/virtual-env/local/lib/python2.7/site-packages/theano/gof/fg.py", line 360, in __import_r__
    self.__import__(variable.owner, reason=reason)
  File "/home/ppotash/virtual-env/local/lib/python2.7/site-packages/theano/gof/fg.py", line 465, in __import__
    detailed_err_msg)
theano.gof.fg.MissingInputError: A variable that is an input to the graph was neither provided as an input to the function nor given a value. A chain of variables leading from this input to an output is [p_mask[t], DimShuffle{0,x}.0, Subtensor{::, ::}.0, Elemwise{mul,no_inplace}.0, Elemwise{add,no_inplace}.0, Shape.0, GPU_mrg_uniform{CudaNdarrayType(float32, matrix),no_inplace}.0]. This chain may not be unique
Backtrace when the variable is created:
  File "ptrnets_j1_d.py", line 973, in <module>
    reload_model=args.reload,
  File "ptrnets_j1_d.py", line 760, in train_lstm
    (use_noise, p, p_mask, x, x_mask, y, y_mask, at, at_mask, preds, cost, f_encode, f_decode, f_probi, f_argt_preds, anneal) = build_model(tparams, model_options)
  File "ptrnets_j1_d.py", line 690, in build_model
    preds, f_encode, f_decode, f_probi, argt_preds, f_argt_preds = ptr_network(tparams, p, p_mask, x, x_mask, xi, xi_mask, hidi, celi, hids, use_noise, options)
  File "ptrnets_j1_d.py", line 350, in ptr_network
    n_steps=n_sizes)

一种想法是,当你给它一个符号形状时,用于扫描的函数中使用的二项式生成器会导致错误。这篇文章中已经提到了这一点

我需要做一些小的改动,还是必须完全修改我使用 dropout 的方式?

编辑 更完整的代码版本:

在此函数中调用扫描:

def ptr_network(tparams, p, p_mask, x, x_mask, xi, xi_mask, hidi, celi, hids, use_noise, options):
    n_sizes = p.shape[0]
    n_samples = p.shape[1] if p.ndim == 3 else 1
    n_steps = x.shape[0]
    beam_width = xi.shape[0]

    assert p_mask is not None
    assert x_mask is not None
    assert xi_mask is not None


    def _slice(_x, n, dim):
        if _x.ndim == 3:
            return _x[:, :, n * dim:(n + 1) * dim]
        if _x.ndim == 2:
            return _x[:, n * dim:(n + 1) * dim]
        return _x[n * dim:(n + 1) * dim]

    def softmax(m_, x_):
        maxes = tensor.max(x_, axis=0, keepdims=True)
        e = tensor.exp(x_ - maxes)
        dist = e / tensor.sum(e * m_, axis=0)
        return dist

    def _lstm_e(m_, x_, h_, c_):
        preact = tensor.dot(x_, tparams[_p('lstm_en', 'W')]) + tparams[_p('lstm_en', 'b')]
        preact += tensor.dot(h_, tparams[_p('lstm_en', 'U')])

        i = tensor.nnet.sigmoid(_slice(preact, 0, options['dim_proj']))
        f = tensor.nnet.sigmoid(_slice(preact, 1, options['dim_proj']))
        o = tensor.nnet.sigmoid(_slice(preact, 2, options['dim_proj']))
        c = tensor.tanh(_slice(preact, 3, options['dim_proj']))

        c = f * c_ + i * c
        c = m_[:, None] * c + (1. - m_)[:, None] * c_
        h = o * tensor.tanh(c)
        h = m_[:, None] * h + (1. - m_)[:, None] * h_
        h = dropout_layer(h, use_noise)#, trng)

        u = tensor.dot(h, tparams['fc_arg_W']) + tparams['fc_arg_b']


        prob = tensor.nnet.nnet.softmax(u)
        #prob = softmax(at_mask, u)
        return h, c, prob

    def _lstm_d(m_, x_, h_, c_):
        preact = tensor.dot(x_, tparams[_p('lstm_de', 'W')]) + tparams[_p('lstm_de', 'b')]
        preact += tensor.dot(h_, tparams[_p('lstm_de', 'U')])

        i = tensor.nnet.sigmoid(_slice(preact, 0, options['dim_proj']))
        f = tensor.nnet.sigmoid(_slice(preact, 1, options['dim_proj']))
        o = tensor.nnet.sigmoid(_slice(preact, 2, options['dim_proj']))
        c = tensor.tanh(_slice(preact, 3, options['dim_proj']))

        c = f * c_ + i * c
        c = m_[:, None] * c + (1. - m_)[:, None] * c_
        h = o * tensor.tanh(c)
        h = m_[:, None] * h + (1. - m_)[:, None] * h_
        h = dropout_layer(h, use_noise)#, trng)

        return h, c

    def _ptr_probs(xm_, x_, h_, c_, _, hprevs, hprevs_m):
        xemb = p[x_, tensor.arange(n_samples), :]  # n_samples * dim_proj
        h, c = _lstm_d(xm_, xemb, h_, c_)
        u = tensor.dot(hprevs, tparams['ptr_W1']) + tensor.dot(h, tparams['ptr_W2'])  # n_steps * n_samples * dim
        u = tensor.tanh(u)  # n_sizes * n_samples * dim_proj
        u = tensor.dot(u, tparams['ptr_v'])  # n_sizes * n_samples
        # prob = tensor.nnet.softmax(u.T).T  # n_sizes * n_samples
        prob = softmax(hprevs_m, u)
        return h, c, prob

    ones = tensor.ones((n_samples,), dtype=p.dtype)
    h0 = tensor.outer(ones, tparams['lstm_hterm'])  # n_samples * dim_proj; T.tile doesn't work on non-constant reps
    c0 = tensor.alloc(numpy_floatX(0.), n_samples, options['dim_proj'])
    at_mask = tensor.set_subtensor(p_mask[0, :], tensor.constant(1, dtype=config.floatX))
    rval, updates = theano.scan(_lstm_e,
                          sequences=[p_mask, p],
                          outputs_info=[h0, c0, None],
                          #non_sequences=trng,
                          name='encoding',
                          n_steps=n_sizes)
    hiddens, cells, argt_probs = rval  # hiddens: n_sizes * n_samples * dim_proj
    # hiddens = tensor.concatenate([tensor.shape_padleft(h0), hiddens], axis=0)
    f_encode = theano.function([p_mask, p], hiddens, updates=updates)

    f_argt_probs = theano.function([p_mask, p], argt_probs, updates=updates)
    # decoding
    hiddens_mask = tensor.set_subtensor(p_mask[0, :], tensor.constant(1, dtype=config.floatX))
    # hiddens_mask = tensor.concatenate([tensor.ones((1, n_samples), dtype=config.floatX), p_mask], axis=0)
    rval, updates2 = theano.scan(_ptr_probs,
                          sequences=[x_mask, x],
                          outputs_info=[hiddens[-1],  # n_samples * dim_proj
                                        tensor.alloc(numpy_floatX(0.), n_samples, options['dim_proj']),  # cells[-1],
                                        tensor.alloc(numpy_floatX(0.), n_sizes, n_samples)],
                                non_sequences=[hiddens, hiddens_mask],
                          name='decoding',
                          n_steps=n_steps)
    preds = rval[2]
    f_decode = theano.function([p_mask, p, x_mask, x], preds, updates=updates2)

    u0 = tensor.alloc(numpy_floatX(0.), hidi.shape[0], beam_width)
    hiddeni, celli, probi = _ptr_probs(xi_mask, xi, hidi, celi, u0, hids, hiddens_mask)
    f_probi = theano.function(inputs=[xi_mask, xi, hidi, celi, hids, p_mask, p], outputs=[hiddeni, celli, probi])

    return preds, f_encode, f_decode, f_probi, argt_probs, f_argt_probs

并调用该函数ptr_network

def build_model(tparams, options):
    #trng = RandomStreams()
    # for training
    p = tensor.tensor3('p', dtype=config.floatX)  # Problems, n_sizes * n_samples * data_dim
    p_mask = tensor.matrix('p_mask', dtype=config.floatX)
    x = tensor.matrix('x', dtype='int64')  # n_steps * n_samples
    x_mask = tensor.matrix('x_mask', dtype=config.floatX)
    y = tensor.matrix('y', dtype='int64')  # n_steps * n_samples
    y_mask = tensor.matrix('y_mask', dtype=config.floatX)
    at = tensor.matrix('at', dtype='int64')  # n_steps * n_samples
    at_mask = tensor.matrix('at_mask', dtype=config.floatX)

    use_noise = theano.shared(numpy_floatX(0.))

    anneal = tensor.scalar('anneal', dtype=config.floatX)

    # for generation
    hidi = tensor.matrix('hidi', dtype=config.floatX)
    celi = tensor.matrix('celi', dtype=config.floatX)
    hids = tensor.tensor3('hids', dtype=config.floatX)
    xi = tensor.vector('xi', dtype='int64')
    xi_mask = tensor.vector('xi_mask', dtype=config.floatX)

    n_steps = x.shape[0]
    n_samples = x.shape[1]

    preds, f_encode, f_decode, f_probi, argt_preds, f_argt_preds = ptr_network(tparams, p, p_mask, x, x_mask, xi, xi_mask, hidi, celi, hids, use_noise, options)
4

0 回答 0