python – Theano中的GRU实现

官方Theano教程(http://deeplearning.net/tutorial/code/lstm.py)中提供的LSTM代码使用return rval [0],因为outputs_info包含2个元素：

rval,dim_proj),tensor.alloc(numpy_floatX(0.),n_steps=nsteps)
return rval[0]

在GRU中,outputs_info只包含一个元素：

outputs_info=[tensor.alloc(numpy_floatX(0.),

尽管有括号,但它不会返回表示扫描输出的Theano变量列表的列表,而是直接返回Theano变量.

然后将rval馈送到池化层(在本例中为平均池化层)：

通过在GRU中仅使用rval [0],因为在GRU代码中rval是Theano变量而不是Theano变量的列表,所以删除了红色矩形中的部分：

这意味着您尝试使用第一个单词执行句子分类.

另一个可插入LSTM教程的GRU实现：

# weight initializer,normal by default
def norm_weight(nin,nout=None,scale=0.01,ortho=True):
    if nout is None:
        nout = nin
    if nout == nin and ortho:
        W = ortho_weight(nin)
    else:
        W = scale * numpy.random.randn(nin,nout)
    return W.astype('float32')

def param_init_lstm(options,prefix='lstm'):
    """
    GRU. Source: https://github.com/kyunghyuncho/dl4mt-material/blob/master/session0/lm.py
    """
    nin = options['dim_proj']
    dim = options['dim_proj']
    # embedding to gates transformation weights,biases
    W = numpy.concatenate([norm_weight(nin,dim),norm_weight(nin,dim)],axis=1)
    params[_p(prefix,'W')] = W
    params[_p(prefix,'b')] = numpy.zeros((2 * dim,)).astype('float32')

    # recurrent transformation weights for gates
    U = numpy.concatenate([ortho_weight(dim),ortho_weight(dim)],'U')] = U

    # embedding to hidden state proposal weights,biases
    Wx = norm_weight(nin,dim)
    params[_p(prefix,'Wx')] = Wx
    params[_p(prefix,'bx')] = numpy.zeros((dim,)).astype('float32')

    # recurrent transformation weights for hidden state proposal
    Ux = ortho_weight(dim)
    params[_p(prefix,'Ux')] = Ux
    return params


def lstm_layer(tparams,mask=None):

    nsteps = state_below.shape[0]

    if state_below.ndim == 3:
        n_samples = state_below.shape[1]
    else:
        n_samples = state_below.shape[0]

    dim = tparams[_p(prefix,'Ux')].shape[1]

    if mask is None:
        mask = tensor.alloc(1.,state_below.shape[0],1)

    # utility function to slice a tensor
    def _slice(_x,n*dim:(n+1)*dim]
        return _x[:,n*dim:(n+1)*dim]

    # state_below is the input word embeddings
    # input to the gates,concatenated
    state_below_ = tensor.dot(state_below,'W')]) + 
        tparams[_p(prefix,'b')]
    # input to compute the hidden state proposal
    state_belowx = tensor.dot(state_below,'Wx')]) + 
        tparams[_p(prefix,'bx')]

    # step function to be used by scan
    # arguments    | sequences |outputs-info| non-seqs
    def _step_slice(m_,xx_,h_,U,Ux):
        preact = tensor.dot(h_,U)
        preact += x_

        # reset and update gates
        r = tensor.nnet.sigmoid(_slice(preact,dim))
        u = tensor.nnet.sigmoid(_slice(preact,dim))

        # compute the hidden state proposal
        preactx = tensor.dot(h_,Ux)
        preactx = preactx * r
        preactx = preactx + xx_

        # hidden state proposal
        h = tensor.tanh(preactx)

        # leaky integrate and obtain next hidden state
        h = u * h_ + (1. - u) * h
        h = m_[:,None] * h_

        return h

    # prepare scan arguments
    seqs = [mask,state_below_,state_belowx]
    _step = _step_slice
    shared_vars = [tparams[_p(prefix,'Ux')]]

    init_state = tensor.unbroadcast(tensor.alloc(0.,0)

    rval,sequences=seqs,outputs_info=[init_state],non_sequences=shared_vars,n_steps=nsteps,strict=True)
    return rval

作为旁注,Keras将此问题修复为follows：

results,_ = theano.scan(
    _step,sequences=inputs,outputs_info=[None] + initial_states,go_backwards=go_backwards)

# deal with Theano API inconsistency
if type(results) is list:
    outputs = results[0]
    states = results[1:]
else:
    outputs = results
    states = []