(tparams, options, sampling=True)
return trng, use_noise, [x, mask, ctx], alphas, alpha_sample, cost, opt_outs
Builds the entire computational graph used for training

    [This function builds a model described in Section 3.1.2 onwards
    as the convolutional feature are precomputed, some extra features
    which were not used are also implemented here.]

    Parameters
    ----------
    tparams : OrderedDict
        maps names of variables to theano shared variables
    options : dict
        big dictionary with all the settings and hyperparameters
    sampling : boolean
        [If it is true, when using stochastic attention, follows
        the learning rule described in section 4. at the bottom left of
        page 5]
    Returns
    -------
    trng: theano random number generator
        Used for dropout, stochastic attention, etc
    use_noise: theano shared variable
        flag that toggles noise on and off
    [x, mask, ctx]: theano variables
        Represent the captions, binary mask, and annotations
        for a single batch (see dimensions below)
    alphas: theano variables
        Attention weights
    alpha_sample: theano variable
        Sampled attention weights used in REINFORCE for stochastic
        attention: [see the learning rule in eq (12)]
    cost: theano variable
        negative log likelihood
    opt_outs: OrderedDict
        extra outputs required depending on configuration in options
Builds the entire computational graph used for training
Builds the entire computational graph used for training
def build_model(tparams, options, sampling=True):
    """ Builds the entire computational graph used for training

    [This function builds a model described in Section 3.1.2 onwards
    as the convolutional feature are precomputed, some extra features
    which were not used are also implemented here.]

    Parameters
    ----------
    tparams : OrderedDict
        maps names of variables to theano shared variables
    options : dict
        big dictionary with all the settings and hyperparameters
    sampling : boolean
        [If it is true, when using stochastic attention, follows
        the learning rule described in section 4. at the bottom left of
        page 5]
    Returns
    -------
    trng: theano random number generator
        Used for dropout, stochastic attention, etc
    use_noise: theano shared variable
        flag that toggles noise on and off
    [x, mask, ctx]: theano variables
        Represent the captions, binary mask, and annotations
        for a single batch (see dimensions below)
    alphas: theano variables
        Attention weights
    alpha_sample: theano variable
        Sampled attention weights used in REINFORCE for stochastic
        attention: [see the learning rule in eq (12)]
    cost: theano variable
        negative log likelihood
    opt_outs: OrderedDict
        extra outputs required depending on configuration in options
    """
    trng = RandomStreams(1234)
    use_noise = theano.shared(numpy.float32(0.))

    # description string: #words x #samples,
    x = tensor.matrix('x', dtype='int64')
    mask = tensor.matrix('mask', dtype='float32')
    # context: #samples x #annotations x dim
    ctx = tensor.tensor3('ctx', dtype='float32')

    n_timesteps = x.shape[0]
    n_samples = x.shape[1]

    # index into the word embedding matrix, shift it forward in time
    emb = tparams['Wemb'][x.flatten()].reshape([n_timesteps, n_samples, options['dim_word']])
    emb_shifted = tensor.zeros_like(emb)
    emb_shifted = tensor.set_subtensor(emb_shifted[1:], emb[:-1])
    emb = emb_shifted
    if options['lstm_encoder']:
        # encoder
        ctx_fwd = get_layer('lstm')[1](tparams, ctx.dimshuffle(1,0,2),
                                       options, prefix='encoder')[0].dimshuffle(1,0,2)
        ctx_rev = get_layer('lstm')[1](tparams, ctx.dimshuffle(1,0,2)[:,::-1,:],
                                       options, prefix='encoder_rev')[0][:,::-1,:].dimshuffle(1,0,2)
        ctx0 = tensor.concatenate((ctx_fwd, ctx_rev), axis=2)
    else:
        ctx0 = ctx

    # initial state/cell [top right on page 4]
    ctx_mean = ctx0.mean(1)
    for lidx in xrange(1, options['n_layers_init']):
        ctx_mean = get_layer('ff')[1](tparams, ctx_mean, options,
                                      prefix='ff_init_%d'%lidx, activ='rectifier')
        if options['use_dropout']:
            ctx_mean = dropout_layer(ctx_mean, use_noise, trng)

    init_state = get_layer('ff')[1](tparams, ctx_mean, options, prefix='ff_state', activ='tanh')
    init_memory = get_layer('ff')[1](tparams, ctx_mean, options, prefix='ff_memory', activ='tanh')
    # lstm decoder
    # [equation (1), (2), (3) in section 3.1.2]
    attn_updates = []
    proj, updates = get_layer('lstm_cond')[1](tparams, emb, options,
                                              prefix='decoder',
                                              mask=mask, context=ctx0,
                                              one_step=False,
                                              init_state=init_state,
                                              init_memory=init_memory,
                                              trng=trng,
                                              use_noise=use_noise,
                                              sampling=sampling)
    attn_updates += updates
    proj_h = proj[0]
    # optional deep attention
    if options['n_layers_lstm'] > 1:
        for lidx in xrange(1, options['n_layers_lstm']):
            init_state = get_layer('ff')[1](tparams, ctx_mean, options, prefix='ff_state_%d'%lidx, activ='tanh')
            init_memory = get_layer('ff')[1](tparams, ctx_mean, options, prefix='ff_memory_%d'%lidx, activ='tanh')
            proj, updates = get_layer('lstm_cond')[1](tparams, proj_h, options,
                                                      prefix='decoder_%d'%lidx,
                                                      mask=mask, context=ctx0,
                                                      one_step=False,
                                                      init_state=init_state,
                                                      init_memory=init_memory,
                                                      trng=trng,
                                                      use_noise=use_noise,
                                                      sampling=sampling)
            attn_updates += updates
            proj_h = proj[0]

    alphas = proj[2]
    alpha_sample = proj[3]
    ctxs = proj[4]

    # [beta value explained in note 4.2.1 "doubly stochastic attention"]
    if options['selector']:
        sels = proj[5]

    if options['use_dropout']:
        proj_h = dropout_layer(proj_h, use_noise, trng)

    # compute word probabilities
    # [equation (7)]
    logit = get_layer('ff')[1](tparams, proj_h, options, prefix='ff_logit_lstm', activ='linear')
    if options['prev2out']:
        logit += emb
    if options['ctx2out']:
        logit += get_layer('ff')[1](tparams, ctxs, options, prefix='ff_logit_ctx', activ='linear')
    logit = tanh(logit)
    if options['use_dropout']:
        logit = dropout_layer(logit, use_noise, trng)
    if options['n_layers_out'] > 1:
        for lidx in xrange(1, options['n_layers_out']):
            logit = get_layer('ff')[1](tparams, logit, options, prefix='ff_logit_h%d'%lidx, activ='rectifier')
            if options['use_dropout']:
                logit = dropout_layer(logit, use_noise, trng)

    # compute softmax
    logit = get_layer('ff')[1](tparams, logit, options, prefix='ff_logit', activ='linear')
    logit_shp = logit.shape
    probs = tensor.nnet.softmax(logit.reshape([logit_shp[0]*logit_shp[1], logit_shp[2]]))

    # Index into the computed probability to give the log likelihood
    x_flat = x.flatten()
    p_flat = probs.flatten()
    cost = -tensor.log(p_flat[tensor.arange(x_flat.shape[0])*probs.shape[1]+x_flat]+1e-8)
    cost = cost.reshape([x.shape[0], x.shape[1]])
    masked_cost = cost * mask
    cost = (masked_cost).sum(0)

    # optional outputs
    opt_outs = dict() 
    if options['selector']:
        opt_outs['selector'] = sels
    if options['attn_type'] == 'stochastic':
        opt_outs['masked_cost'] = masked_cost # need this for reinforce later
        opt_outs['attn_updates'] = attn_updates # this is to update the rng

    return trng, use_noise, [x, mask, ctx], alphas, alpha_sample, cost, opt_outs
def build_model ( tparams , options , sampling = True ) : trng = RandomStreams ( 1234 ) use_noise = theano . shared ( numpy . float32 ( 0. ) ) # description string: #words x #samples, x = tensor . matrix ( 'x' , dtype = 'int64' ) mask = tensor . matrix ( 'mask' , dtype = 'float32' ) # context: #samples x #annotations x dim ctx = tensor . tensor3 ( 'ctx' , dtype = 'float32' ) n_timesteps = x . shape [ 0 ] n_samples = x . shape [ 1 ] # index into the word embedding matrix, shift it forward in time emb = tparams [ 'Wemb' ] [ x . flatten ( ) ] . reshape ( [ n_timesteps , n_samples , options [ 'dim_word' ] ] ) emb_shifted = tensor . zeros_like ( emb ) emb_shifted = tensor . set_subtensor ( emb_shifted [ 1 : ] , emb [ : - 1 ] ) emb = emb_shifted if options [ 'lstm_encoder' ] : # encoder ctx_fwd = get_layer ( 'lstm' ) [ 1 ] ( tparams , ctx . dimshuffle ( 1 , 0 , 2 ) , options , prefix = 'encoder' ) [ 0 ] . dimshuffle ( 1 , 0 , 2 ) ctx_rev = get_layer ( 'lstm' ) [ 1 ] ( tparams , ctx . dimshuffle ( 1 , 0 , 2 ) [ : , : : - 1 , : ] , options , prefix = 'encoder_rev' ) [ 0 ] [ : , : : - 1 , : ] . dimshuffle ( 1 , 0 , 2 ) ctx0 = tensor . concatenate ( ( ctx_fwd , ctx_rev ) , axis = 2 ) else : ctx0 = ctx # initial state/cell [top right on page 4] ctx_mean = ctx0 . mean ( 1 ) for lidx in xrange ( 1 , options [ 'n_layers_init' ] ) : ctx_mean = get_layer ( 'ff' ) [ 1 ] ( tparams , ctx_mean , options , prefix = 'ff_init_%d' % lidx , activ = 'rectifier' ) if options [ 'use_dropout' ] : ctx_mean = dropout_layer ( ctx_mean , use_noise , trng ) init_state = get_layer ( 'ff' ) [ 1 ] ( tparams , ctx_mean , options , prefix = 'ff_state' , activ = 'tanh' ) init_memory = get_layer ( 'ff' ) [ 1 ] ( tparams , ctx_mean , options , prefix = 'ff_memory' , activ = 'tanh' ) # lstm decoder # [equation (1), (2), (3) in section 3.1.2] attn_updates = [ ] proj , updates = get_layer ( 'lstm_cond' ) [ 1 ] ( tparams , emb , options , prefix = 'decoder' , mask = mask , context = ctx0 , one_step = False , init_state = init_state , init_memory = init_memory , trng = trng , use_noise = use_noise , sampling = sampling ) attn_updates += updates proj_h = proj [ 0 ] # optional deep attention if options [ 'n_layers_lstm' ] > 1 : for lidx in xrange ( 1 , options [ 'n_layers_lstm' ] ) : init_state = get_layer ( 'ff' ) [ 1 ] ( tparams , ctx_mean , options , prefix = 'ff_state_%d' % lidx , activ = 'tanh' ) init_memory = get_layer ( 'ff' ) [ 1 ] ( tparams , ctx_mean , options , prefix = 'ff_memory_%d' % lidx , activ = 'tanh' ) proj , updates = get_layer ( 'lstm_cond' ) [ 1 ] ( tparams , proj_h , options , prefix = 'decoder_%d' % lidx , mask = mask , context = ctx0 , one_step = False , init_state = init_state , init_memory = init_memory , trng = trng , use_noise = use_noise , sampling = sampling ) attn_updates += updates proj_h = proj [ 0 ] alphas = proj [ 2 ] alpha_sample = proj [ 3 ] ctxs = proj [ 4 ] # [beta value explained in note 4.2.1 "doubly stochastic attention"] if options [ 'selector' ] : sels = proj [ 5 ] if options [ 'use_dropout' ] : proj_h = dropout_layer ( proj_h , use_noise , trng ) # compute word probabilities # [equation (7)] logit = get_layer ( 'ff' ) [ 1 ] ( tparams , proj_h , options , prefix = 'ff_logit_lstm' , activ = 'linear' ) if options [ 'prev2out' ] : logit += emb if options [ 'ctx2out' ] : logit += get_layer ( 'ff' ) [ 1 ] ( tparams , ctxs , options , prefix = 'ff_logit_ctx' , activ = 'linear' ) logit = tanh ( logit ) if options [ 'use_dropout' ] : logit = dropout_layer ( logit , use_noise , trng ) if options [ 'n_layers_out' ] > 1 : for lidx in xrange ( 1 , options [ 'n_layers_out' ] ) : logit = get_layer ( 'ff' ) [ 1 ] ( tparams , logit , options , prefix = 'ff_logit_h%d' % lidx , activ = 'rectifier' ) if options [ 'use_dropout' ] : logit = dropout_layer ( logit , use_noise , trng ) # compute softmax logit = get_layer ( 'ff' ) [ 1 ] ( tparams , logit , options , prefix = 'ff_logit' , activ = 'linear' ) logit_shp = logit . shape probs = tensor . nnet . softmax ( logit . reshape ( [ logit_shp [ 0 ] * logit_shp [ 1 ] , logit_shp [ 2 ] ] ) ) # Index into the computed probability to give the log likelihood x_flat = x . flatten ( ) p_flat = probs . flatten ( ) cost = - tensor . log ( p_flat [ tensor . arange ( x_flat . shape [ 0 ] ) * probs . shape [ 1 ] + x_flat ] + 1e-8 ) cost = cost . reshape ( [ x . shape [ 0 ] , x . shape [ 1 ] ] ) masked_cost = cost * mask cost = ( masked_cost ) . sum ( 0 ) # optional outputs opt_outs = dict ( ) if options [ 'selector' ] : opt_outs [ 'selector' ] = sels if options [ 'attn_type' ] == 'stochastic' : opt_outs [ 'masked_cost' ] = masked_cost # need this for reinforce later opt_outs [ 'attn_updates' ] = attn_updates # this is to update the rng return trng , use_noise , [ x , mask , ctx ] , alphas , alpha_sample , cost , opt_outs