(inputs, inputs_lengths)
return attn_content
Args:
        inputs: shape(b_sz, tstp, rep_sz)
        input_lengths: shape(b_sz,)
Args:
        inputs: shape(b_sz, tstp, rep_sz)
        input_lengths: shape(b_sz,)
Args : inputs : shape ( b_sz tstp rep_sz ) input_lengths : shape ( b_sz )
def self_attn(inputs, inputs_lengths):
    '''
    Args:
        inputs: shape(b_sz, tstp, rep_sz)
        input_lengths: shape(b_sz,)
    '''

    attn_sz = np.int(inputs.get_shape()[-1])
    inputs_shape = tf.shape(inputs)
    tstp = inputs_shape[1]
    b_sz = inputs_shape[0]
    small_num = -np.Inf

    mask = mkMask(inputs_lengths, tstp)  # shape(b_sz, tstp)
    attn_mask = tf.expand_dims(mask, 1)  # shape(b_sz, 1, tstp)
    attn_mask = tf.tile(attn_mask, [1, tstp, 1])    # shape(b_sz, tstp, tstp)

    attn_matrix_1 = last_dim_linear(inputs, attn_sz, bias=False,
                                    scope='self_Attn_W1')  # shape(b_sz, tstp, attn_sz)
    attn_matrix_2 = last_dim_linear(inputs, attn_sz, bias=False,
                                    scope='self_Attn_W2')  # shape(b_sz, tstp, attn_sz)
    attn_matrix_1 = tf.expand_dims(attn_matrix_1, axis=2)    # shape(b_sz, tstp, 1, attn_sz)
    attn_matrix_2 = tf.expand_dims(attn_matrix_2, axis=1)    # shape(b_sz, 1, tstp, attn_sz)
    attn_matrix_1 = tf.tile(attn_matrix_1, [1, 1, tstp, 1])     # shape(b_sz, tstp, tstp, attn_sz)
    attn_matrix_2 = tf.tile(attn_matrix_2, [1, tstp, 1, 1])     # shape(b_sz, tstp, tstp, attn_sz)

    attn_matrix = tf.tanh(attn_matrix_1+attn_matrix_2)  #shape(b_sz, tstp, tstp, attn_sz)
    attn_logits = last_dim_linear(attn_matrix, 1, bias=False,
                                  scope='self_Attn_V')    #shape(b_sz, tstp, tstp, 1)
    attn_logits = tf.squeeze(attn_logits, [3])  #shape(b_sz, tstp, tstp)
    attn_logits = tf.where(attn_mask, attn_logits, tf.ones_like(attn_logits)*small_num)
    attn_prob = tf.nn.softmax(attn_logits, dim=-1)  # shape(b_sz, tstp, tstp)
    attn_prob = tf.expand_dims(attn_prob, 3)    # shape(b_sz, tstp, tstp, 1)
    attn_input = tf.expand_dims(inputs, 1)  # shape(b_sz, 1, tstp, rep_sz)
    attn_input = tf.tile(attn_input, [1, tstp, 1, 1])   # shape(b_sz, tstp, tstp, rep_sz)

    attn_content = tf.reduce_sum(attn_input * attn_prob,
                                 axis=[2])     # shape(b_sz, tstp, rep_sz)
    return attn_content
def self_attn ( inputs , inputs_lengths ) : attn_sz = np . int ( inputs . get_shape ( ) [ - 1 ] ) inputs_shape = tf . shape ( inputs ) tstp = inputs_shape [ 1 ] b_sz = inputs_shape [ 0 ] small_num = - np . Inf mask = mkMask ( inputs_lengths , tstp ) # shape(b_sz, tstp) attn_mask = tf . expand_dims ( mask , 1 ) # shape(b_sz, 1, tstp) attn_mask = tf . tile ( attn_mask , [ 1 , tstp , 1 ] ) # shape(b_sz, tstp, tstp) attn_matrix_1 = last_dim_linear ( inputs , attn_sz , bias = False , scope = 'self_Attn_W1' ) # shape(b_sz, tstp, attn_sz) attn_matrix_2 = last_dim_linear ( inputs , attn_sz , bias = False , scope = 'self_Attn_W2' ) # shape(b_sz, tstp, attn_sz) attn_matrix_1 = tf . expand_dims ( attn_matrix_1 , axis = 2 ) # shape(b_sz, tstp, 1, attn_sz) attn_matrix_2 = tf . expand_dims ( attn_matrix_2 , axis = 1 ) # shape(b_sz, 1, tstp, attn_sz) attn_matrix_1 = tf . tile ( attn_matrix_1 , [ 1 , 1 , tstp , 1 ] ) # shape(b_sz, tstp, tstp, attn_sz) attn_matrix_2 = tf . tile ( attn_matrix_2 , [ 1 , tstp , 1 , 1 ] ) # shape(b_sz, tstp, tstp, attn_sz) attn_matrix = tf . tanh ( attn_matrix_1 + attn_matrix_2 ) #shape(b_sz, tstp, tstp, attn_sz) attn_logits = last_dim_linear ( attn_matrix , 1 , bias = False , scope = 'self_Attn_V' ) #shape(b_sz, tstp, tstp, 1) attn_logits = tf . squeeze ( attn_logits , [ 3 ] ) #shape(b_sz, tstp, tstp) attn_logits = tf . where ( attn_mask , attn_logits , tf . ones_like ( attn_logits ) * small_num ) attn_prob = tf . nn . softmax ( attn_logits , dim = - 1 ) # shape(b_sz, tstp, tstp) attn_prob = tf . expand_dims ( attn_prob , 3 ) # shape(b_sz, tstp, tstp, 1) attn_input = tf . expand_dims ( inputs , 1 ) # shape(b_sz, 1, tstp, rep_sz) attn_input = tf . tile ( attn_input , [ 1 , tstp , 1 , 1 ] ) # shape(b_sz, tstp, tstp, rep_sz) attn_content = tf . reduce_sum ( attn_input * attn_prob , axis = [ 2 ] ) # shape(b_sz, tstp, rep_sz) return attn_content