(inputs, attention_size, time_major=False, return_alphas=False)
return output, alphas
Attention layer.
Attention layer.
Attention layer .
def attention(inputs, attention_size, time_major=False, return_alphas=False):
  """Attention layer."""
  if isinstance(inputs, tuple):
    # In case of Bi-RNN, concatenate the forward and the backward RNN outputs.
    inputs = tf.concat(inputs, 2)

  if time_major:
    # (T,B,D) => (B,T,D)
    inputs = tf.transpose(inputs, [1, 0, 2])

  time_size = inputs.shape[1].value  # T value - time size of the RNN layer
  hidden_size = inputs.shape[2].value  # D value - hidden size of the RNN layer

  # Trainable parameters
  W_omega = tf.get_variable(
      name='W_omega',
      initializer=tf.random_normal([hidden_size, attention_size], stddev=0.1))
  b_omega = tf.get_variable(
      name='b_omega',
      initializer=tf.random_normal([attention_size], stddev=0.1))
  u_omega = tf.get_variable(
      name='u_omega',
      initializer=tf.random_normal([attention_size, 1], stddev=0.1))

  # Applying fully connected layer with non-linear activation to each of the B*T timestamps;
  #  the shape of `v` is (B,T,D)*(D,A)=(B,T,A), where A=attention_size
  #v = tf.tanh(tf.tensordot(inputs, W_omega, axes=1) + b_omega)
  #v = tf.sigmoid(tf.tensordot(inputs, W_omega, axes=1) + b_omega)
  # (B, T, D) dot (D, Atten)

  logging.info('attention inputs: {}'.format(inputs.shape))
  inputs_reshaped = tf.reshape(inputs, [-1, hidden_size])
  dot = tf.matmul(inputs_reshaped, W_omega)
  dot = tf.reshape(dot, [-1, time_size, attention_size])
  v = tf.sigmoid(dot + b_omega)
  logging.info(f'attention vector: {v.shape}')
  # For each of the timestamps its vector of size A from `v` is reduced with `u` vector
  # (B, T, Atten) dot (Atten)
  #vu = tf.tensordot(v, u_omega, axes=1)   # (B,T) shape
  v = tf.reshape(v, [-1, attention_size])
  vu = tf.matmul(v, u_omega)  # (B,T) shape
  vu = tf.squeeze(vu, axis=-1)
  vu = tf.reshape(vu, [-1, time_size])
  logging.info(f'attention energe: {vu.shape}')
  alphas = tf.nn.softmax(vu)  # (B,T) shape also

  # Output of (Bi-)RNN is reduced with attention vector; the result has (B,D) shape
  # [batch, time] -> [batch, time, 1]
  alphas = tf.expand_dims(alphas, -1)
  # [batch, time, dim] -> [batch, dim]
  output = tf.reduce_sum(inputs * alphas, 1)

  if not return_alphas:
    return output

  return output, alphas
def attention ( inputs , attention_size , time_major = False , return_alphas = False ) : if isinstance ( inputs , tuple ) : # In case of Bi-RNN, concatenate the forward and the backward RNN outputs. inputs = tf . concat ( inputs , 2 ) if time_major : # (T,B,D) => (B,T,D) inputs = tf . transpose ( inputs , [ 1 , 0 , 2 ] ) time_size = inputs . shape [ 1 ] . value # T value - time size of the RNN layer hidden_size = inputs . shape [ 2 ] . value # D value - hidden size of the RNN layer # Trainable parameters W_omega = tf . get_variable ( name = 'W_omega' , initializer = tf . random_normal ( [ hidden_size , attention_size ] , stddev = 0.1 ) ) b_omega = tf . get_variable ( name = 'b_omega' , initializer = tf . random_normal ( [ attention_size ] , stddev = 0.1 ) ) u_omega = tf . get_variable ( name = 'u_omega' , initializer = tf . random_normal ( [ attention_size , 1 ] , stddev = 0.1 ) ) # Applying fully connected layer with non-linear activation to each of the B*T timestamps; #  the shape of `v` is (B,T,D)*(D,A)=(B,T,A), where A=attention_size #v = tf.tanh(tf.tensordot(inputs, W_omega, axes=1) + b_omega) #v = tf.sigmoid(tf.tensordot(inputs, W_omega, axes=1) + b_omega) # (B, T, D) dot (D, Atten) logging . info ( 'attention inputs: {}' . format ( inputs . shape ) ) inputs_reshaped = tf . reshape ( inputs , [ - 1 , hidden_size ] ) dot = tf . matmul ( inputs_reshaped , W_omega ) dot = tf . reshape ( dot , [ - 1 , time_size , attention_size ] ) v = tf . sigmoid ( dot + b_omega ) logging . info ( f'attention vector: {v.shape}' ) # For each of the timestamps its vector of size A from `v` is reduced with `u` vector # (B, T, Atten) dot (Atten) #vu = tf.tensordot(v, u_omega, axes=1)   # (B,T) shape v = tf . reshape ( v , [ - 1 , attention_size ] ) vu = tf . matmul ( v , u_omega ) # (B,T) shape vu = tf . squeeze ( vu , axis = - 1 ) vu = tf . reshape ( vu , [ - 1 , time_size ] ) logging . info ( f'attention energe: {vu.shape}' ) alphas = tf . nn . softmax ( vu ) # (B,T) shape also # Output of (Bi-)RNN is reduced with attention vector; the result has (B,D) shape # [batch, time] -> [batch, time, 1] alphas = tf . expand_dims ( alphas , - 1 ) # [batch, time, dim] -> [batch, dim] output = tf . reduce_sum ( inputs * alphas , 1 ) if not return_alphas : return output return output , alphas