(q, k, v, attn_mask, d_model, n_head, d_head, dropout,
                   dropatt, is_training, kernel_initializer, residual=True,
                   scope='abs_attn', reuse=None)
return output
Standard multi-head attention with absolute positional embedding.
Standard multi-head attention with absolute positional embedding.
Standard multi - head attention with absolute positional embedding .
def multihead_attn(q, k, v, attn_mask, d_model, n_head, d_head, dropout,
                   dropatt, is_training, kernel_initializer, residual=True,
                   scope='abs_attn', reuse=None):
  """Standard multi-head attention with absolute positional embedding."""

  scale = 1 / (d_head ** 0.5)
  with tf.variable_scope(scope, reuse=reuse):
    # attention heads
    q_head = head_projection(
        q, d_model, n_head, d_head, kernel_initializer, 'q')
    k_head = head_projection(
        k, d_model, n_head, d_head, kernel_initializer, 'k')
    v_head = head_projection(
        v, d_model, n_head, d_head, kernel_initializer, 'v')

    # attention vector
    attn_vec = abs_attn_core(q_head, k_head, v_head, attn_mask, dropatt,
                             is_training, scale)

    # post processing
    output = post_attention(v, attn_vec, d_model, n_head, d_head, dropout,
                            is_training, kernel_initializer, residual)

  return output
def multihead_attn ( q , k , v , attn_mask , d_model , n_head , d_head , dropout , dropatt , is_training , kernel_initializer , residual = True , scope = 'abs_attn' , reuse = None ) : scale = 1 / ( d_head ** 0.5 ) with tf . variable_scope ( scope , reuse = reuse ) : # attention heads q_head = head_projection ( q , d_model , n_head , d_head , kernel_initializer , 'q' ) k_head = head_projection ( k , d_model , n_head , d_head , kernel_initializer , 'k' ) v_head = head_projection ( v , d_model , n_head , d_head , kernel_initializer , 'v' ) # attention vector attn_vec = abs_attn_core ( q_head , k_head , v_head , attn_mask , dropatt , is_training , scale ) # post processing output = post_attention ( v , attn_vec , d_model , n_head , d_head , dropout , is_training , kernel_initializer , residual ) return output