(h, r, r_w_bias, r_r_bias, seg_mat, r_s_bias, seg_embed,
                       attn_mask, mems, d_model, n_head, d_head, dropout,
                       dropatt, is_training, kernel_initializer,
                       scope='rel_attn', reuse=None)
return output
Multi-head attention with relative positional encoding.
Multi-head attention with relative positional encoding.
Multi - head attention with relative positional encoding .
def rel_multihead_attn(h, r, r_w_bias, r_r_bias, seg_mat, r_s_bias, seg_embed,
                       attn_mask, mems, d_model, n_head, d_head, dropout,
                       dropatt, is_training, kernel_initializer,
                       scope='rel_attn', reuse=None):
  """Multi-head attention with relative positional encoding."""

  scale = 1 / (d_head ** 0.5)
  with tf.variable_scope(scope, reuse=reuse):
    if mems is not None and mems.shape.ndims > 1:
      cat = tf.concat([mems, h], 0)
    else:
      cat = h

    # content heads
    q_head_h = head_projection(
        h, d_model, n_head, d_head, kernel_initializer, 'q')
    k_head_h = head_projection(
        cat, d_model, n_head, d_head, kernel_initializer, 'k')
    v_head_h = head_projection(
        cat, d_model, n_head, d_head, kernel_initializer, 'v')

    # positional heads
    k_head_r = head_projection(
        r, d_model, n_head, d_head, kernel_initializer, 'r')

    # core attention ops
    attn_vec = rel_attn_core(
        q_head_h, k_head_h, v_head_h, k_head_r, seg_embed, seg_mat, r_w_bias,
        r_r_bias, r_s_bias, attn_mask, dropatt, is_training, scale)

    # post processing
    output = post_attention(h, attn_vec, d_model, n_head, d_head, dropout,
                            is_training, kernel_initializer)

  return output
def rel_multihead_attn ( h , r , r_w_bias , r_r_bias , seg_mat , r_s_bias , seg_embed , attn_mask , mems , d_model , n_head , d_head , dropout , dropatt , is_training , kernel_initializer , scope = 'rel_attn' , reuse = None ) : scale = 1 / ( d_head ** 0.5 ) with tf . variable_scope ( scope , reuse = reuse ) : if mems is not None and mems . shape . ndims > 1 : cat = tf . concat ( [ mems , h ] , 0 ) else : cat = h # content heads q_head_h = head_projection ( h , d_model , n_head , d_head , kernel_initializer , 'q' ) k_head_h = head_projection ( cat , d_model , n_head , d_head , kernel_initializer , 'k' ) v_head_h = head_projection ( cat , d_model , n_head , d_head , kernel_initializer , 'v' ) # positional heads k_head_r = head_projection ( r , d_model , n_head , d_head , kernel_initializer , 'r' ) # core attention ops attn_vec = rel_attn_core ( q_head_h , k_head_h , v_head_h , k_head_r , seg_embed , seg_mat , r_w_bias , r_r_bias , r_s_bias , attn_mask , dropatt , is_training , scale ) # post processing output = post_attention ( h , attn_vec , d_model , n_head , d_head , dropout , is_training , kernel_initializer ) return output