(q_head, k_head, v_head, attn_mask, dropatt, is_training,
                  scale)
return attn_vec
Core absolute positional attention operations.
Core absolute positional attention operations.
Core absolute positional attention operations .
def abs_attn_core(q_head, k_head, v_head, attn_mask, dropatt, is_training,
                  scale):
  """Core absolute positional attention operations."""

  attn_score = tf.einsum('ibnd,jbnd->ijbn', q_head, k_head)
  attn_score *= scale
  if attn_mask is not None:
    attn_score = attn_score - 1e30 * attn_mask

  # attention probability
  attn_prob = tf.nn.softmax(attn_score, 1)
  attn_prob = tf.layers.dropout(attn_prob, dropatt, training=is_training)

  # attention output
  attn_vec = tf.einsum('ijbn,jbnd->ibnd', attn_prob, v_head)

  return attn_vec
def abs_attn_core ( q_head , k_head , v_head , attn_mask , dropatt , is_training , scale ) : attn_score = tf . einsum ( 'ibnd,jbnd->ijbn' , q_head , k_head ) attn_score *= scale if attn_mask is not None : attn_score = attn_score - 1e30 * attn_mask # attention probability attn_prob = tf . nn . softmax ( attn_score , 1 ) attn_prob = tf . layers . dropout ( attn_prob , dropatt , training = is_training ) # attention output attn_vec = tf . einsum ( 'ijbn,jbnd->ibnd' , attn_prob , v_head ) return attn_vec