(h, attn_vec, d_model, n_head, d_head, dropout, is_training,
                   kernel_initializer, residual=True)
return output
Post-attention processing.
Post-attention processing.
Post - attention processing .
def post_attention(h, attn_vec, d_model, n_head, d_head, dropout, is_training,
                   kernel_initializer, residual=True):
  """Post-attention processing."""
  # post-attention projection (back to `d_model`)
  proj_o = tf.get_variable('o/kernel', [d_model, n_head, d_head],
                           dtype=h.dtype, initializer=kernel_initializer)
  attn_out = tf.einsum('ibnd,hnd->ibh', attn_vec, proj_o)

  attn_out = tf.layers.dropout(attn_out, dropout, training=is_training)
  if residual:
    output = tf.contrib.layers.layer_norm(attn_out + h, begin_norm_axis=-1,
                                          scope='LayerNorm')
  else:
    output = tf.contrib.layers.layer_norm(attn_out, begin_norm_axis=-1,
                                          scope='LayerNorm')

  return output
def post_attention ( h , attn_vec , d_model , n_head , d_head , dropout , is_training , kernel_initializer , residual = True ) : # post-attention projection (back to `d_model`) proj_o = tf . get_variable ( 'o/kernel' , [ d_model , n_head , d_head ] , dtype = h . dtype , initializer = kernel_initializer ) attn_out = tf . einsum ( 'ibnd,hnd->ibh' , attn_vec , proj_o ) attn_out = tf . layers . dropout ( attn_out , dropout , training = is_training ) if residual : output = tf . contrib . layers . layer_norm ( attn_out + h , begin_norm_axis = - 1 , scope = 'LayerNorm' ) else : output = tf . contrib . layers . layer_norm ( attn_out , begin_norm_axis = - 1 , scope = 'LayerNorm' ) return output