(summary_type, hidden, d_model, n_head, d_head, dropout,
                       dropatt, input_mask, is_training, initializer,
                       scope=None, reuse=None, use_proj=True)
return summary
Different classification tasks may not may not share the same parameters
      to summarize the sequence features.

      If shared, one can keep the `scope` to the default value `None`.
      Otherwise, one should specify a different `scope` for each task.
Different classification tasks may not may not share the same parameters
      to summarize the sequence features.
Different classification tasks may not may not share the same parameters to summarize the sequence features .
def summarize_sequence(summary_type, hidden, d_model, n_head, d_head, dropout,
                       dropatt, input_mask, is_training, initializer,
                       scope=None, reuse=None, use_proj=True):

  """
      Different classification tasks may not may not share the same parameters
      to summarize the sequence features.

      If shared, one can keep the `scope` to the default value `None`.
      Otherwise, one should specify a different `scope` for each task.
  """

  with tf.variable_scope(scope, 'sequnece_summary', reuse=reuse):
    if summary_type == 'last':
      summary = hidden[-1]
    elif summary_type == 'first':
      summary = hidden[0]
    elif summary_type == 'mean':
      summary = tf.reduce_mean(hidden, axis=0)
    elif summary_type == 'attn':
      bsz = tf.shape(hidden)[1]

      summary_bias = tf.get_variable('summary_bias', [d_model],
                                     dtype=hidden.dtype,
                                     initializer=initializer)
      summary_bias = tf.tile(summary_bias[None, None], [1, bsz, 1])

      if input_mask is not None:
        input_mask = input_mask[None, :, :, None]

      summary = multihead_attn(summary_bias, hidden, hidden, input_mask,
                               d_model, n_head, d_head, dropout, dropatt,
                               is_training, initializer, residual=False)
      summary = summary[0]
    else:
      raise ValueError('Unsupported summary type {}'.format(summary_type))

    # use another projection as in BERT
    if use_proj:
      summary = tf.layers.dense(
          summary,
          d_model,
          activation=tf.tanh,
          kernel_initializer=initializer,
          name='summary')

    # dropout
    summary = tf.layers.dropout(
        summary, dropout, training=is_training,
        name='dropout')

  return summary
def summarize_sequence ( summary_type , hidden , d_model , n_head , d_head , dropout , dropatt , input_mask , is_training , initializer , scope = None , reuse = None , use_proj = True ) : with tf . variable_scope ( scope , 'sequnece_summary' , reuse = reuse ) : if summary_type == 'last' : summary = hidden [ - 1 ] elif summary_type == 'first' : summary = hidden [ 0 ] elif summary_type == 'mean' : summary = tf . reduce_mean ( hidden , axis = 0 ) elif summary_type == 'attn' : bsz = tf . shape ( hidden ) [ 1 ] summary_bias = tf . get_variable ( 'summary_bias' , [ d_model ] , dtype = hidden . dtype , initializer = initializer ) summary_bias = tf . tile ( summary_bias [ None , None ] , [ 1 , bsz , 1 ] ) if input_mask is not None : input_mask = input_mask [ None , : , : , None ] summary = multihead_attn ( summary_bias , hidden , hidden , input_mask , d_model , n_head , d_head , dropout , dropatt , is_training , initializer , residual = False ) summary = summary [ 0 ] else : raise ValueError ( 'Unsupported summary type {}' . format ( summary_type ) ) # use another projection as in BERT if use_proj : summary = tf . layers . dense ( summary , d_model , activation = tf . tanh , kernel_initializer = initializer , name = 'summary' ) # dropout summary = tf . layers . dropout ( summary , dropout , training = is_training , name = 'dropout' ) return summary