(inp, d_model, d_inner, dropout, kernel_initializer,
                     activation_type='relu', scope='ff', is_training=True,
                     reuse=None)
return output
Position-wise Feed-forward Network.
Position-wise Feed-forward Network.
Position - wise Feed - forward Network .
def positionwise_ffn(inp, d_model, d_inner, dropout, kernel_initializer,
                     activation_type='relu', scope='ff', is_training=True,
                     reuse=None):
  """Position-wise Feed-forward Network."""
  if activation_type == 'relu':
    activation = tf.nn.relu
  elif activation_type == 'gelu':
    activation = gelu
  else:
    raise ValueError('Unsupported activation type {}'.format(activation_type))

  output = inp
  with tf.variable_scope(scope, reuse=reuse):
    output = tf.layers.dense(output, d_inner, activation=activation,
                             kernel_initializer=kernel_initializer,
                             name='layer_1')
    output = tf.layers.dropout(output, dropout, training=is_training,
                               name='drop_1')
    output = tf.layers.dense(output, d_model,
                             kernel_initializer=kernel_initializer,
                             name='layer_2')
    output = tf.layers.dropout(output, dropout, training=is_training,
                               name='drop_2')
    output = tf.contrib.layers.layer_norm(output + inp, begin_norm_axis=-1,
                                          scope='LayerNorm')
  return output
def positionwise_ffn ( inp , d_model , d_inner , dropout , kernel_initializer , activation_type = 'relu' , scope = 'ff' , is_training = True , reuse = None ) : if activation_type == 'relu' : activation = tf . nn . relu elif activation_type == 'gelu' : activation = gelu else : raise ValueError ( 'Unsupported activation type {}' . format ( activation_type ) ) output = inp with tf . variable_scope ( scope , reuse = reuse ) : output = tf . layers . dense ( output , d_inner , activation = activation , kernel_initializer = kernel_initializer , name = 'layer_1' ) output = tf . layers . dropout ( output , dropout , training = is_training , name = 'drop_1' ) output = tf . layers . dense ( output , d_model , kernel_initializer = kernel_initializer , name = 'layer_2' ) output = tf . layers . dropout ( output , dropout , training = is_training , name = 'drop_2' ) output = tf . contrib . layers . layer_norm ( output + inp , begin_norm_axis = - 1 , scope = 'LayerNorm' ) return output