(self, x)
return embedding, y
Embedding layers.
Embedding layers.
Embedding layers .
def dense_layer(self, x):
    ''' Embedding layers. '''
    with tf.variable_scope('dense'):
      shape = x.shape[-1].value
      hidden_dims = self.netconf['hidden_dims']
      y = x
      use_bn = self.netconf['use_bn']
      remove_nonlin = self.netconf['remove_last_nonlinearity']

      for idx, hidden in enumerate(hidden_dims):
        last_layer = idx == (len(hidden_dims) - 1)
        layer_add_nonlin = not last_layer or not remove_nonlin
        y = common_layers.linear(
            y,
            'dense-matmul-%d' % (idx + 1), [shape, hidden],
            has_bias=(layer_add_nonlin or not use_bn))
        shape = hidden
        embedding = y
        if layer_add_nonlin:
          y = tf.nn.relu(y)
        if use_bn:
          y = tf.layers.batch_normalization(
              y,
              axis=-1,
              momentum=0.99,
              training=self.train,
              name='dense-bn-%d' % (idx + 1))
        if self.netconf['use_dropout'] and layer_add_nonlin:
          y = tf.layers.dropout(
              y, self.netconf['dropout_rate'], training=self.train)
      if self.netconf['embedding_after_linear']:
        logging.info('Output embedding right after linear layer.')
      else:
        logging.info('Output embedding after non-lin, batch norm and dropout.')
        embedding = y
    return embedding, y
def dense_layer ( self , x ) : with tf . variable_scope ( 'dense' ) : shape = x . shape [ - 1 ] . value hidden_dims = self . netconf [ 'hidden_dims' ] y = x use_bn = self . netconf [ 'use_bn' ] remove_nonlin = self . netconf [ 'remove_last_nonlinearity' ] for idx , hidden in enumerate ( hidden_dims ) : last_layer = idx == ( len ( hidden_dims ) - 1 ) layer_add_nonlin = not last_layer or not remove_nonlin y = common_layers . linear ( y , 'dense-matmul-%d' % ( idx + 1 ) , [ shape , hidden ] , has_bias = ( layer_add_nonlin or not use_bn ) ) shape = hidden embedding = y if layer_add_nonlin : y = tf . nn . relu ( y ) if use_bn : y = tf . layers . batch_normalization ( y , axis = - 1 , momentum = 0.99 , training = self . train , name = 'dense-bn-%d' % ( idx + 1 ) ) if self . netconf [ 'use_dropout' ] and layer_add_nonlin : y = tf . layers . dropout ( y , self . netconf [ 'dropout_rate' ] , training = self . train ) if self . netconf [ 'embedding_after_linear' ] : logging . info ( 'Output embedding right after linear layer.' ) else : logging . info ( 'Output embedding after non-lin, batch norm and dropout.' ) embedding = y return embedding , y