(self,
               config,
               is_training,
               input_ids,
               input_mask=None,
               token_type_ids=None,
               use_one_hot_embeddings=False,
               scope=None)
Constructor for BertModel.

    Args:
      config: `BertConfig` instance.
      is_training: bool. true for training model, false for eval model. Controls
        whether dropout will be applied.
      input_ids: int32 Tensor of shape [batch_size, seq_length].
      input_mask: (optional) int32 Tensor of shape [batch_size, seq_length].
      token_type_ids: (optional) int32 Tensor of shape [batch_size, seq_length].
      use_one_hot_embeddings: (optional) bool. Whether to use one-hot word
        embeddings or tf.embedding_lookup() for the word embeddings.
      scope: (optional) variable scope. Defaults to "bert".

    Raises:
      ValueError: The config is invalid or one of the input tensor shapes
        is invalid.
Constructor for BertModel.
Constructor for BertModel .
def __init__(self,
               config,
               is_training,
               input_ids,
               input_mask=None,
               token_type_ids=None,
               use_one_hot_embeddings=False,
               scope=None):
    """Constructor for BertModel.

    Args:
      config: `BertConfig` instance.
      is_training: bool. true for training model, false for eval model. Controls
        whether dropout will be applied.
      input_ids: int32 Tensor of shape [batch_size, seq_length].
      input_mask: (optional) int32 Tensor of shape [batch_size, seq_length].
      token_type_ids: (optional) int32 Tensor of shape [batch_size, seq_length].
      use_one_hot_embeddings: (optional) bool. Whether to use one-hot word
        embeddings or tf.embedding_lookup() for the word embeddings.
      scope: (optional) variable scope. Defaults to "bert".

    Raises:
      ValueError: The config is invalid or one of the input tensor shapes
        is invalid.
    """
    config = copy.deepcopy(config)
    if not is_training:
      config.hidden_dropout_prob = 0.0
      config.attention_probs_dropout_prob = 0.0

    input_shape = get_shape_list(input_ids, expected_rank=2)
    batch_size = input_shape[0]
    seq_length = input_shape[1]

    if input_mask is None:
      input_mask = tf.ones(shape=[batch_size, seq_length], dtype=tf.int32)

    if token_type_ids is None:
      token_type_ids = tf.zeros(shape=[batch_size, seq_length], dtype=tf.int32)

    with tf.compat.v1.variable_scope(scope, default_name="bert"):
      with tf.compat.v1.variable_scope("embeddings"):
        # Perform embedding lookup on the word ids.
        (self.embedding_output, self.embedding_table) = embedding_lookup(
            input_ids=input_ids,
            vocab_size=config.vocab_size,
            embedding_size=config.hidden_size,
            initializer_range=config.initializer_range,
            word_embedding_name="word_embeddings",
            use_one_hot_embeddings=use_one_hot_embeddings)

        # Add positional embeddings and token type embeddings, then layer
        # normalize and perform dropout.
        self.embedding_output = embedding_postprocessor(
            input_tensor=self.embedding_output,
            use_token_type=True,
            token_type_ids=token_type_ids,
            token_type_vocab_size=config.type_vocab_size,
            token_type_embedding_name="token_type_embeddings",
            use_position_embeddings=True,
            position_embedding_name="position_embeddings",
            initializer_range=config.initializer_range,
            max_position_embeddings=config.max_position_embeddings,
            dropout_prob=config.hidden_dropout_prob)

      with tf.compat.v1.variable_scope("encoder"):
        # This converts a 2D mask of shape [batch_size, seq_length] to a 3D
        # mask of shape [batch_size, seq_length, seq_length] which is used
        # for the attention scores.
        attention_mask = create_attention_mask_from_input_mask(
            input_ids, input_mask)

        # Run the stacked transformer.
        # `sequence_output` shape = [batch_size, seq_length, hidden_size].
        self.all_encoder_layers = transformer_model(
            input_tensor=self.embedding_output,
            attention_mask=attention_mask,
            hidden_size=config.hidden_size,
            num_hidden_layers=config.num_hidden_layers,
            num_attention_heads=config.num_attention_heads,
            intermediate_size=config.intermediate_size,
            intermediate_act_fn=get_activation(config.hidden_act),
            hidden_dropout_prob=config.hidden_dropout_prob,
            attention_probs_dropout_prob=config.attention_probs_dropout_prob,
            initializer_range=config.initializer_range,
            do_return_all_layers=True)

      self.sequence_output = self.all_encoder_layers[-1]
      # The "pooler" converts the encoded sequence tensor of shape
      # [batch_size, seq_length, hidden_size] to a tensor of shape
      # [batch_size, hidden_size]. This is necessary for segment-level
      # (or segment-pair-level) classification tasks where we need a fixed
      # dimensional representation of the segment.
      with tf.compat.v1.variable_scope("pooler"):
        # We "pool" the model by simply taking the hidden state corresponding
        # to the first token. We assume that this has been pre-trained
        first_token_tensor = tf.squeeze(self.sequence_output[:, 0:1, :], axis=1)
        self.pooled_output = tf.compat.v1.layers.dense(
            first_token_tensor,
            config.hidden_size,
            activation=tf.tanh,
            kernel_initializer=create_initializer(config.initializer_range))
def __init__ ( self , config , is_training , input_ids , input_mask = None , token_type_ids = None , use_one_hot_embeddings = False , scope = None ) : config = copy . deepcopy ( config ) if not is_training : config . hidden_dropout_prob = 0.0 config . attention_probs_dropout_prob = 0.0 input_shape = get_shape_list ( input_ids , expected_rank = 2 ) batch_size = input_shape [ 0 ] seq_length = input_shape [ 1 ] if input_mask is None : input_mask = tf . ones ( shape = [ batch_size , seq_length ] , dtype = tf . int32 ) if token_type_ids is None : token_type_ids = tf . zeros ( shape = [ batch_size , seq_length ] , dtype = tf . int32 ) with tf . compat . v1 . variable_scope ( scope , default_name = "bert" ) : with tf . compat . v1 . variable_scope ( "embeddings" ) : # Perform embedding lookup on the word ids. ( self . embedding_output , self . embedding_table ) = embedding_lookup ( input_ids = input_ids , vocab_size = config . vocab_size , embedding_size = config . hidden_size , initializer_range = config . initializer_range , word_embedding_name = "word_embeddings" , use_one_hot_embeddings = use_one_hot_embeddings ) # Add positional embeddings and token type embeddings, then layer # normalize and perform dropout. self . embedding_output = embedding_postprocessor ( input_tensor = self . embedding_output , use_token_type = True , token_type_ids = token_type_ids , token_type_vocab_size = config . type_vocab_size , token_type_embedding_name = "token_type_embeddings" , use_position_embeddings = True , position_embedding_name = "position_embeddings" , initializer_range = config . initializer_range , max_position_embeddings = config . max_position_embeddings , dropout_prob = config . hidden_dropout_prob ) with tf . compat . v1 . variable_scope ( "encoder" ) : # This converts a 2D mask of shape [batch_size, seq_length] to a 3D # mask of shape [batch_size, seq_length, seq_length] which is used # for the attention scores. attention_mask = create_attention_mask_from_input_mask ( input_ids , input_mask ) # Run the stacked transformer. # `sequence_output` shape = [batch_size, seq_length, hidden_size]. self . all_encoder_layers = transformer_model ( input_tensor = self . embedding_output , attention_mask = attention_mask , hidden_size = config . hidden_size , num_hidden_layers = config . num_hidden_layers , num_attention_heads = config . num_attention_heads , intermediate_size = config . intermediate_size , intermediate_act_fn = get_activation ( config . hidden_act ) , hidden_dropout_prob = config . hidden_dropout_prob , attention_probs_dropout_prob = config . attention_probs_dropout_prob , initializer_range = config . initializer_range , do_return_all_layers = True ) self . sequence_output = self . all_encoder_layers [ - 1 ] # The "pooler" converts the encoded sequence tensor of shape # [batch_size, seq_length, hidden_size] to a tensor of shape # [batch_size, hidden_size]. This is necessary for segment-level # (or segment-pair-level) classification tasks where we need a fixed # dimensional representation of the segment. with tf . compat . v1 . variable_scope ( "pooler" ) : # We "pool" the model by simply taking the hidden state corresponding # to the first token. We assume that this has been pre-trained first_token_tensor = tf . squeeze ( self . sequence_output [ : , 0 : 1 , : ] , axis = 1 ) self . pooled_output = tf . compat . v1 . layers . dense ( first_token_tensor , config . hidden_size , activation = tf . tanh , kernel_initializer = create_initializer ( config . initializer_range ) )