(input_ids, # Factorized embedding parameterization provide by albert
                     vocab_size,
                     hidden_size,
                     embedding_size=128,
                     initializer_range=0.02,
                     word_embedding_name="word_embeddings",
                     use_one_hot_embeddings=False)
return (output, embedding_table, project_variable)
Looks up words embeddings for id tensor, but in a factorized style followed by albert. it is used to reduce much percentage of parameters previous exists.
       Check "Factorized embedding parameterization" session in the paper.

     Args:
       input_ids: int32 Tensor of shape [batch_size, seq_length] containing word
         ids.
       vocab_size: int. Size of the embedding vocabulary.
       embedding_size: int. Width of the word embeddings.
       initializer_range: float. Embedding initialization range.
       word_embedding_name: string. Name of the embedding table.
       use_one_hot_embeddings: bool. If True, use one-hot method for word
         embeddings. If False, use `tf.gather()`.

     Returns:
       float Tensor of shape [batch_size, seq_length, embedding_size].
Looks up words embeddings for id tensor, but in a factorized style followed by albert. it is used to reduce much percentage of parameters previous exists.
       Check "Factorized embedding parameterization" session in the paper.
Looks up words embeddings for id tensor but in a factorized style followed by albert . it is used to reduce much percentage of parameters previous exists . Check Factorized embedding parameterization session in the paper .
def embedding_lookup_factorized(input_ids, # Factorized embedding parameterization provide by albert
                     vocab_size,
                     hidden_size,
                     embedding_size=128,
                     initializer_range=0.02,
                     word_embedding_name="word_embeddings",
                     use_one_hot_embeddings=False):
    """Looks up words embeddings for id tensor, but in a factorized style followed by albert. it is used to reduce much percentage of parameters previous exists.
       Check "Factorized embedding parameterization" session in the paper.

     Args:
       input_ids: int32 Tensor of shape [batch_size, seq_length] containing word
         ids.
       vocab_size: int. Size of the embedding vocabulary.
       embedding_size: int. Width of the word embeddings.
       initializer_range: float. Embedding initialization range.
       word_embedding_name: string. Name of the embedding table.
       use_one_hot_embeddings: bool. If True, use one-hot method for word
         embeddings. If False, use `tf.gather()`.

     Returns:
       float Tensor of shape [batch_size, seq_length, embedding_size].
     """
    # This function assumes that the input is of shape [batch_size, seq_length,
    # num_inputs].
    #
    # If the input is a 2D tensor of shape [batch_size, seq_length], we
    # reshape to [batch_size, seq_length, 1].

    # 1.first project one-hot vectors into a lower dimensional embedding space of size E
    print("embedding_lookup_factorized. factorized embedding parameterization is used.")
    if input_ids.shape.ndims == 2:
        input_ids = tf.expand_dims(input_ids, axis=[-1])  # shape of input_ids is:[ batch_size, seq_length, 1]

    embedding_table = tf.get_variable(  # [vocab_size, embedding_size]
        name=word_embedding_name,
        shape=[vocab_size, embedding_size],
        initializer=create_initializer(initializer_range))

    flat_input_ids = tf.reshape(input_ids, [-1])  # one rank. shape as (batch_size * sequence_length,)
    if use_one_hot_embeddings:
        one_hot_input_ids = tf.one_hot(flat_input_ids,depth=vocab_size)  # one_hot_input_ids=[batch_size * sequence_length,vocab_size]
        output_middle = tf.matmul(one_hot_input_ids, embedding_table)  # output=[batch_size * sequence_length,embedding_size]
    else:
        output_middle = tf.gather(embedding_table,flat_input_ids)  # [vocab_size, embedding_size]*[batch_size * sequence_length,]--->[batch_size * sequence_length,embedding_size]

    # 2. project vector(output_middle) to the hidden space
    project_variable = tf.get_variable(  # [embedding_size, hidden_size]
        name=word_embedding_name+"_2",
        shape=[embedding_size, hidden_size],
        initializer=create_initializer(initializer_range))
    output = tf.matmul(output_middle, project_variable) # ([batch_size * sequence_length, embedding_size] * [embedding_size, hidden_size])--->[batch_size * sequence_length, hidden_size]
    # reshape back to 3 rank
    input_shape = get_shape_list(input_ids)  # input_shape=[ batch_size, seq_length, 1]
    batch_size, sequene_length, _=input_shape
    output = tf.reshape(output, (batch_size,sequene_length,hidden_size))  # output=[batch_size, sequence_length, hidden_size]
    return (output, embedding_table, project_variable)
def embedding_lookup_factorized ( input_ids , # Factorized embedding parameterization provide by albert vocab_size , hidden_size , embedding_size = 128 , initializer_range = 0.02 , word_embedding_name = "word_embeddings" , use_one_hot_embeddings = False ) : # This function assumes that the input is of shape [batch_size, seq_length, # num_inputs]. # # If the input is a 2D tensor of shape [batch_size, seq_length], we # reshape to [batch_size, seq_length, 1]. # 1.first project one-hot vectors into a lower dimensional embedding space of size E print ( "embedding_lookup_factorized. factorized embedding parameterization is used." ) if input_ids . shape . ndims == 2 : input_ids = tf . expand_dims ( input_ids , axis = [ - 1 ] ) # shape of input_ids is:[ batch_size, seq_length, 1] embedding_table = tf . get_variable ( # [vocab_size, embedding_size] name = word_embedding_name , shape = [ vocab_size , embedding_size ] , initializer = create_initializer ( initializer_range ) ) flat_input_ids = tf . reshape ( input_ids , [ - 1 ] ) # one rank. shape as (batch_size * sequence_length,) if use_one_hot_embeddings : one_hot_input_ids = tf . one_hot ( flat_input_ids , depth = vocab_size ) # one_hot_input_ids=[batch_size * sequence_length,vocab_size] output_middle = tf . matmul ( one_hot_input_ids , embedding_table ) # output=[batch_size * sequence_length,embedding_size] else : output_middle = tf . gather ( embedding_table , flat_input_ids ) # [vocab_size, embedding_size]*[batch_size * sequence_length,]--->[batch_size * sequence_length,embedding_size] # 2. project vector(output_middle) to the hidden space project_variable = tf . get_variable ( # [embedding_size, hidden_size] name = word_embedding_name + "_2" , shape = [ embedding_size , hidden_size ] , initializer = create_initializer ( initializer_range ) ) output = tf . matmul ( output_middle , project_variable ) # ([batch_size * sequence_length, embedding_size] * [embedding_size, hidden_size])--->[batch_size * sequence_length, hidden_size] # reshape back to 3 rank input_shape = get_shape_list ( input_ids ) # input_shape=[ batch_size, seq_length, 1] batch_size , sequene_length , _ = input_shape output = tf . reshape ( output , ( batch_size , sequene_length , hidden_size ) ) # output=[batch_size, sequence_length, hidden_size] return ( output , embedding_table , project_variable )