(max_len, embed_dim, use_const, name)
return pos_embed
generate position embedding matrix, two optional types:
    constant(untrainable) and trainable.
    Args:
      max_len, embed_dim, use_const

    Return:
      pos_embed: [max_len, embed_dim]
generate position embedding matrix, two optional types:
    constant(untrainable) and trainable.
    Args:
      max_len, embed_dim, use_const
generate position embedding matrix two optional types : constant ( untrainable ) and trainable . Args : max_len embed_dim use_const
def get_pos_embedding_matrix(max_len, embed_dim, use_const, name):
    """
    generate position embedding matrix, two optional types:
    constant(untrainable) and trainable.
    Args:
      max_len, embed_dim, use_const

    Return:
      pos_embed: [max_len, embed_dim]
    """
    # First part of the PE function: sin and cos argument
    if use_const:
      pos_embed = np.array([[
          pos / np.power(10000, (i - i % 2) / embed_dim)
          for i in range(embed_dim)
      ]
                            for pos in range(max_len)])

      # Second part, apply the cosine to even columns and sin to odds.
      pos_embed[:, 0::2] = np.sin(pos_embed[:, 0::2])  # dim 2i
      pos_embed[:, 1::2] = np.cos(pos_embed[:, 1::2])  # dim 2i+1
      pos_embed = pos_embed[np.newaxis, ...]
      pos_embed = tf.cast(pos_embed, dtype=tf.float32)
    else:
      pos_embed = tf.get_variable(
          name=name,
          shape=[max_len, embed_dim],
          initializer=tf.random_uniform_initializer(-0.1, 0.1))
      pos_embed = tf.expand_dims(pos_embed, 0)

    return pos_embed
def get_pos_embedding_matrix ( max_len , embed_dim , use_const , name ) : # First part of the PE function: sin and cos argument if use_const : pos_embed = np . array ( [ [ pos / np . power ( 10000 , ( i - i % 2 ) / embed_dim ) for i in range ( embed_dim ) ] for pos in range ( max_len ) ] ) # Second part, apply the cosine to even columns and sin to odds. pos_embed [ : , 0 : : 2 ] = np . sin ( pos_embed [ : , 0 : : 2 ] ) # dim 2i pos_embed [ : , 1 : : 2 ] = np . cos ( pos_embed [ : , 1 : : 2 ] ) # dim 2i+1 pos_embed = pos_embed [ np . newaxis , ... ] pos_embed = tf . cast ( pos_embed , dtype = tf . float32 ) else : pos_embed = tf . get_variable ( name = name , shape = [ max_len , embed_dim ] , initializer = tf . random_uniform_initializer ( - 0.1 , 0.1 ) ) pos_embed = tf . expand_dims ( pos_embed , 0 ) return pos_embed