(qlen, klen, d_model, clamp_len, attn_type,
                                 bi_data, bsz=None, dtype=None)
return pos_emb
create relative positional encoding.
create relative positional encoding.
create relative positional encoding .
def relative_positional_encoding(qlen, klen, d_model, clamp_len, attn_type,
                                 bi_data, bsz=None, dtype=None):
  """create relative positional encoding."""
  freq_seq = tf.range(0, d_model, 2.0)
  if dtype is not None and dtype != tf.float32:
    freq_seq = tf.cast(freq_seq, dtype=dtype)
  inv_freq = 1 / (10000 ** (freq_seq / d_model))

  if attn_type == 'bi':
    # beg, end = klen - 1, -qlen
    beg, end = klen, -qlen
  elif attn_type == 'uni':
    # beg, end = klen - 1, -1
    beg, end = klen, -1
  else:
    raise ValueError('Unknown `attn_type` {}.'.format(attn_type))

  if bi_data:
    fwd_pos_seq = tf.range(beg, end, -1.0)
    bwd_pos_seq = tf.range(-beg, -end, 1.0)

    if dtype is not None and dtype != tf.float32:
      fwd_pos_seq = tf.cast(fwd_pos_seq, dtype=dtype)
      bwd_pos_seq = tf.cast(bwd_pos_seq, dtype=dtype)

    if clamp_len > 0:
      fwd_pos_seq = tf.clip_by_value(fwd_pos_seq, -clamp_len, clamp_len)
      bwd_pos_seq = tf.clip_by_value(bwd_pos_seq, -clamp_len, clamp_len)

    if bsz is not None:
      # With bi_data, the batch size should be divisible by 2.
      assert bsz%2 == 0
      fwd_pos_emb = positional_embedding(fwd_pos_seq, inv_freq, bsz//2)
      bwd_pos_emb = positional_embedding(bwd_pos_seq, inv_freq, bsz//2)
    else:
      fwd_pos_emb = positional_embedding(fwd_pos_seq, inv_freq)
      bwd_pos_emb = positional_embedding(bwd_pos_seq, inv_freq)

    pos_emb = tf.concat([fwd_pos_emb, bwd_pos_emb], axis=1)
  else:
    fwd_pos_seq = tf.range(beg, end, -1.0)
    if dtype is not None and dtype != tf.float32:
      fwd_pos_seq = tf.cast(fwd_pos_seq, dtype=dtype)
    if clamp_len > 0:
      fwd_pos_seq = tf.clip_by_value(fwd_pos_seq, -clamp_len, clamp_len)
    pos_emb = positional_embedding(fwd_pos_seq, inv_freq, bsz)

  return pos_emb
def relative_positional_encoding ( qlen , klen , d_model , clamp_len , attn_type , bi_data , bsz = None , dtype = None ) : freq_seq = tf . range ( 0 , d_model , 2.0 ) if dtype is not None and dtype != tf . float32 : freq_seq = tf . cast ( freq_seq , dtype = dtype ) inv_freq = 1 / ( 10000 ** ( freq_seq / d_model ) ) if attn_type == 'bi' : # beg, end = klen - 1, -qlen beg , end = klen , - qlen elif attn_type == 'uni' : # beg, end = klen - 1, -1 beg , end = klen , - 1 else : raise ValueError ( 'Unknown `attn_type` {}.' . format ( attn_type ) ) if bi_data : fwd_pos_seq = tf . range ( beg , end , - 1.0 ) bwd_pos_seq = tf . range ( - beg , - end , 1.0 ) if dtype is not None and dtype != tf . float32 : fwd_pos_seq = tf . cast ( fwd_pos_seq , dtype = dtype ) bwd_pos_seq = tf . cast ( bwd_pos_seq , dtype = dtype ) if clamp_len > 0 : fwd_pos_seq = tf . clip_by_value ( fwd_pos_seq , - clamp_len , clamp_len ) bwd_pos_seq = tf . clip_by_value ( bwd_pos_seq , - clamp_len , clamp_len ) if bsz is not None : # With bi_data, the batch size should be divisible by 2. assert bsz % 2 == 0 fwd_pos_emb = positional_embedding ( fwd_pos_seq , inv_freq , bsz // 2 ) bwd_pos_emb = positional_embedding ( bwd_pos_seq , inv_freq , bsz // 2 ) else : fwd_pos_emb = positional_embedding ( fwd_pos_seq , inv_freq ) bwd_pos_emb = positional_embedding ( bwd_pos_seq , inv_freq ) pos_emb = tf . concat ( [ fwd_pos_emb , bwd_pos_emb ] , axis = 1 ) else : fwd_pos_seq = tf . range ( beg , end , - 1.0 ) if dtype is not None and dtype != tf . float32 : fwd_pos_seq = tf . cast ( fwd_pos_seq , dtype = dtype ) if clamp_len > 0 : fwd_pos_seq = tf . clip_by_value ( fwd_pos_seq , - clamp_len , clamp_len ) pos_emb = positional_embedding ( fwd_pos_seq , inv_freq , bsz ) return pos_emb