(inputs_BxIxD, start_index=None)
return inputs_BxIxD + signal_BxIxD
Adds a transformer-style timing signal to inputs.

  Using periodic signals as in https://arxiv.org/abs/1706.03762.
  Generalized to allow each example in a batch to begin at a different index.

  Args:
    inputs_BxIxD: input representation.
    start_index: tensor of starting pos. [batch_size]

  Returns:
    output: representation with time signal added, same shape as input.
Adds a transformer-style timing signal to inputs.
Adds a transformer - style timing signal to inputs .
def add_time_signal(inputs_BxIxD, start_index=None):
  """Adds a transformer-style timing signal to inputs.

  Using periodic signals as in https://arxiv.org/abs/1706.03762.
  Generalized to allow each example in a batch to begin at a different index.

  Args:
    inputs_BxIxD: input representation.
    start_index: tensor of starting pos. [batch_size]

  Returns:
    output: representation with time signal added, same shape as input.
  """

  dtype = inputs_BxIxD.dtype
  B, I, D = inputs_BxIxD.shape
  if D % 2 != 0:
    raise ValueError("Input dimension must be even.")
  start_Bx1 = tf.zeros((B, 1), tf.int32) if start_index is None else start_index

  pos_1xI = tf.expand_dims(tf.range(I), 0)
  pos_BxI = tf.tile(pos_1xI, [B, 1]) + tf.cast(start_Bx1, tf.int32)
  pos_BxI = tf.cast(pos_BxI, dtype)
  N = D // 2
  log_time_incr = (
      math.log(_MAX_TIMESCALE / _MIN_TIMESCALE) /
      tf.maximum(tf.cast(N, dtype) - 1, 1))
  inv_scale_N = _MIN_TIMESCALE * tf.exp(
      tf.cast(tf.range(N), dtype) * -log_time_incr)
  time_BxIxN = tf.expand_dims(pos_BxI, 2) * tf.reshape(inv_scale_N, [1, 1, -1])
  signal_BxIxD = tf.concat([tf.sin(time_BxIxN), tf.cos(time_BxIxN)], axis=2)
  signal_BxIxD = tf.reshape(signal_BxIxD, [B, I, D])
  return inputs_BxIxD + signal_BxIxD
def add_time_signal ( inputs_BxIxD , start_index = None ) : dtype = inputs_BxIxD . dtype B , I , D = inputs_BxIxD . shape if D % 2 != 0 : raise ValueError ( "Input dimension must be even." ) start_Bx1 = tf . zeros ( ( B , 1 ) , tf . int32 ) if start_index is None else start_index pos_1xI = tf . expand_dims ( tf . range ( I ) , 0 ) pos_BxI = tf . tile ( pos_1xI , [ B , 1 ] ) + tf . cast ( start_Bx1 , tf . int32 ) pos_BxI = tf . cast ( pos_BxI , dtype ) N = D // 2 log_time_incr = ( math . log ( _MAX_TIMESCALE / _MIN_TIMESCALE ) / tf . maximum ( tf . cast ( N , dtype ) - 1 , 1 ) ) inv_scale_N = _MIN_TIMESCALE * tf . exp ( tf . cast ( tf . range ( N ) , dtype ) * - log_time_incr ) time_BxIxN = tf . expand_dims ( pos_BxI , 2 ) * tf . reshape ( inv_scale_N , [ 1 , 1 , - 1 ] ) signal_BxIxD = tf . concat ( [ tf . sin ( time_BxIxN ) , tf . cos ( time_BxIxN ) ] , axis = 2 ) signal_BxIxD = tf . reshape ( signal_BxIxD , [ B , I , D ] ) return inputs_BxIxD + signal_BxIxD