(dataset, training)
return dataset
Concatenates inputs, 0, targets, with masking only for targets.
Concatenates inputs, 0, targets, with masking only for targets.
Concatenates inputs 0 targets with masking only for targets .
def lm_token_preprocessing(dataset, training):
  """Concatenates inputs, 0, targets, with masking only for targets."""
  del training

  def concat_and_add_mask(x):
    inp = x['inputs']
    targets = x['targets']
    pad = tf.expand_dims(tf.zeros_like(inp[0]), axis=0)
    concat = tf.concat([inp, pad, targets], axis=0)
    mask = tf.concat([tf.zeros_like(inp), pad, tf.ones_like(targets)], axis=0)
    x['inputs'] = concat
    x['targets'] = concat
    x['mask'] = mask
    return x

  dataset = dataset.map(concat_and_add_mask)
  return dataset
def lm_token_preprocessing ( dataset , training ) : del training def concat_and_add_mask ( x ) : inp = x [ 'inputs' ] targets = x [ 'targets' ] pad = tf . expand_dims ( tf . zeros_like ( inp [ 0 ] ) , axis = 0 ) concat = tf . concat ( [ inp , pad , targets ] , axis = 0 ) mask = tf . concat ( [ tf . zeros_like ( inp ) , pad , tf . ones_like ( targets ) ] , axis = 0 ) x [ 'inputs' ] = concat x [ 'targets' ] = concat x [ 'mask' ] = mask return x dataset = dataset . map ( concat_and_add_mask ) return dataset