(dataset, training, pad_symbol=0)
return dataset
Pre-processing function that concatenates input and target for LM.
Pre-processing function that concatenates input and target for LM.
Pre - processing function that concatenates input and target for LM .
def concat_preprocess(dataset, training, pad_symbol=0):
  """Pre-processing function that concatenates input and target for LM."""
  del training

  def concat(features, targets):
    inp = features['inputs']
    pad = tf.expand_dims(tf.zeros_like(inp[0]) + pad_symbol, axis=0)
    concat = tf.concat([pad, inp, pad, targets], axis=0)
    # Note: we're updating existing features dictionary here, so make sure
    # it is not re-used in some other ways outside of this function.
    features['inputs'] = concat
    return features, concat

  dataset = dataset.map(concat)
  return dataset
def concat_preprocess ( dataset , training , pad_symbol = 0 ) : del training def concat ( features , targets ) : inp = features [ 'inputs' ] pad = tf . expand_dims ( tf . zeros_like ( inp [ 0 ] ) + pad_symbol , axis = 0 ) concat = tf . concat ( [ pad , inp , pad , targets ] , axis = 0 ) # Note: we're updating existing features dictionary here, so make sure # it is not re-used in some other ways outside of this function. features [ 'inputs' ] = concat return features , concat dataset = dataset . map ( concat ) return dataset