(dataset, training, max_length=-1, max_eval_length=-1)
return dataset
Preprocessing for WMT: filter exceeding maximum length and concatenate.
Preprocessing for WMT: filter exceeding maximum length and concatenate.
Preprocessing for WMT : filter exceeding maximum length and concatenate .
def wmt_concat_preprocess(dataset, training, max_length=-1, max_eval_length=-1):
  """Preprocessing for WMT: filter exceeding maximum length and concatenate."""
  dataset = wmt_preprocess(dataset, training, max_length, max_eval_length)

  def concat_and_add_mask(features, targets):
    inp = features['inputs']
    pad = tf.expand_dims(tf.zeros_like(inp[0]), axis=0)
    concat = tf.concat([inp, pad, targets], axis=0)
    mask = tf.concat([tf.zeros_like(inp), pad, tf.ones_like(targets)], axis=0)
    features['inputs'] = concat
    features['mask'] = mask
    return features, concat

  dataset = dataset.map(concat_and_add_mask)
  return dataset
def wmt_concat_preprocess ( dataset , training , max_length = - 1 , max_eval_length = - 1 ) : dataset = wmt_preprocess ( dataset , training , max_length , max_eval_length ) def concat_and_add_mask ( features , targets ) : inp = features [ 'inputs' ] pad = tf . expand_dims ( tf . zeros_like ( inp [ 0 ] ) , axis = 0 ) concat = tf . concat ( [ inp , pad , targets ] , axis = 0 ) mask = tf . concat ( [ tf . zeros_like ( inp ) , pad , tf . ones_like ( targets ) ] , axis = 0 ) features [ 'inputs' ] = concat features [ 'mask' ] = mask return features , concat dataset = dataset . map ( concat_and_add_mask ) return dataset