(dataset, training, max_length=-1, max_eval_length=-1)
return dataset
Preprocessing for LM1B: filter out targets exceeding maximum length.
Preprocessing for LM1B: filter out targets exceeding maximum length.
Preprocessing for LM1B : filter out targets exceeding maximum length .
def wmt_preprocess(dataset, training, max_length=-1, max_eval_length=-1):
  """Preprocessing for LM1B: filter out targets exceeding maximum length."""

  def train_right_length(example, target):
    l = tf.maximum(tf.shape(example['inputs'])[0], tf.shape(target)[0])
    return tf.less(l, max_length + 1)

  def eval_right_length(example, target):
    l = tf.maximum(tf.shape(example['inputs'])[0], tf.shape(target)[0])
    return tf.less(l, max_eval_length + 1)

  if max_length > 0 and training:
    dataset = dataset.filter(train_right_length)

  if max_eval_length > 0 and not training:
    dataset = dataset.filter(eval_right_length)

  return dataset
def wmt_preprocess ( dataset , training , max_length = - 1 , max_eval_length = - 1 ) : def train_right_length ( example , target ) : l = tf . maximum ( tf . shape ( example [ 'inputs' ] ) [ 0 ] , tf . shape ( target ) [ 0 ] ) return tf . less ( l , max_length + 1 ) def eval_right_length ( example , target ) : l = tf . maximum ( tf . shape ( example [ 'inputs' ] ) [ 0 ] , tf . shape ( target ) [ 0 ] ) return tf . less ( l , max_eval_length + 1 ) if max_length > 0 and training : dataset = dataset . filter ( train_right_length ) if max_eval_length > 0 and not training : dataset = dataset . filter ( eval_right_length ) return dataset