(dataset,
                    training,
                    max_target_length=-1,
                    max_eval_target_length=-1)
return dataset
Preprocessing for LM1B: filter out targets exceeding maximum length.
Preprocessing for LM1B: filter out targets exceeding maximum length.
Preprocessing for LM1B : filter out targets exceeding maximum length .
def lm1b_preprocess(dataset,
                    training,
                    max_target_length=-1,
                    max_eval_target_length=-1):
  """Preprocessing for LM1B: filter out targets exceeding maximum length."""

  def target_right_length(_, target):
    return tf.less(tf.shape(target)[0], max_target_length + 1)

  def eval_target_right_length(_, target):
    return tf.less(tf.shape(target)[0], max_eval_target_length + 1)

  if max_target_length > 0 and training:
    dataset = dataset.filter(target_right_length)

  if max_eval_target_length > 0 and not training:
    dataset = dataset.filter(eval_target_right_length)

  return dataset
def lm1b_preprocess ( dataset , training , max_target_length = - 1 , max_eval_target_length = - 1 ) : def target_right_length ( _ , target ) : return tf . less ( tf . shape ( target ) [ 0 ] , max_target_length + 1 ) def eval_target_right_length ( _ , target ) : return tf . less ( tf . shape ( target ) [ 0 ] , max_eval_target_length + 1 ) if max_target_length > 0 and training : dataset = dataset . filter ( target_right_length ) if max_eval_target_length > 0 and not training : dataset = dataset . filter ( eval_target_right_length ) return dataset