(dataset,
                  training,
                  max_target_length=-1,
                  tokenization=None,
                  spm_path=None)
return dataset
Pre-processing function for C4 dataset.
Pre-processing function for C4 dataset.
Pre - processing function for C4 dataset .
def c4_preprocess(dataset,
                  training,
                  max_target_length=-1,
                  tokenization=None,
                  spm_path=None):
  """Pre-processing function for C4 dataset."""
  del training

  def unicode_decode_chars(features, targets):
    targets = tf.strings.unicode_decode(features['text'], 'UTF-8')
    targets = tf.cast(targets, tf.int64)
    features['targets'] = targets
    features['inputs'] = targets
    return (features, targets)

  def spc_tokenize(tokenizer, features, targets):
    del targets
    tokenized_text = tokenizer.tokenize(features['text'])
    features['targets'] = tf.cast(tokenized_text, tf.int64)
    features['inputs'] = features['targets']
    return features, features['targets']

  if tokenization == 'spc':
    spm_path = spm_path or t5_data().DEFAULT_SPM_PATH
    with tf.compat.v1.gfile.GFile(spm_path, 'rb') as f:
      spc_model = f.read()
    tokenizer = tf_text.SentencepieceTokenizer(model=spc_model)
    dataset = dataset.map(functools.partial(spc_tokenize, tokenizer))
  else:
    dataset = dataset.map(unicode_decode_chars)

  def target_right_length(_, target):
    return tf.less(tf.shape(target)[0], max_target_length + 1)

  if max_target_length > 0:
    dataset = dataset.filter(target_right_length)

  return dataset
def c4_preprocess ( dataset , training , max_target_length = - 1 , tokenization = None , spm_path = None ) : del training def unicode_decode_chars ( features , targets ) : targets = tf . strings . unicode_decode ( features [ 'text' ] , 'UTF-8' ) targets = tf . cast ( targets , tf . int64 ) features [ 'targets' ] = targets features [ 'inputs' ] = targets return ( features , targets ) def spc_tokenize ( tokenizer , features , targets ) : del targets tokenized_text = tokenizer . tokenize ( features [ 'text' ] ) features [ 'targets' ] = tf . cast ( tokenized_text , tf . int64 ) features [ 'inputs' ] = features [ 'targets' ] return features , features [ 'targets' ] if tokenization == 'spc' : spm_path = spm_path or t5_data ( ) . DEFAULT_SPM_PATH with tf . compat . v1 . gfile . GFile ( spm_path , 'rb' ) as f : spc_model = f . read ( ) tokenizer = tf_text . SentencepieceTokenizer ( model = spc_model ) dataset = dataset . map ( functools . partial ( spc_tokenize , tokenizer ) ) else : dataset = dataset . map ( unicode_decode_chars ) def target_right_length ( _ , target ) : return tf . less ( tf . shape ( target ) [ 0 ] , max_target_length + 1 ) if max_target_length > 0 : dataset = dataset . filter ( target_right_length ) return dataset