(dataset,
                          training=True,
                          spm_path=None,
                          copy_pretokenized=True,
                          sequence_length=None)
return dataset
Returns a dataset that contains 'inputs' and 'targets' from C4.
Returns a dataset that contains 'inputs' and 'targets' from C4.
Returns a dataset that contains inputs and targets from C4 .
def c4_bare_preprocess_fn(dataset,
                          training=True,
                          spm_path=None,
                          copy_pretokenized=True,
                          sequence_length=None):
  """Returns a dataset that contains 'inputs' and 'targets' from C4."""
  # Set target key to be equal to the text content.
  dataset = t5_data().preprocessors.rekey(
      dataset, key_map={
          'targets': 'text',
          'inputs': None
      })

  # Vocabulary for tokenization.
  extra_ids = 0
  vocab = t5_data().SentencePieceVocabulary(
      sentencepiece_model_file=spm_path or t5_data().DEFAULT_SPM_PATH,
      extra_ids=extra_ids)
  feature = t5_data().Feature(vocab)
  output_features = {'targets': feature, 'inputs': feature}

  # Tokenize the targets.
  keys = output_features

  def encode_string_features_fn(features):
    """Encode all specified feature that are strings and return a dictionary.

    Args:
      features: a dictionary

    Returns:
      a dictionary
    """
    ret = {}
    for k, v in features.items():
      if k in keys and v.dtype == tf.string:
        if copy_pretokenized:
          ret['%s_pretokenized' % k] = v
        v = tf.cast(output_features[k].vocabulary.encode_tf(v), tf.int64)
      ret[k] = v
    return ret

  dataset = dataset.map(
      encode_string_features_fn,
      num_parallel_calls=tf.data.experimental.AUTOTUNE)

  # Preprocess the tokens - the exact preprocessors are set via gin.
  dataset = t5_data().preprocessors.unsupervised(
      dataset, sequence_length=sequence_length, output_features=output_features)

  # Add EOS.
  dataset = add_eos_to_output_features(dataset, training)

  # Truncate and then pad the examples -- all examples have the same shape.
  dataset = truncate_dataset_on_len(dataset, training, sequence_length, True)
  dataset = pad_dataset_to_length(dataset, training, sequence_length)

  return dataset
def c4_bare_preprocess_fn ( dataset , training = True , spm_path = None , copy_pretokenized = True , sequence_length = None ) : # Set target key to be equal to the text content. dataset = t5_data ( ) . preprocessors . rekey ( dataset , key_map = { 'targets' : 'text' , 'inputs' : None } ) # Vocabulary for tokenization. extra_ids = 0 vocab = t5_data ( ) . SentencePieceVocabulary ( sentencepiece_model_file = spm_path or t5_data ( ) . DEFAULT_SPM_PATH , extra_ids = extra_ids ) feature = t5_data ( ) . Feature ( vocab ) output_features = { 'targets' : feature , 'inputs' : feature } # Tokenize the targets. keys = output_features def encode_string_features_fn ( features ) : """Encode all specified feature that are strings and return a dictionary.

    Args:
      features: a dictionary

    Returns:
      a dictionary
    """ ret = { } for k , v in features . items ( ) : if k in keys and v . dtype == tf . string : if copy_pretokenized : ret [ '%s_pretokenized' % k ] = v v = tf . cast ( output_features [ k ] . vocabulary . encode_tf ( v ) , tf . int64 ) ret [ k ] = v return ret dataset = dataset . map ( encode_string_features_fn , num_parallel_calls = tf . data . experimental . AUTOTUNE ) # Preprocess the tokens - the exact preprocessors are set via gin. dataset = t5_data ( ) . preprocessors . unsupervised ( dataset , sequence_length = sequence_length , output_features = output_features ) # Add EOS. dataset = add_eos_to_output_features ( dataset , training ) # Truncate and then pad the examples -- all examples have the same shape. dataset = truncate_dataset_on_len ( dataset , training , sequence_length , True ) dataset = pad_dataset_to_length ( dataset , training , sequence_length ) return dataset