(dataset,
                                       training=True,
                                       text_preprocess_fns=None,
                                       token_preprocess_fns=None,
                                       spm_path=None,
                                       copy_pretokenized=False,
                                       debug_print_examples=False,
                                       debug_print_examples_rate=0.01)
return dataset
Pre-processes, tokenizes and post-processes a `tf.data.Dataset`.

  Args:
    dataset: `tf.data.Dataset` to process.
    training: boolean, set to True if training, False otherwise.
    text_preprocess_fns: None or list of callables: `tf.data.Dataset`, bool ->
      `tf.data.Dataset` this operates before tokenization. Typically used to
      select which fields we want to learn over or change something into "text
      to text" form.
    token_preprocess_fns: None or list of callables: `tf.data.Dataset`, bool ->
      `tf.data.Dataset`, this operates after tokenization. Since this can view
      the tokenized fields, this can be used to filter on length etc.
    spm_path: None or str, path to a sentencepiece model to use for tokenization
      by default uses the 32k vocabulary from T5.
    copy_pretokenized: bool, if True retains the original fields after
      tokenization.
    debug_print_examples: bool, if True this prints examples to the logging
      stream for inspection, both before and after tokenization.
    debug_print_examples_rate: float, [0, 1.0], on average this fraction of
      dataset examples will be printed out in each phase i.e. pre and post
      tokenization.

  Returns:
    a `tf.data.Dataset` with all the preprocessing and tokenization performed.
Pre-processes, tokenizes and post-processes a `tf.data.Dataset`.
Pre - processes tokenizes and post - processes a tf . data . Dataset .
def generic_text_dataset_preprocess_fn(dataset,
                                       training=True,
                                       text_preprocess_fns=None,
                                       token_preprocess_fns=None,
                                       spm_path=None,
                                       copy_pretokenized=False,
                                       debug_print_examples=False,
                                       debug_print_examples_rate=0.01):
  """Pre-processes, tokenizes and post-processes a `tf.data.Dataset`.

  Args:
    dataset: `tf.data.Dataset` to process.
    training: boolean, set to True if training, False otherwise.
    text_preprocess_fns: None or list of callables: `tf.data.Dataset`, bool ->
      `tf.data.Dataset` this operates before tokenization. Typically used to
      select which fields we want to learn over or change something into "text
      to text" form.
    token_preprocess_fns: None or list of callables: `tf.data.Dataset`, bool ->
      `tf.data.Dataset`, this operates after tokenization. Since this can view
      the tokenized fields, this can be used to filter on length etc.
    spm_path: None or str, path to a sentencepiece model to use for tokenization
      by default uses the 32k vocabulary from T5.
    copy_pretokenized: bool, if True retains the original fields after
      tokenization.
    debug_print_examples: bool, if True this prints examples to the logging
      stream for inspection, both before and after tokenization.
    debug_print_examples_rate: float, [0, 1.0], on average this fraction of
      dataset examples will be printed out in each phase i.e. pre and post
      tokenization.

  Returns:
    a `tf.data.Dataset` with all the preprocessing and tokenization performed.
  """

  # The assumption is that `text_preprocess_fns` finally gives us a dataset
  # which has `inputs` and `targets`.
  if text_preprocess_fns is not None:
    for text_preprocess_fn in text_preprocess_fns:
      dataset = text_preprocess_fn(dataset, training)

  # Print debugging examples if needed before tokenization.
  if debug_print_examples:

    def print_examples(x):
      if np.random.uniform() < debug_print_examples_rate:
        tf.print(x, output_stream=logging.info)
      return x

    dataset = dataset.map(print_examples)

  # Vocabulary for tokenization.
  extra_ids = 0
  vocab = t5_data().SentencePieceVocabulary(
      sentencepiece_model_file=spm_path or t5_data().DEFAULT_SPM_PATH,
      extra_ids=extra_ids)
  feature = t5_data().Feature(vocab)
  output_features = {'targets': feature, 'inputs': feature}

  # Tokenize the inputs and targets.
  dataset = t5_data().preprocessors.tokenize(
      dataset, output_features, copy_pretokenized=copy_pretokenized)

  # Apply the token-preprocessors.
  if token_preprocess_fns is not None:
    for token_preprocess_fn in token_preprocess_fns:
      dataset = token_preprocess_fn(dataset, training)

  if debug_print_examples:

    def print_examples_and_shapes(x):
      if np.random.uniform() < debug_print_examples_rate:
        tf.print(
            {
                'inputs_shape': tf.size(x['inputs']),
                'targets_shape': tf.size(x['targets']),
                'inputs': x['inputs'],
                'targets': x['targets'],
            },
            output_stream=logging.info)
      return x

    dataset = dataset.map(print_examples_and_shapes)

  return dataset
def generic_text_dataset_preprocess_fn ( dataset , training = True , text_preprocess_fns = None , token_preprocess_fns = None , spm_path = None , copy_pretokenized = False , debug_print_examples = False , debug_print_examples_rate = 0.01 ) : # The assumption is that `text_preprocess_fns` finally gives us a dataset # which has `inputs` and `targets`. if text_preprocess_fns is not None : for text_preprocess_fn in text_preprocess_fns : dataset = text_preprocess_fn ( dataset , training ) # Print debugging examples if needed before tokenization. if debug_print_examples : def print_examples ( x ) : if np . random . uniform ( ) < debug_print_examples_rate : tf . print ( x , output_stream = logging . info ) return x dataset = dataset . map ( print_examples ) # Vocabulary for tokenization. extra_ids = 0 vocab = t5_data ( ) . SentencePieceVocabulary ( sentencepiece_model_file = spm_path or t5_data ( ) . DEFAULT_SPM_PATH , extra_ids = extra_ids ) feature = t5_data ( ) . Feature ( vocab ) output_features = { 'targets' : feature , 'inputs' : feature } # Tokenize the inputs and targets. dataset = t5_data ( ) . preprocessors . tokenize ( dataset , output_features , copy_pretokenized = copy_pretokenized ) # Apply the token-preprocessors. if token_preprocess_fns is not None : for token_preprocess_fn in token_preprocess_fns : dataset = token_preprocess_fn ( dataset , training ) if debug_print_examples : def print_examples_and_shapes ( x ) : if np . random . uniform ( ) < debug_print_examples_rate : tf . print ( { 'inputs_shape' : tf . size ( x [ 'inputs' ] ) , 'targets_shape' : tf . size ( x [ 'targets' ] ) , 'inputs' : x [ 'inputs' ] , 'targets' : x [ 'targets' ] , } , output_stream = logging . info ) return x dataset = dataset . map ( print_examples_and_shapes ) return dataset