(benchmark_id)
return stream_of_inputs_targets_weights
Returns a GLUE data split prepared with the standard T5 preprocessor.
Returns a GLUE data split prepared with the standard T5 preprocessor.
Returns a GLUE data split prepared with the standard T5 preprocessor .
def _t5_glue_data_split(benchmark_id):
  """Returns a GLUE data split prepared with the standard T5 preprocessor."""
  benchmark, split = _t5_glue_benchmark_and_split(benchmark_id)
  dataset = tfds.load(name=f'glue/{benchmark}', split=split)
  processed_dataset = generic_text_dataset_preprocess_fn(
      dataset,
      spm_path=t5_data().DEFAULT_SPM_PATH,
      text_preprocess_fns=[
          lambda ds, training: t5_data().preprocessors.glue(  # pylint: disable=g-long-lambda
              ds,
              benchmark_name=benchmark,
              label_names=_GLUE_LABELS[benchmark])
      ],
      copy_pretokenized=True,
      debug_print_examples=True,
      debug_print_examples_rate=0.05)
  dataset_as_numpy = tfds.as_numpy(processed_dataset)

  def stream_of_inputs_targets_weights(generator=None):
    del generator
    while True:
      for example in dataset_as_numpy:
        input_values = example['inputs']
        target_values = example['targets']
        yield (jnp.array(input_values),
               jnp.array(target_values),
               jnp.array([1] * len(target_values)))

  return stream_of_inputs_targets_weights
def _t5_glue_data_split ( benchmark_id ) : benchmark , split = _t5_glue_benchmark_and_split ( benchmark_id ) dataset = tfds . load ( name = f'glue/{benchmark}' , split = split ) processed_dataset = generic_text_dataset_preprocess_fn ( dataset , spm_path = t5_data ( ) . DEFAULT_SPM_PATH , text_preprocess_fns = [ lambda ds , training : t5_data ( ) . preprocessors . glue ( # pylint: disable=g-long-lambda ds , benchmark_name = benchmark , label_names = _GLUE_LABELS [ benchmark ] ) ] , copy_pretokenized = True , debug_print_examples = True , debug_print_examples_rate = 0.05 ) dataset_as_numpy = tfds . as_numpy ( processed_dataset ) def stream_of_inputs_targets_weights ( generator = None ) : del generator while True : for example in dataset_as_numpy : input_values = example [ 'inputs' ] target_values = example [ 'targets' ] yield ( jnp . array ( input_values ) , jnp . array ( target_values ) , jnp . array ( [ 1 ] * len ( target_values ) ) ) return stream_of_inputs_targets_weights