(
    config,
    tf_file_reader=tf.data.TFRecordDataset,
    is_training=False,
    cache_dataset=True)
return dataset
Get input tensors from dataset for training or evaluation.

  Args:
    config: A Config object containing dataset information.
    tf_file_reader: The tf.data.Dataset class to use for reading files.
    is_training: Whether or not the dataset is used in training. Determines
      whether dataset is shuffled and repeated, etc.
    cache_dataset: Whether to cache the dataset in memory for improved
      performance.

  Returns:
    A tf.data.Dataset containing input, output, control, and length tensors.

  Raises:
    ValueError: If no files match examples path.
Get input tensors from dataset for training or evaluation.
Get input tensors from dataset for training or evaluation .
def get_dataset(
    config,
    tf_file_reader=tf.data.TFRecordDataset,
    is_training=False,
    cache_dataset=True):
  """Get input tensors from dataset for training or evaluation.

  Args:
    config: A Config object containing dataset information.
    tf_file_reader: The tf.data.Dataset class to use for reading files.
    is_training: Whether or not the dataset is used in training. Determines
      whether dataset is shuffled and repeated, etc.
    cache_dataset: Whether to cache the dataset in memory for improved
      performance.

  Returns:
    A tf.data.Dataset containing input, output, control, and length tensors.

  Raises:
    ValueError: If no files match examples path.
  """
  batch_size = config.hparams.batch_size
  examples_path = (
      config.train_examples_path if is_training else config.eval_examples_path)
  note_sequence_augmenter = (
      config.note_sequence_augmenter if is_training else None)
  data_converter = config.data_converter
  data_converter.set_mode('train' if is_training else 'eval')

  if examples_path:
    tf.logging.info('Reading examples from file: %s', examples_path)
    num_files = len(tf.gfile.Glob(examples_path))
    if not num_files:
      raise ValueError(
          'No files were found matching examples path: %s' %  examples_path)
    files = tf.data.Dataset.list_files(examples_path)
    dataset = files.interleave(
        tf_file_reader,
        cycle_length=tf.data.experimental.AUTOTUNE,
        num_parallel_calls=tf.data.experimental.AUTOTUNE)
    # Use instead of `deterministic` kwarg for backward compatibility.
    options = tf.data.Options()
    options.experimental_deterministic = not is_training
    dataset = dataset.with_options(options)
  elif config.tfds_name:
    tf.logging.info('Reading examples from TFDS: %s', config.tfds_name)
    dataset = tfds.load(
        config.tfds_name,
        split=tfds.Split.TRAIN if is_training else tfds.Split.VALIDATION,
        shuffle_files=is_training,
        try_gcs=True)
    def _tf_midi_to_note_sequence(ex):
      return tf.py_function(
          lambda x:  # pylint:disable=g-long-lambda
          [note_seq.midi_to_note_sequence(x.numpy()).SerializeToString()],
          inp=[ex['midi']],
          Tout=tf.string,
          name='midi_to_note_sequence')
    dataset = dataset.map(
        _tf_midi_to_note_sequence,
        num_parallel_calls=tf.data.experimental.AUTOTUNE)
  else:
    raise ValueError(
        'One of `config.examples_path` or `config.tfds_name` must be defined.')

  def _remove_pad_fn(padded_seq_1, padded_seq_2, padded_seq_3, length):
    if length.shape.ndims == 0:
      return (padded_seq_1[0:length], padded_seq_2[0:length],
              padded_seq_3[0:length], length)
    else:
      # Don't remove padding for hierarchical examples.
      return padded_seq_1, padded_seq_2, padded_seq_3, length

  if note_sequence_augmenter is not None:
    dataset = dataset.map(
        note_sequence_augmenter.tf_augment,
        num_parallel_calls=tf.data.experimental.AUTOTUNE)

  dataset = dataset.map(
      tf.autograph.experimental.do_not_convert(
          functools.partial(convert_to_tensors_op, converter=data_converter)),
      num_parallel_calls=tf.data.experimental.AUTOTUNE)
  dataset = dataset.unbatch()
  dataset = dataset.map(
      _remove_pad_fn, num_parallel_calls=tf.data.experimental.AUTOTUNE)
  if cache_dataset:
    dataset = dataset.cache()
  if is_training:
    dataset = dataset.shuffle(buffer_size=10 * batch_size).repeat()

  dataset = dataset.padded_batch(
      batch_size,
      tf.data.get_output_shapes(dataset),
      drop_remainder=True).prefetch(tf.data.experimental.AUTOTUNE)

  return dataset
def get_dataset ( config , tf_file_reader = tf . data . TFRecordDataset , is_training = False , cache_dataset = True ) : batch_size = config . hparams . batch_size examples_path = ( config . train_examples_path if is_training else config . eval_examples_path ) note_sequence_augmenter = ( config . note_sequence_augmenter if is_training else None ) data_converter = config . data_converter data_converter . set_mode ( 'train' if is_training else 'eval' ) if examples_path : tf . logging . info ( 'Reading examples from file: %s' , examples_path ) num_files = len ( tf . gfile . Glob ( examples_path ) ) if not num_files : raise ValueError ( 'No files were found matching examples path: %s' % examples_path ) files = tf . data . Dataset . list_files ( examples_path ) dataset = files . interleave ( tf_file_reader , cycle_length = tf . data . experimental . AUTOTUNE , num_parallel_calls = tf . data . experimental . AUTOTUNE ) # Use instead of `deterministic` kwarg for backward compatibility. options = tf . data . Options ( ) options . experimental_deterministic = not is_training dataset = dataset . with_options ( options ) elif config . tfds_name : tf . logging . info ( 'Reading examples from TFDS: %s' , config . tfds_name ) dataset = tfds . load ( config . tfds_name , split = tfds . Split . TRAIN if is_training else tfds . Split . VALIDATION , shuffle_files = is_training , try_gcs = True ) def _tf_midi_to_note_sequence ( ex ) : return tf . py_function ( lambda x : # pylint:disable=g-long-lambda [ note_seq . midi_to_note_sequence ( x . numpy ( ) ) . SerializeToString ( ) ] , inp = [ ex [ 'midi' ] ] , Tout = tf . string , name = 'midi_to_note_sequence' ) dataset = dataset . map ( _tf_midi_to_note_sequence , num_parallel_calls = tf . data . experimental . AUTOTUNE ) else : raise ValueError ( 'One of `config.examples_path` or `config.tfds_name` must be defined.' ) def _remove_pad_fn ( padded_seq_1 , padded_seq_2 , padded_seq_3 , length ) : if length . shape . ndims == 0 : return ( padded_seq_1 [ 0 : length ] , padded_seq_2 [ 0 : length ] , padded_seq_3 [ 0 : length ] , length ) else : # Don't remove padding for hierarchical examples. return padded_seq_1 , padded_seq_2 , padded_seq_3 , length if note_sequence_augmenter is not None : dataset = dataset . map ( note_sequence_augmenter . tf_augment , num_parallel_calls = tf . data . experimental . AUTOTUNE ) dataset = dataset . map ( tf . autograph . experimental . do_not_convert ( functools . partial ( convert_to_tensors_op , converter = data_converter ) ) , num_parallel_calls = tf . data . experimental . AUTOTUNE ) dataset = dataset . unbatch ( ) dataset = dataset . map ( _remove_pad_fn , num_parallel_calls = tf . data . experimental . AUTOTUNE ) if cache_dataset : dataset = dataset . cache ( ) if is_training : dataset = dataset . shuffle ( buffer_size = 10 * batch_size ) . repeat ( ) dataset = dataset . padded_batch ( batch_size , tf . data . get_output_shapes ( dataset ) , drop_remainder = True ) . prefetch ( tf . data . experimental . AUTOTUNE ) return dataset