(self, build, split, shuffle_files)
return dataset, num_examples
One of the default setting to build dataset.
One of the default setting to build dataset.
One of the default setting to build dataset .
def _split_train_98_1_1(self, build, split, shuffle_files):
    """One of the default setting to build dataset."""
    # Those large pretraining datasets have a single dataset and do not provide
    # train/validation/test splits. We split the dataset 98/01/01.
    if self.s3_enabled:
      split_patterns = {
          "train": "train[:98%]",
          "validation": "train[98%:99%]",
          "test": "train[99%:]"
      }
    else:
      split_patterns = {
          "train": tfds.Split.TRAIN.subsplit(tfds.percent[:98]),
          "validation": tfds.Split.TRAIN.subsplit(tfds.percent[98:99]),
          "test": tfds.Split.TRAIN.subsplit(tfds.percent[99:]),
      }
    dataset = tfds.load(
        self.override_build(build),
        as_supervised=self.is_supervised,
        split=split_patterns[split],
        shuffle_files=shuffle_files,
        data_dir=self.data_dir)
    if self.num_examples is None:
      raise ValueError("Must set valid num examples.")
    num_examples = int(self.num_examples * (0.98 if split == "train" else 0.01))
    return dataset, num_examples
def _split_train_98_1_1 ( self , build , split , shuffle_files ) : # Those large pretraining datasets have a single dataset and do not provide # train/validation/test splits. We split the dataset 98/01/01. if self . s3_enabled : split_patterns = { "train" : "train[:98%]" , "validation" : "train[98%:99%]" , "test" : "train[99%:]" } else : split_patterns = { "train" : tfds . Split . TRAIN . subsplit ( tfds . percent [ : 98 ] ) , "validation" : tfds . Split . TRAIN . subsplit ( tfds . percent [ 98 : 99 ] ) , "test" : tfds . Split . TRAIN . subsplit ( tfds . percent [ 99 : ] ) , } dataset = tfds . load ( self . override_build ( build ) , as_supervised = self . is_supervised , split = split_patterns [ split ] , shuffle_files = shuffle_files , data_dir = self . data_dir ) if self . num_examples is None : raise ValueError ( "Must set valid num examples." ) num_examples = int ( self . num_examples * ( 0.98 if split == "train" else 0.01 ) ) return dataset , num_examples