(config, mode)
return {'data': minibatches, 'n_utts': utts}
make batches of metas and get dataset size
make batches of metas and get dataset size
make batches of metas and get dataset size
def get_batches(config, mode):
  ''' make batches of metas and get dataset size'''
  assert mode in (utils.TRAIN, utils.EVAL, utils.INFER)

  # read meta of json
  json_path = config['data'][mode]['paths']
  assert len(json_path) == 1
  logging.info(f"=== load json data {json_path} ===")
  logging.info(f" # learning phase: {mode}")

  #pylint: disable=invalid-name
  with open(json_path[0], 'r', encoding='utf-8') as f:
    metas_raw = json.load(f)['utts']

  # sort by utts id
  metas = OrderedDict(sorted(metas_raw.items(), key=lambda t: t[0]))

  # dataset size
  utts = len(metas.keys())
  logging.info(' # utts: ' + str(utts))

  # make batchset
  use_sortagrad = config['data']['task']['sortagrad']
  logging.info(f' # sortagrad: {use_sortagrad}')

  task = config['data']['task']['type']
  assert task in list(TASK_SET.keys())
  # using same json for asr and tts task
  if task == TASK_SET['asr']:
    src = 'src'
    tgt = 'tgt'
  elif task == TASK_SET['tts']:
    src = 'tgt'
    tgt = 'src'
  else:
    raise ValueError("task type must int : {} get : {}".format(
        list(TASK_SET.keys()), task))
  logging.info(f" # task: {task}")

  # delta config
  maxlen_src = config['data']['task'][src]['max_len']
  maxlen_tgt = config['data']['task'][tgt]['max_len']
  batch_sort_key = config['data']['task']['batch_sort_key']
  num_batches = config['data']['task']['num_batches']  # for debug
  global_batch_size = config['solver']['optimizer']['batch_size']
  batch_bins = config['data']['task']['batch']['batch_bins']
  batch_frames_in = config['data']['task']['batch']['batch_frames_in']
  batch_frames_out = config['data']['task']['batch']['batch_frames_out']
  batch_frames_inout = config['data']['task']['batch']['batch_frames_inout']
  batch_strategy = config['data']['task']['batch']['batch_strategy']

  _, ngpu = utils.gpu_device_names()
  min_batch_size = ngpu if ngpu else 1
  logging.info(f" # ngpu: {ngpu}")
  logging.info(f" # min_batch_size: {min_batch_size}")

  minibatches = make_batchset(
      task=task,
      data=metas,
      batch_size=global_batch_size,
      max_length_in=maxlen_src,
      max_length_out=maxlen_tgt,
      num_batches=num_batches,
      batch_sort_key=batch_sort_key,
      min_batch_size=min_batch_size,
      shortest_first=use_sortagrad,
      batch_bins=batch_bins,
      batch_frames_in=batch_frames_in,
      batch_frames_out=batch_frames_out,
      batch_frames_inout=batch_frames_inout,
      batch_strategy=batch_strategy)

  return {'data': minibatches, 'n_utts': utts}
def get_batches ( config , mode ) : assert mode in ( utils . TRAIN , utils . EVAL , utils . INFER ) # read meta of json json_path = config [ 'data' ] [ mode ] [ 'paths' ] assert len ( json_path ) == 1 logging . info ( f"=== load json data {json_path} ===" ) logging . info ( f" # learning phase: {mode}" ) #pylint: disable=invalid-name with open ( json_path [ 0 ] , 'r' , encoding = 'utf-8' ) as f : metas_raw = json . load ( f ) [ 'utts' ] # sort by utts id metas = OrderedDict ( sorted ( metas_raw . items ( ) , key = lambda t : t [ 0 ] ) ) # dataset size utts = len ( metas . keys ( ) ) logging . info ( ' # utts: ' + str ( utts ) ) # make batchset use_sortagrad = config [ 'data' ] [ 'task' ] [ 'sortagrad' ] logging . info ( f' # sortagrad: {use_sortagrad}' ) task = config [ 'data' ] [ 'task' ] [ 'type' ] assert task in list ( TASK_SET . keys ( ) ) # using same json for asr and tts task if task == TASK_SET [ 'asr' ] : src = 'src' tgt = 'tgt' elif task == TASK_SET [ 'tts' ] : src = 'tgt' tgt = 'src' else : raise ValueError ( "task type must int : {} get : {}" . format ( list ( TASK_SET . keys ( ) ) , task ) ) logging . info ( f" # task: {task}" ) # delta config maxlen_src = config [ 'data' ] [ 'task' ] [ src ] [ 'max_len' ] maxlen_tgt = config [ 'data' ] [ 'task' ] [ tgt ] [ 'max_len' ] batch_sort_key = config [ 'data' ] [ 'task' ] [ 'batch_sort_key' ] num_batches = config [ 'data' ] [ 'task' ] [ 'num_batches' ] # for debug global_batch_size = config [ 'solver' ] [ 'optimizer' ] [ 'batch_size' ] batch_bins = config [ 'data' ] [ 'task' ] [ 'batch' ] [ 'batch_bins' ] batch_frames_in = config [ 'data' ] [ 'task' ] [ 'batch' ] [ 'batch_frames_in' ] batch_frames_out = config [ 'data' ] [ 'task' ] [ 'batch' ] [ 'batch_frames_out' ] batch_frames_inout = config [ 'data' ] [ 'task' ] [ 'batch' ] [ 'batch_frames_inout' ] batch_strategy = config [ 'data' ] [ 'task' ] [ 'batch' ] [ 'batch_strategy' ] _ , ngpu = utils . gpu_device_names ( ) min_batch_size = ngpu if ngpu else 1 logging . info ( f" # ngpu: {ngpu}" ) logging . info ( f" # min_batch_size: {min_batch_size}" ) minibatches = make_batchset ( task = task , data = metas , batch_size = global_batch_size , max_length_in = maxlen_src , max_length_out = maxlen_tgt , num_batches = num_batches , batch_sort_key = batch_sort_key , min_batch_size = min_batch_size , shortest_first = use_sortagrad , batch_bins = batch_bins , batch_frames_in = batch_frames_in , batch_frames_out = batch_frames_out , batch_frames_inout = batch_frames_inout , batch_strategy = batch_strategy ) return { 'data' : minibatches , 'n_utts' : utts }