text: from tqdm import tqdm
  from pathlib import Path
  from os.path import join,getsize
  from joblib import Parallel, delayed
  from torch.utils.data import Dataset
  
  OFFICIAL_TXT_SRC  = ['librispeech-lm-norm.txt']  # Additional (official) text src provided
  REMOVE_TOP_N_TXT  = 5000000                      # Remove longest N sentence in librispeech-lm-norm.txt
  READ_FILE_THREADS = 16                           # Default num. of threads used for loading LibriSpeech
  
  def read_text(file):
      '''Get transcription of target wave file, 
         it's somewhat redundant for accessing each txt multiplt times,
         but it works fine with multi-thread'''
      src_file = '-'.join(file.split('-')[:-1])+'.trans.txt'
      idx = file.split('/')[-1].split('.')[0]
  
      with open(src_file,'r') as fp:
          for line in fp:
              if idx == line.split(' ')[0]:
                  return line[:-1].split(' ',1)[1]
  
  class LibriDataset(Dataset):
      def __init__(self, path, split, tokenizer, bucket_size=1, 
              ascending=False, read_audio=False, sort_by_text=False):
          # Setup
          self.path = path
          self.bucket_size = bucket_size
  
          # List all wave files
          file_list = []
          for s in split:
              if s[0] == 't' or s[0] == 'd':
                  file_list += list(Path(join(path,s)).rglob("*.flac"))
          assert len(file_list)>0, "No data found @ {}".format(path)
          
          # Read text
          text = Parallel(n_jobs=READ_FILE_THREADS)(delayed(read_text)(str(f)) for f in file_list)
          # text = Parallel(n_jobs=-1)(delayed(tokenizer.encode)(txt) for txt in text)
          text = [tokenizer.encode(txt) for txt in text]
          
          # Sort dataset by text length
          # file_len = Parallel(n_jobs=READ_FILE_THREADS)(delayed(getsize)(f) for f in file_list) 
          if sort_by_text == False:
              file_len = Parallel(n_jobs=READ_FILE_THREADS)(delayed(getsize)(f) for f in file_list)
          else:
              file_len = [len(txt) for txt in text]
          
          self.file_list, self.text = zip(*[(str(f_name),txt) \
                  for time_len,f_name,txt in sorted(zip(file_len,file_list,text), reverse=not ascending, key=lambda x:x[0])])
          if read_audio:
              from src.audio import ReadAudio
              audio_reader = ReadAudio(16000)##
              self.file_list = [audio_reader(str(f)) for i, f in tqdm(enumerate(self.file_list))]
  
          print('[INFO] LibriSpeech', split[-1], 'set :',len(self.file_list),'audio files found')
  
      def __getitem__(self,index):
          if self.bucket_size>1:
              # Return a bucket
              index = min(len(self.file_list)-self.bucket_size,index)
              return [(f_path, txt) for f_path,txt in \
                       zip(self.file_list[index:index+self.bucket_size], self.text[index:index+self.bucket_size])]
          else:
              return self.file_list[index], self.text[index]
  
      def __len__(self):
          return len(self.file_list)
  
  class LibriTextDataset(Dataset):
      def __init__(self, path, split, tokenizer, bucket_size):
          # Setup
          self.path = path
          self.bucket_size = bucket_size
          self.encode_on_fly = False
          read_txt_src = []
  
          # List all wave files
          file_list, all_sent = [],[]
  
          for s in split:
              if s in OFFICIAL_TXT_SRC:
                  self.encode_on_fly = True
                  with open(join(path,s),'r') as f:
                      all_sent += f.readlines()
              file_list += list(Path(join(path,s)).rglob("*.flac"))
          assert (len(file_list)>0) or (len(all_sent)>0), "No data found @ {}".format(path)
          
          # Read text
          text = Parallel(n_jobs=READ_FILE_THREADS)(delayed(read_text)(str(f)) for f in file_list)
          all_sent.extend(text)
          del text
  
          # Encode text
          if self.encode_on_fly:
              self.tokenizer = tokenizer
              self.text = [sent for sent in all_sent]
          else:
              self.text = [tokenizer.encode(txt) for txt in tqdm(all_sent)]
          del all_sent
  
          # Read file size and sort dataset by file size (Note: feature len. may be different)
          self.text = sorted(self.text, reverse=True, key=lambda x:len(x))
          if self.encode_on_fly:
              del self.text[:REMOVE_TOP_N_TXT]
  
      def __getitem__(self,index):
          if self.bucket_size>1:
              index = min(len(self.text)-self.bucket_size,index)
              if self.encode_on_fly:
                  for i in range(index,index+self.bucket_size):
                      if type(self.text[i]) is str:
                          self.text[i] = self.tokenizer.encode(self.text[i])
              # Return a bucket
              return self.text[index:index+self.bucket_size]
          else:
              if self.encode_on_fly and type(self.text[index]) is str:
                  self.text[index] = self.tokenizer.encode(self.text[index])
              return self.text[index]
  
      def __len__(self):
          return len(self.text)
  