(tokenizer, ascending, name, path, bucketing, batch_size,
                   train_split=None, dev_split=None, test_split=None)
Interface for creating all kinds of dataset
Interface for creating all kinds of dataset
Interface for creating all kinds of dataset
def create_dataset(tokenizer, ascending, name, path, bucketing, batch_size,
                   train_split=None, dev_split=None, test_split=None):
    ''' Interface for creating all kinds of dataset'''

    # Recognize corpus
    if name.lower() == "librispeech":
        from corpus.librispeech import LibriDataset as Dataset
    else:
        raise NotImplementedError

    # Create dataset
    if train_split is not None:
        # Training mode
        mode = 'train'
        tr_loader_bs = 1 if bucketing and (not ascending) else batch_size
        bucket_size = batch_size if bucketing and (
            not ascending) else 1  # Ascending without bucketing
        # Do not use bucketing for dev set
        dv_set = Dataset(path, dev_split, tokenizer, 1)
        tr_set = Dataset(path, train_split, tokenizer,
                         bucket_size, ascending=ascending)
        # Messages to show
        msg_list = _data_msg(name, path, train_split.__str__(), len(tr_set),
                             dev_split.__str__(), len(dv_set), batch_size, bucketing)

        return tr_set, dv_set, tr_loader_bs, batch_size, mode, msg_list
    else:
        # Testing model
        mode = 'test'
        # Do not use bucketing for dev set
        dv_set = Dataset(path, dev_split, tokenizer, 1)
        # Do not use bucketing for test set
        tt_set = Dataset(path, test_split, tokenizer, 1)
        # Messages to show
        msg_list = _data_msg(name, path, dev_split.__str__(), len(dv_set),
                             test_split.__str__(), len(tt_set), batch_size, False)
        msg_list = [m.replace('Dev', 'Test').replace(
            'Train', 'Dev') for m in msg_list]
        return dv_set, tt_set, batch_size, batch_size, mode, msg_list
def create_dataset ( tokenizer , ascending , name , path , bucketing , batch_size , train_split = None , dev_split = None , test_split = None ) : # Recognize corpus if name . lower ( ) == "librispeech" : from corpus . librispeech import LibriDataset as Dataset else : raise NotImplementedError # Create dataset if train_split is not None : # Training mode mode = 'train' tr_loader_bs = 1 if bucketing and ( not ascending ) else batch_size bucket_size = batch_size if bucketing and ( not ascending ) else 1 # Ascending without bucketing # Do not use bucketing for dev set dv_set = Dataset ( path , dev_split , tokenizer , 1 ) tr_set = Dataset ( path , train_split , tokenizer , bucket_size , ascending = ascending ) # Messages to show msg_list = _data_msg ( name , path , train_split . __str__ ( ) , len ( tr_set ) , dev_split . __str__ ( ) , len ( dv_set ) , batch_size , bucketing ) return tr_set , dv_set , tr_loader_bs , batch_size , mode , msg_list else : # Testing model mode = 'test' # Do not use bucketing for dev set dv_set = Dataset ( path , dev_split , tokenizer , 1 ) # Do not use bucketing for test set tt_set = Dataset ( path , test_split , tokenizer , 1 ) # Messages to show msg_list = _data_msg ( name , path , dev_split . __str__ ( ) , len ( dv_set ) , test_split . __str__ ( ) , len ( tt_set ) , batch_size , False ) msg_list = [ m . replace ( 'Dev' , 'Test' ) . replace ( 'Train' , 'Dev' ) for m in msg_list ] return dv_set , tt_set , batch_size , batch_size , mode , msg_list