(tokenizer, train_split, dev_split, name, path, bucketing, batch_size)
return tr_set, dv_set, tr_loader_bs, batch_size, msg_list
Interface for creating all kinds of text dataset
Interface for creating all kinds of text dataset
Interface for creating all kinds of text dataset
def create_textset(tokenizer, train_split, dev_split, name, path, bucketing, batch_size):
    ''' Interface for creating all kinds of text dataset'''
    msg_list = []

    # Recognize corpus
    if name.lower() == "librispeech":
        from corpus.librispeech import LibriTextDataset as Dataset
    else:
        raise NotImplementedError

    # Create dataset
    bucket_size = batch_size if bucketing else 1
    tr_loader_bs = 1 if bucketing else batch_size
    # Do not use bucketing for dev set
    dv_set = Dataset(path, dev_split, tokenizer, 1)
    tr_set = Dataset(path, train_split, tokenizer, bucket_size)

    # Messages to show
    msg_list = _data_msg(name, path, train_split.__str__(), len(tr_set),
                         dev_split.__str__(), len(dv_set), batch_size, bucketing)

    return tr_set, dv_set, tr_loader_bs, batch_size, msg_list
def create_textset ( tokenizer , train_split , dev_split , name , path , bucketing , batch_size ) : msg_list = [ ] # Recognize corpus if name . lower ( ) == "librispeech" : from corpus . librispeech import LibriTextDataset as Dataset else : raise NotImplementedError # Create dataset bucket_size = batch_size if bucketing else 1 tr_loader_bs = 1 if bucketing else batch_size # Do not use bucketing for dev set dv_set = Dataset ( path , dev_split , tokenizer , 1 ) tr_set = Dataset ( path , train_split , tokenizer , bucket_size ) # Messages to show msg_list = _data_msg ( name , path , train_split . __str__ ( ) , len ( tr_set ) , dev_split . __str__ ( ) , len ( dv_set ) , batch_size , bucketing ) return tr_set , dv_set , tr_loader_bs , batch_size , msg_list