(n_jobs, use_gpu, pin_memory, ascending, corpus, audio, text)
return tr_set, dv_set, feat_dim, tokenizer.vocab_size, tokenizer, data_msg
Prepare dataloader for training/validation
Prepare dataloader for training/validation
Prepare dataloader for training / validation
def load_dataset(n_jobs, use_gpu, pin_memory, ascending, corpus, audio, text):
    ''' Prepare dataloader for training/validation'''

    # Audio feature extractor
    audio_transform, feat_dim = create_transform(audio.copy())
    # Text tokenizer
    tokenizer = load_text_encoder(**text)
    # Dataset (in testing mode, tr_set=dv_set, dv_set=tt_set)
    tr_set, dv_set, tr_loader_bs, dv_loader_bs, mode, data_msg = create_dataset(
        tokenizer, ascending, **corpus)
    # Collect function
    collect_tr = partial(collect_audio_batch,
                         audio_transform=audio_transform, mode=mode)
    collect_dv = partial(collect_audio_batch,
                         audio_transform=audio_transform, mode='test')
    # Shuffle/drop applied to training set only
    shuffle = (mode == 'train' and not ascending)
    drop_last = shuffle
    # Create data loader
    tr_set = DataLoader(tr_set, batch_size=tr_loader_bs, shuffle=shuffle, drop_last=drop_last, collate_fn=collect_tr,
                        num_workers=n_jobs, pin_memory=use_gpu)
    dv_set = DataLoader(dv_set, batch_size=dv_loader_bs, shuffle=False, drop_last=False, collate_fn=collect_dv,
                        num_workers=n_jobs, pin_memory=pin_memory)
    # Messages to show
    data_msg.append('I/O spec.  | Audio feature = {}\t| feature dim = {}\t| Token type = {}\t| Vocab size = {}'
                    .format(audio['feat_type'], feat_dim, tokenizer.token_type, tokenizer.vocab_size))

    return tr_set, dv_set, feat_dim, tokenizer.vocab_size, tokenizer, data_msg
def load_dataset ( n_jobs , use_gpu , pin_memory , ascending , corpus , audio , text ) : # Audio feature extractor audio_transform , feat_dim = create_transform ( audio . copy ( ) ) # Text tokenizer tokenizer = load_text_encoder ( * * text ) # Dataset (in testing mode, tr_set=dv_set, dv_set=tt_set) tr_set , dv_set , tr_loader_bs , dv_loader_bs , mode , data_msg = create_dataset ( tokenizer , ascending , * * corpus ) # Collect function collect_tr = partial ( collect_audio_batch , audio_transform = audio_transform , mode = mode ) collect_dv = partial ( collect_audio_batch , audio_transform = audio_transform , mode = 'test' ) # Shuffle/drop applied to training set only shuffle = ( mode == 'train' and not ascending ) drop_last = shuffle # Create data loader tr_set = DataLoader ( tr_set , batch_size = tr_loader_bs , shuffle = shuffle , drop_last = drop_last , collate_fn = collect_tr , num_workers = n_jobs , pin_memory = use_gpu ) dv_set = DataLoader ( dv_set , batch_size = dv_loader_bs , shuffle = False , drop_last = False , collate_fn = collect_dv , num_workers = n_jobs , pin_memory = pin_memory ) # Messages to show data_msg . append ( 'I/O spec.  | Audio feature = {}\t| feature dim = {}\t| Token type = {}\t| Vocab size = {}' . format ( audio [ 'feat_type' ] , feat_dim , tokenizer . token_type , tokenizer . vocab_size ) ) return tr_set , dv_set , feat_dim , tokenizer . vocab_size , tokenizer , data_msg