(args, tokenizer, max_length, clf_token)
return train_loader, valid_loader, train_sampler, valid_sampler
Prepare the dataloaders for training and evaluation.
        Add a classification token at the end of each sample if needed.
Prepare the dataloaders for training and evaluation.
        Add a classification token at the end of each sample if needed.
Prepare the dataloaders for training and evaluation . Add a classification token at the end of each sample if needed .
def get_data_loaders(args, tokenizer, max_length, clf_token):
    """ Prepare the dataloaders for training and evaluation.
        Add a classification token at the end of each sample if needed. """
    datasets = get_and_tokenize_dataset(tokenizer, args.dataset_path, args.dataset_cache, with_labels=True)

    logger.info("Convert to Tensor, pad and trim to trim_length")
    for split_name in ['train', 'valid']:
        datasets[split_name] = [x[:max_length-1] + [clf_token] for x in datasets[split_name]]  # trim dataset
        datasets[split_name] = pad_dataset(datasets[split_name])  # pad dataset
        tensor = torch.tensor(datasets[split_name], dtype=torch.long)
        labels = torch.tensor(datasets[split_name + '_labels'], dtype=torch.long)
        datasets[split_name] = TensorDataset(tensor, labels)

    logger.info("Build train and validation dataloaders")
    train_sampler = torch.utils.data.distributed.DistributedSampler(datasets['train']) if args.distributed else None
    valid_sampler = torch.utils.data.distributed.DistributedSampler(datasets['valid']) if args.distributed else None
    train_loader = DataLoader(datasets['train'], sampler=train_sampler, batch_size=args.train_batch_size, shuffle=(not args.distributed))
    valid_loader = DataLoader(datasets['valid'], sampler=valid_sampler, batch_size=args.valid_batch_size, shuffle=False)

    logger.info("Train dataset (Batch, Seq length): {}".format(datasets['train'].tensors[0].shape))
    logger.info("Valid dataset (Batch, Seq length): {}".format(datasets['valid'].tensors[0].shape))
    return train_loader, valid_loader, train_sampler, valid_sampler
def get_data_loaders ( args , tokenizer , max_length , clf_token ) : datasets = get_and_tokenize_dataset ( tokenizer , args . dataset_path , args . dataset_cache , with_labels = True ) logger . info ( "Convert to Tensor, pad and trim to trim_length" ) for split_name in [ 'train' , 'valid' ] : datasets [ split_name ] = [ x [ : max_length - 1 ] + [ clf_token ] for x in datasets [ split_name ] ] # trim dataset datasets [ split_name ] = pad_dataset ( datasets [ split_name ] ) # pad dataset tensor = torch . tensor ( datasets [ split_name ] , dtype = torch . long ) labels = torch . tensor ( datasets [ split_name + '_labels' ] , dtype = torch . long ) datasets [ split_name ] = TensorDataset ( tensor , labels ) logger . info ( "Build train and validation dataloaders" ) train_sampler = torch . utils . data . distributed . DistributedSampler ( datasets [ 'train' ] ) if args . distributed else None valid_sampler = torch . utils . data . distributed . DistributedSampler ( datasets [ 'valid' ] ) if args . distributed else None train_loader = DataLoader ( datasets [ 'train' ] , sampler = train_sampler , batch_size = args . train_batch_size , shuffle = ( not args . distributed ) ) valid_loader = DataLoader ( datasets [ 'valid' ] , sampler = valid_sampler , batch_size = args . valid_batch_size , shuffle = False ) logger . info ( "Train dataset (Batch, Seq length): {}" . format ( datasets [ 'train' ] . tensors [ 0 ] . shape ) ) logger . info ( "Valid dataset (Batch, Seq length): {}" . format ( datasets [ 'valid' ] . tensors [ 0 ] . shape ) ) return train_loader , valid_loader , train_sampler , valid_sampler