(args, tokenizer)
return train_loader, valid_loader, train_sampler, valid_sampler, datasets['train_num_words'], datasets['valid_num_words']
Prepare the dataloaders for training and evaluation
Prepare the dataloaders for training and evaluation
Prepare the dataloaders for training and evaluation
def get_data_loaders(args, tokenizer):
    """ Prepare the dataloaders for training and evaluation """
    datasets = get_and_tokenize_dataset(tokenizer, args.dataset_path, args.dataset_cache)

    logger.info("Convert to Tensor and reshape in blocks of the transformer's input length")
    for split_name in ['train', 'valid']:
        tensor = torch.tensor(datasets[split_name], dtype=torch.long)
        num_sequences = (tensor.size(0) // args.num_max_positions) * args.num_max_positions
        datasets[split_name] = tensor.narrow(0, 0, num_sequences).view(-1, args.num_max_positions)

    logger.info("Build train and validation dataloaders")
    train_sampler = torch.utils.data.distributed.DistributedSampler(datasets['train']) if args.distributed else None
    valid_sampler = torch.utils.data.distributed.DistributedSampler(datasets['valid']) if args.distributed else None
    train_loader = DataLoader(datasets['train'], sampler=train_sampler, batch_size=args.train_batch_size, shuffle=(not args.distributed))
    valid_loader = DataLoader(datasets['valid'], sampler=valid_sampler, batch_size=args.valid_batch_size, shuffle=False)

    logger.info("Train dataset (Batch, Seq length): {}".format(datasets['train'].shape))
    logger.info("Valid dataset (Batch, Seq length): {}".format(datasets['valid'].shape))
    return train_loader, valid_loader, train_sampler, valid_sampler, datasets['train_num_words'], datasets['valid_num_words']
def get_data_loaders ( args , tokenizer ) : datasets = get_and_tokenize_dataset ( tokenizer , args . dataset_path , args . dataset_cache ) logger . info ( "Convert to Tensor and reshape in blocks of the transformer's input length" ) for split_name in [ 'train' , 'valid' ] : tensor = torch . tensor ( datasets [ split_name ] , dtype = torch . long ) num_sequences = ( tensor . size ( 0 ) // args . num_max_positions ) * args . num_max_positions datasets [ split_name ] = tensor . narrow ( 0 , 0 , num_sequences ) . view ( - 1 , args . num_max_positions ) logger . info ( "Build train and validation dataloaders" ) train_sampler = torch . utils . data . distributed . DistributedSampler ( datasets [ 'train' ] ) if args . distributed else None valid_sampler = torch . utils . data . distributed . DistributedSampler ( datasets [ 'valid' ] ) if args . distributed else None train_loader = DataLoader ( datasets [ 'train' ] , sampler = train_sampler , batch_size = args . train_batch_size , shuffle = ( not args . distributed ) ) valid_loader = DataLoader ( datasets [ 'valid' ] , sampler = valid_sampler , batch_size = args . valid_batch_size , shuffle = False ) logger . info ( "Train dataset (Batch, Seq length): {}" . format ( datasets [ 'train' ] . shape ) ) logger . info ( "Valid dataset (Batch, Seq length): {}" . format ( datasets [ 'valid' ] . shape ) ) return train_loader , valid_loader , train_sampler , valid_sampler , datasets [ 'train_num_words' ] , datasets [ 'valid_num_words' ]