(compute, dataset, sample_size=None, batch_size=10,
        cachefile=None, **kwargs)
Computes a concatenated tensor for data batches that can be
    computed from a dataset.  The compute function should return
    a tensor that should be concatenated to the others along its
    first dimension.
Computes a concatenated tensor for data batches that can be
    computed from a dataset.  The compute function should return
    a tensor that should be concatenated to the others along its
    first dimension.
Computes a concatenated tensor for data batches that can be computed from a dataset . The compute function should return a tensor that should be concatenated to the others along its first dimension .
def tally_cat(compute, dataset, sample_size=None, batch_size=10,
        cachefile=None, **kwargs):
    '''
    Computes a concatenated tensor for data batches that can be
    computed from a dataset.  The compute function should return
    a tensor that should be concatenated to the others along its
    first dimension.
    '''
    with torch.no_grad():
        args = dict(sample_size=sample_size)
        cached_state = load_cached_state(cachefile, args)
        if cached_state is not None:
            return torch.from_numpy(cached_state['data'])
        loader = make_loader(dataset, sample_size, batch_size, **kwargs)
        result = []
        for batch in pbar(loader):
            result.append(call_compute(compute, batch).cpu())
        data = torch.cat(result)
        class SavedTensor:
            def __init__(self, data):
                self.data = data
            def state_dict(self):
                return dict(data=self.data.numpy())
        save_cached_state(cachefile, SavedTensor(data), args)
        return data
def tally_cat ( compute , dataset , sample_size = None , batch_size = 10 , cachefile = None , * * kwargs ) : with torch . no_grad ( ) : args = dict ( sample_size = sample_size ) cached_state = load_cached_state ( cachefile , args ) if cached_state is not None : return torch . from_numpy ( cached_state [ 'data' ] ) loader = make_loader ( dataset , sample_size , batch_size , * * kwargs ) result = [ ] for batch in pbar ( loader ) : result . append ( call_compute ( compute , batch ) . cpu ( ) ) data = torch . cat ( result ) class SavedTensor : def __init__ ( self , data ) : self . data = data def state_dict ( self ) : return dict ( data = self . data . numpy ( ) ) save_cached_state ( cachefile , SavedTensor ( data ) , args ) return data