(compute, dataset,
        sample_size=None, batch_size=1, gpu_cache=64, r=1024,
        cachefile=None, **kwargs)
Computes conditional quantile sketches for a large data sample that
    can be computed from a dataset.  The compute function should return a
    sequence of sample batch tuples (condition, (sample, unit)-tensor),
    one for each condition relevant to the batch.
Computes conditional quantile sketches for a large data sample that
    can be computed from a dataset.  The compute function should return a
    sequence of sample batch tuples (condition, (sample, unit)-tensor),
    one for each condition relevant to the batch.
Computes conditional quantile sketches for a large data sample that can be computed from a dataset . The compute function should return a sequence of sample batch tuples ( condition ( sample unit ) - tensor ) one for each condition relevant to the batch .
def tally_conditional_quantile(compute, dataset,
        sample_size=None, batch_size=1, gpu_cache=64, r=1024,
        cachefile=None, **kwargs):
    '''
    Computes conditional quantile sketches for a large data sample that
    can be computed from a dataset.  The compute function should return a
    sequence of sample batch tuples (condition, (sample, unit)-tensor),
    one for each condition relevant to the batch.
    '''
    with torch.no_grad():
        args = dict(sample_size=sample_size, r=r)
        cached_state = load_cached_state(cachefile, args)
        if cached_state is not None:
            return runningstats.RunningConditionalQuantile(state=cached_state)
        loader = make_loader(dataset, sample_size, batch_size, **kwargs)
        cq = runningstats.RunningConditionalQuantile(r=r)
        most_common_conditions = set()
        for i, batch in enumerate(pbar(loader)):
            sample_set = call_compute(compute, batch)
            for cond, sample in sample_set:
                # Move uncommon conditional data to the cpu before collating.
                if cond not in most_common_conditions:
                    sample = sample.cpu()
                cq.add(cond, sample)
            # Move uncommon conditions off the GPU.
            if i and not i & (i - 1):  # if i is a power of 2:
                common_conditions = set(cq.most_common_conditions(gpu_cache))
                cq.to_('cpu', [k for k in cq.keys()
                        if k not in common_conditions])
        # At the end, move all to the CPU
        cq.to_('cpu')
        save_cached_state(cachefile, cq, args)
        return cq
def tally_conditional_quantile ( compute , dataset , sample_size = None , batch_size = 1 , gpu_cache = 64 , r = 1024 , cachefile = None , * * kwargs ) : with torch . no_grad ( ) : args = dict ( sample_size = sample_size , r = r ) cached_state = load_cached_state ( cachefile , args ) if cached_state is not None : return runningstats . RunningConditionalQuantile ( state = cached_state ) loader = make_loader ( dataset , sample_size , batch_size , * * kwargs ) cq = runningstats . RunningConditionalQuantile ( r = r ) most_common_conditions = set ( ) for i , batch in enumerate ( pbar ( loader ) ) : sample_set = call_compute ( compute , batch ) for cond , sample in sample_set : # Move uncommon conditional data to the cpu before collating. if cond not in most_common_conditions : sample = sample . cpu ( ) cq . add ( cond , sample ) # Move uncommon conditions off the GPU. if i and not i & ( i - 1 ) : # if i is a power of 2: common_conditions = set ( cq . most_common_conditions ( gpu_cache ) ) cq . to_ ( 'cpu' , [ k for k in cq . keys ( ) if k not in common_conditions ] ) # At the end, move all to the CPU cq . to_ ( 'cpu' ) save_cached_state ( cachefile , cq , args ) return cq