(compute, dataset, topk, k=None,
        cachefile=None, **kwargs)
Gathers data from topk examples computed over a dataset.
    The compute function will be provided a batch of data together
    with a list of units [[(unit, rank), ...], [(unit, rank),...]]
    with one list of relevant units for each batch item.  It should
    return a generator that yields [(unit, rank), data]
Gathers data from topk examples computed over a dataset.
    The compute function will be provided a batch of data together
    with a list of units [[(unit, rank), ...], [(unit, rank),...]]
    with one list of relevant units for each batch item.  It should
    return a generator that yields [(unit, rank), data]
Gathers data from topk examples computed over a dataset . The compute function will be provided a batch of data together with a list of units [[ ( unit rank ) ... ] [ ( unit rank ) ... ]] with one list of relevant units for each batch item . It should return a generator that yields [ ( unit rank ) data ]
def gather_topk(compute, dataset, topk, k=None,
        cachefile=None, **kwargs):
    '''
    Gathers data from topk examples computed over a dataset.
    The compute function will be provided a batch of data together
    with a list of units [[(unit, rank), ...], [(unit, rank),...]]
    with one list of relevant units for each batch item.  It should
    return a generator that yields [(unit, rank), data]
    '''
    with torch.no_grad():
        if k is None:
            k = topk.k
        args = dict(k=k, count=topk.count)
        cached_state = load_cached_state(cachefile, args)
        if cached_state is not None:
            return runningstats.GatherTensor(state=cached_state)
        gt = runningstats.GatherTensor(topk=topk, k=k)
        needed_images = defaultdict(list)
        for unit, imgnums in enumerate(topk.result()[1][:,:k]):
            for rank, imgnum in enumerate(imgnums.numpy()):
                needed_images[imgnum].append((unit, rank))
        needed_sample = FixedSubsetSampler(sorted(needed_images.keys()))
        loader = make_loader(dataset, sampler=needed_sample, **kwargs)
        for batchnum, batch in enumerate(pbar(loader)):
            assert isinstance(batch, list)
            batch.insert(0, [needed_images[
                needed_sample[index + batchnum * loader.batch_size]]
                    for index in range(len(batch[0]))])
            sample = call_compute(compute, batch)
            for (unit, rank), data in sample:
                gt.add(unit, rank, data)
        save_cached_state(cachefile, gt, args)
        return gt
def gather_topk ( compute , dataset , topk , k = None , cachefile = None , * * kwargs ) : with torch . no_grad ( ) : if k is None : k = topk . k args = dict ( k = k , count = topk . count ) cached_state = load_cached_state ( cachefile , args ) if cached_state is not None : return runningstats . GatherTensor ( state = cached_state ) gt = runningstats . GatherTensor ( topk = topk , k = k ) needed_images = defaultdict ( list ) for unit , imgnums in enumerate ( topk . result ( ) [ 1 ] [ : , : k ] ) : for rank , imgnum in enumerate ( imgnums . numpy ( ) ) : needed_images [ imgnum ] . append ( ( unit , rank ) ) needed_sample = FixedSubsetSampler ( sorted ( needed_images . keys ( ) ) ) loader = make_loader ( dataset , sampler = needed_sample , * * kwargs ) for batchnum , batch in enumerate ( pbar ( loader ) ) : assert isinstance ( batch , list ) batch . insert ( 0 , [ needed_images [ needed_sample [ index + batchnum * loader . batch_size ] ] for index in range ( len ( batch [ 0 ] ) ) ] ) sample = call_compute ( compute , batch ) for ( unit , rank ) , data in sample : gt . add ( unit , rank , data ) save_cached_state ( cachefile , gt , args ) return gt