(input_dict, average=True)
return reduced_dict
Args:
        input_dict (dict): all the values will be reduced
        average (bool): whether to do average or sum
    Reduce the values in the dictionary from all processes so that process with rank
    0 has the averaged results. Returns a dict with the same fields as
    input_dict, after reduction.
Args:
        input_dict (dict): all the values will be reduced
        average (bool): whether to do average or sum
    Reduce the values in the dictionary from all processes so that process with rank
    0 has the averaged results. Returns a dict with the same fields as
    input_dict, after reduction.
Args : input_dict ( dict ) : all the values will be reduced average ( bool ) : whether to do average or sum Reduce the values in the dictionary from all processes so that process with rank 0 has the averaged results . Returns a dict with the same fields as input_dict after reduction .
def reduce_dict(input_dict, average=True):
    """
    Args:
        input_dict (dict): all the values will be reduced
        average (bool): whether to do average or sum
    Reduce the values in the dictionary from all processes so that process with rank
    0 has the averaged results. Returns a dict with the same fields as
    input_dict, after reduction.
    """
    world_size = get_world_size()
    if world_size < 2:
        return input_dict
    with torch.no_grad():
        names = []
        values = []
        # sort the keys so that they are consistent across processes
        for k in sorted(input_dict.keys()):
            names.append(k)
            values.append(input_dict[k])
        values = torch.stack(values, dim=0)
        dist.reduce(values, dst=0)
        if dist.get_rank() == 0 and average:
            # only main process gets accumulated, so only divide by
            # world_size in this case
            values /= world_size
        reduced_dict = {k: v for k, v in zip(names, values)}
    return reduced_dict
def reduce_dict ( input_dict , average = True ) : world_size = get_world_size ( ) if world_size < 2 : return input_dict with torch . no_grad ( ) : names = [ ] values = [ ] # sort the keys so that they are consistent across processes for k in sorted ( input_dict . keys ( ) ) : names . append ( k ) values . append ( input_dict [ k ] ) values = torch . stack ( values , dim = 0 ) dist . reduce ( values , dst = 0 ) if dist . get_rank ( ) == 0 and average : # only main process gets accumulated, so only divide by # world_size in this case values /= world_size reduced_dict = { k : v for k , v in zip ( names , values ) } return reduced_dict