(scalar, args)
return scalar_t.item()
Average a scalar over the nodes if we are in distributed training. We use this for distributed evaluation.
Average a scalar over the nodes if we are in distributed training. We use this for distributed evaluation.
Average a scalar over the nodes if we are in distributed training . We use this for distributed evaluation .
def average_distributed_scalar(scalar, args):
    """ Average a scalar over the nodes if we are in distributed training. We use this for distributed evaluation. """
    if args.local_rank == -1:
        return scalar
    scalar_t = torch.tensor(scalar, dtype=torch.float, device=args.device) / torch.distributed.get_world_size()
    torch.distributed.all_reduce(scalar_t, op=torch.distributed.ReduceOp.SUM)
    return scalar_t.item()
def average_distributed_scalar ( scalar , args ) : if args . local_rank == - 1 : return scalar scalar_t = torch . tensor ( scalar , dtype = torch . float , device = args . device ) / torch . distributed . get_world_size ( ) torch . distributed . all_reduce ( scalar_t , op = torch . distributed . ReduceOp . SUM ) return scalar_t . item ( )