(self, num_features, devices=None, eps=1e-5, momentum=0.1, affine=True, activation="leaky_relu",
                 slope=0.01)
Creates a synchronized, InPlace Activated Batch Normalization module

        Parameters
        ----------
        num_features : int
            Number of feature channels in the input and output.
        devices : list of int or None
            IDs of the GPUs that will run the replicas of this module.
        eps : float
            Small constant to prevent numerical issues.
        momentum : float
            Momentum factor applied to compute running statistics as.
        affine : bool
            If `True` apply learned scale and shift transformation after normalization.
        activation : str
            Name of the activation functions, one of: `leaky_relu`, `elu` or `none`.
        slope : float
            Negative slope for the `leaky_relu` activation.
Creates a synchronized, InPlace Activated Batch Normalization module
Creates a synchronized InPlace Activated Batch Normalization module
def __init__(self, num_features, devices=None, eps=1e-5, momentum=0.1, affine=True, activation="leaky_relu",
                 slope=0.01):
        """Creates a synchronized, InPlace Activated Batch Normalization module

        Parameters
        ----------
        num_features : int
            Number of feature channels in the input and output.
        devices : list of int or None
            IDs of the GPUs that will run the replicas of this module.
        eps : float
            Small constant to prevent numerical issues.
        momentum : float
            Momentum factor applied to compute running statistics as.
        affine : bool
            If `True` apply learned scale and shift transformation after normalization.
        activation : str
            Name of the activation functions, one of: `leaky_relu`, `elu` or `none`.
        slope : float
            Negative slope for the `leaky_relu` activation.
        """
        super(InPlaceABNSync, self).__init__()
        self.num_features = num_features
        self.devices = devices if devices else list(range(torch.cuda.device_count()))
        self.affine = affine
        self.eps = eps
        self.momentum = momentum
        self.activation = activation
        self.slope = slope
        if self.affine:
            self.weight = nn.Parameter(torch.Tensor(num_features))
            self.bias = nn.Parameter(torch.Tensor(num_features))
        else:
            self.register_parameter('weight', None)
            self.register_parameter('bias', None)
        self.register_buffer('running_mean', torch.zeros(num_features))
        self.register_buffer('running_var', torch.ones(num_features))
        self.reset_parameters()

        # Initialize queues
        self.worker_ids = self.devices[1:]
        self.master_queue = Queue(len(self.worker_ids))
        self.worker_queues = [Queue(1) for _ in self.worker_ids]
def __init__ ( self , num_features , devices = None , eps = 1e-5 , momentum = 0.1 , affine = True , activation = "leaky_relu" , slope = 0.01 ) : super ( InPlaceABNSync , self ) . __init__ ( ) self . num_features = num_features self . devices = devices if devices else list ( range ( torch . cuda . device_count ( ) ) ) self . affine = affine self . eps = eps self . momentum = momentum self . activation = activation self . slope = slope if self . affine : self . weight = nn . Parameter ( torch . Tensor ( num_features ) ) self . bias = nn . Parameter ( torch . Tensor ( num_features ) ) else : self . register_parameter ( 'weight' , None ) self . register_parameter ( 'bias' , None ) self . register_buffer ( 'running_mean' , torch . zeros ( num_features ) ) self . register_buffer ( 'running_var' , torch . ones ( num_features ) ) self . reset_parameters ( ) # Initialize queues self . worker_ids = self . devices [ 1 : ] self . master_queue = Queue ( len ( self . worker_ids ) ) self . worker_queues = [ Queue ( 1 ) for _ in self . worker_ids ]