(self, d_ff, n_elements_in_block, d_lowrank, quant_prob,
               use_bfloat16, big_weights_in_bfloat16, mode, kernel_initializer,
               bias_initializer, multiply_by_controller_output, kernel_scaling)
Returns a sparse feed-forward block.
Returns a sparse feed-forward block.
Returns a sparse feed - forward block .
def __init__(self, d_ff, n_elements_in_block, d_lowrank, quant_prob,
               use_bfloat16, big_weights_in_bfloat16, mode, kernel_initializer,
               bias_initializer, multiply_by_controller_output, kernel_scaling):
    """Returns a sparse feed-forward block."""
    n_in = 3 if mode == 'train' or multiply_by_controller_output else 2
    super().__init__(name=f'_SparseFFMain_{d_ff}', n_in=n_in, n_out=2)
    self._mode = mode
    self._use_bfloat16 = use_bfloat16
    self._big_weights_in_bfloat16 = big_weights_in_bfloat16
    self._d_ff = d_ff
    self._d_lowrank = d_lowrank
    self._quant_prob = quant_prob
    self._n_elements_in_block = n_elements_in_block
    self._kernel_initializer = kernel_initializer
    self._bias_initializer = bias_initializer
    # Helper numbers as d_ff will be divided by n_elements_in_block.
    assert self._d_ff % self._n_elements_in_block == 0
    self._d1 = self._d_ff // self._n_elements_in_block
    self._d2 = self._n_elements_in_block
    self._multiply_by_controller_output = multiply_by_controller_output
    self._kernel_scaling = kernel_scaling
def __init__ ( self , d_ff , n_elements_in_block , d_lowrank , quant_prob , use_bfloat16 , big_weights_in_bfloat16 , mode , kernel_initializer , bias_initializer , multiply_by_controller_output , kernel_scaling ) : n_in = 3 if mode == 'train' or multiply_by_controller_output else 2 super ( ) . __init__ ( name = f'_SparseFFMain_{d_ff}' , n_in = n_in , n_out = 2 ) self . _mode = mode self . _use_bfloat16 = use_bfloat16 self . _big_weights_in_bfloat16 = big_weights_in_bfloat16 self . _d_ff = d_ff self . _d_lowrank = d_lowrank self . _quant_prob = quant_prob self . _n_elements_in_block = n_elements_in_block self . _kernel_initializer = kernel_initializer self . _bias_initializer = bias_initializer # Helper numbers as d_ff will be divided by n_elements_in_block. assert self . _d_ff % self . _n_elements_in_block == 0 self . _d1 = self . _d_ff // self . _n_elements_in_block self . _d2 = self . _n_elements_in_block self . _multiply_by_controller_output = multiply_by_controller_output self . _kernel_scaling = kernel_scaling