(self, d_ff, n_elements_in_block, d_lowrank, temperature,
               use_bfloat16, mode, kernel_initializer, bias_initializer,
               also_return_nondiscrete_output)
Returns a sparse feed-forward block.
Returns a sparse feed-forward block.
Returns a sparse feed - forward block .
def __init__(self, d_ff, n_elements_in_block, d_lowrank, temperature,
               use_bfloat16, mode, kernel_initializer, bias_initializer,
               also_return_nondiscrete_output):
    """Returns a sparse feed-forward block."""
    n_out = 2 if also_return_nondiscrete_output else 1
    super().__init__(name=f'_SparseFFController_{d_ff}', n_in=2, n_out=n_out)
    self._use_bfloat16 = use_bfloat16
    self._d_ff = d_ff
    self._d_lowrank = d_lowrank
    # Q: what temperature is actually most useful in training?
    self._temperature = temperature if mode == 'train' else 0.0
    self._mode = mode
    self._n_elements_in_block = n_elements_in_block
    self._kernel_initializer = kernel_initializer
    self._bias_initializer = bias_initializer
    # Helper numbers as d_ff will be divided by n_elements_in_block.
    assert self._d_ff % self._n_elements_in_block == 0
    self._d1 = self._d_ff // self._n_elements_in_block
    self._d2 = self._n_elements_in_block
    self._also_return_nondiscrete_output = also_return_nondiscrete_output
def __init__ ( self , d_ff , n_elements_in_block , d_lowrank , temperature , use_bfloat16 , mode , kernel_initializer , bias_initializer , also_return_nondiscrete_output ) : n_out = 2 if also_return_nondiscrete_output else 1 super ( ) . __init__ ( name = f'_SparseFFController_{d_ff}' , n_in = 2 , n_out = n_out ) self . _use_bfloat16 = use_bfloat16 self . _d_ff = d_ff self . _d_lowrank = d_lowrank # Q: what temperature is actually most useful in training? self . _temperature = temperature if mode == 'train' else 0.0 self . _mode = mode self . _n_elements_in_block = n_elements_in_block self . _kernel_initializer = kernel_initializer self . _bias_initializer = bias_initializer # Helper numbers as d_ff will be divided by n_elements_in_block. assert self . _d_ff % self . _n_elements_in_block == 0 self . _d1 = self . _d_ff // self . _n_elements_in_block self . _d2 = self . _n_elements_in_block self . _also_return_nondiscrete_output = also_return_nondiscrete_output