(self,
               d_ff,
               n_experts=64,
               temperature=0.7,
               mode='train',
               kernel_initializer=init.GlorotUniformInitializer(),
               bias_initializer=init.RandomNormalInitializer(1e-6))
Returns a block sparse feed-forward block.
Returns a block sparse feed-forward block.
Returns a block sparse feed - forward block .
def __init__(self,
               d_ff,
               n_experts=64,
               temperature=0.7,
               mode='train',
               kernel_initializer=init.GlorotUniformInitializer(),
               bias_initializer=init.RandomNormalInitializer(1e-6)):
    """Returns a block sparse feed-forward block."""
    super().__init__(name=f'BlockSparseFF_{d_ff}')
    self._mode = mode
    self._d_ff = d_ff
    self._n_experts = n_experts
    self._temperature = temperature if mode == 'train' else 0.0
    self._n_elements_in_block = d_ff // n_experts
    self._kernel_initializer = kernel_initializer
    self._bias_initializer = bias_initializer
    assert self._d_ff % self._n_experts == 0
def __init__ ( self , d_ff , n_experts = 64 , temperature = 0.7 , mode = 'train' , kernel_initializer = init . GlorotUniformInitializer ( ) , bias_initializer = init . RandomNormalInitializer ( 1e-6 ) ) : super ( ) . __init__ ( name = f'BlockSparseFF_{d_ff}' ) self . _mode = mode self . _d_ff = d_ff self . _n_experts = n_experts self . _temperature = temperature if mode == 'train' else 0.0 self . _n_elements_in_block = d_ff // n_experts self . _kernel_initializer = kernel_initializer self . _bias_initializer = bias_initializer assert self . _d_ff % self . _n_experts == 0