(self, input_signature)
Randomly initializes this layer's weights.
Randomly initializes this layer's weights.
Randomly initializes this layer s weights .
def init_weights_and_state(self, input_signature):
    """Randomly initializes this layer's weights."""
    x_input_signature = input_signature[0]
    d_model = x_input_signature.shape[-1]
    shape_m1 = (d_model, self._d_lowrank)
    shape_m2 = (self._d_lowrank, self._d_ff)
    shape_mb = (self._d_ff,)

    rng_m1, rng_m2, rng_mb = fastmath.random.split(self.rng, 3)
    m1 = self._kernel_initializer(shape_m1, rng_m1)
    m2 = self._kernel_initializer(shape_m2, rng_m2)
    mb = self._bias_initializer(shape_mb, rng_mb)
    if self._use_bfloat16:
      m1 = m1.astype(jnp.bfloat16)
      m2 = m2.astype(jnp.bfloat16)
      mb = mb.astype(jnp.bfloat16)

    # Reshapes below, with einsum in feedforward, improve the training speed.
    m2 = jnp.reshape(m2, [self._d_lowrank, self._d1, self._d2])
    mb = jnp.reshape(mb, [self._d1, self._d2])

    self.weights = (m1, m2, mb)
def init_weights_and_state ( self , input_signature ) : x_input_signature = input_signature [ 0 ] d_model = x_input_signature . shape [ - 1 ] shape_m1 = ( d_model , self . _d_lowrank ) shape_m2 = ( self . _d_lowrank , self . _d_ff ) shape_mb = ( self . _d_ff , ) rng_m1 , rng_m2 , rng_mb = fastmath . random . split ( self . rng , 3 ) m1 = self . _kernel_initializer ( shape_m1 , rng_m1 ) m2 = self . _kernel_initializer ( shape_m2 , rng_m2 ) mb = self . _bias_initializer ( shape_mb , rng_mb ) if self . _use_bfloat16 : m1 = m1 . astype ( jnp . bfloat16 ) m2 = m2 . astype ( jnp . bfloat16 ) mb = mb . astype ( jnp . bfloat16 ) # Reshapes below, with einsum in feedforward, improve the training speed. m2 = jnp . reshape ( m2 , [ self . _d_lowrank , self . _d1 , self . _d2 ] ) mb = jnp . reshape ( mb , [ self . _d1 , self . _d2 ] ) self . weights = ( m1 , m2 , mb )