(d_feature, n_heads=1, dropout=0.0,
                numerical_stabilizer=0.001, mode='train')
return tl.ConfigurableAttention(
      core.Dense(d_feature), core.Dense(d_feature), core.Dense(d_feature),
      core.Dense(d_feature), n_heads=n_heads,
      qkv_attention_layer=tl.CausalFavorAttention(numerical_stabilizer,
                                                  mode))
Returns a layer that maps activations to activations, with causal masking.

  Like `CausalAttention`, this layer type represents one pass of multi-head
  causal attention, but using FAVOR fast attention as in the following paper:
  https://arxiv.org/abs/2006.03555

  Args:
    d_feature: Depth/dimensionality of feature embedding.
    n_heads: Number of attention heads.
    dropout: Probababilistic rate for internal dropout applied to attention
        activations (based on query-key pairs) before dotting them with values.
    numerical_stabilizer: float, small number used for numerical stability.
    mode: One of `'train'`, `'eval'`, or `'predict'`.
Returns a layer that maps activations to activations, with causal masking.
Returns a layer that maps activations to activations with causal masking .
def CausalFavor(d_feature, n_heads=1, dropout=0.0,
                numerical_stabilizer=0.001, mode='train'):
  """Returns a layer that maps activations to activations, with causal masking.

  Like `CausalAttention`, this layer type represents one pass of multi-head
  causal attention, but using FAVOR fast attention as in the following paper:
  https://arxiv.org/abs/2006.03555

  Args:
    d_feature: Depth/dimensionality of feature embedding.
    n_heads: Number of attention heads.
    dropout: Probababilistic rate for internal dropout applied to attention
        activations (based on query-key pairs) before dotting them with values.
    numerical_stabilizer: float, small number used for numerical stability.
    mode: One of `'train'`, `'eval'`, or `'predict'`.
  """
  del dropout
  return tl.ConfigurableAttention(
      core.Dense(d_feature), core.Dense(d_feature), core.Dense(d_feature),
      core.Dense(d_feature), n_heads=n_heads,
      qkv_attention_layer=tl.CausalFavorAttention(numerical_stabilizer,
                                                  mode))
def CausalFavor ( d_feature , n_heads = 1 , dropout = 0.0 , numerical_stabilizer = 0.001 , mode = 'train' ) : del dropout return tl . ConfigurableAttention ( core . Dense ( d_feature ) , core . Dense ( d_feature ) , core . Dense ( d_feature ) , core . Dense ( d_feature ) , n_heads = n_heads , qkv_attention_layer = tl . CausalFavorAttention ( numerical_stabilizer , mode ) )