(d_model, d_ff, n_heads,
                  dropout, dropout_shared_axes, mode, ff_activation)
return [
      ResidualZero(
          tl.LayerNorm(),
          causal_attention,
          dropout_,
      ),
      ResidualZero(
          tl.LayerNorm(),
          feed_forward,
          dropout_,
      ),
  ]
Returns a list of layers that implements a Transformer decoder block.

  The input is an activation tensor.

  Args:
    d_model: int:  depth of embedding
    d_ff: int: depth of feed-forward layer
    n_heads: int: number of attention heads
    dropout: float: dropout rate (how much to drop out)
    dropout_shared_axes: axes on which to share dropout mask
    mode: str: 'train' or 'eval'
    ff_activation: the non-linearity in feed-forward layer

  Returns:
    A list of layers that maps an activation tensor to an activation tensor.
Returns a list of layers that implements a Transformer decoder block.
Returns a list of layers that implements a Transformer decoder block .
def _DecoderBlock(d_model, d_ff, n_heads,
                  dropout, dropout_shared_axes, mode, ff_activation):
  """Returns a list of layers that implements a Transformer decoder block.

  The input is an activation tensor.

  Args:
    d_model: int:  depth of embedding
    d_ff: int: depth of feed-forward layer
    n_heads: int: number of attention heads
    dropout: float: dropout rate (how much to drop out)
    dropout_shared_axes: axes on which to share dropout mask
    mode: str: 'train' or 'eval'
    ff_activation: the non-linearity in feed-forward layer

  Returns:
    A list of layers that maps an activation tensor to an activation tensor.
  """
  causal_attention = tl.CausalAttention(
      d_model, n_heads=n_heads, dropout=dropout, mode=mode),

  feed_forward = _FeedForwardBlock(
      d_model, d_ff, dropout, dropout_shared_axes, mode, ff_activation)

  dropout_ = tl.Dropout(
      rate=dropout, shared_axes=dropout_shared_axes, mode=mode)

  return [
      ResidualZero(
          tl.LayerNorm(),
          causal_attention,
          dropout_,
      ),
      ResidualZero(
          tl.LayerNorm(),
          feed_forward,
          dropout_,
      ),
  ]
def _DecoderBlock ( d_model , d_ff , n_heads , dropout , dropout_shared_axes , mode , ff_activation ) : causal_attention = tl . CausalAttention ( d_model , n_heads = n_heads , dropout = dropout , mode = mode ) , feed_forward = _FeedForwardBlock ( d_model , d_ff , dropout , dropout_shared_axes , mode , ff_activation ) dropout_ = tl . Dropout ( rate = dropout , shared_axes = dropout_shared_axes , mode = mode ) return [ ResidualZero ( tl . LayerNorm ( ) , causal_attention , dropout_ , ) , ResidualZero ( tl . LayerNorm ( ) , feed_forward , dropout_ , ) , ]