(d_model,
                         d_ff,
                         n_heads,
                         dropout,
                         dropout_shared_axes,
                         mode,
                         ff_activation)
return [                             # vec_d masks vec_e
      tl.Residual(
          tl.LayerNorm(),
          _CausalAttention(),
          _Dropout(),
      ),
      tl.Residual(
          tl.LayerNorm(),
          tl.Select([0, 2, 2, 1, 2]),  # vec_d vec_e vec_e masks vec_e
          _AttentionQKV(),             # vec_d masks vec_e
          _Dropout(),
      ),
      tl.Residual(
          tl.LayerNorm(),
          _FFBlock(),
          _Dropout(),
      ),
  ]
Returns a list of layers implementing a Transformer encoder-decoder block.

  The block input is a triple (decoder_activations, mask, encoder_activations)
  where the mask was created from the original input token IDs to prevent
  attending to padding positions for that input.

  Args:
    d_model: Last/innermost dimension of activation arrays at most points in
        the model, including the initial embedding output.
    d_ff: Last/innermost dimension of special (typically wider)
        :py:class:`Dense` layer in the feedforward part of each block.
    n_heads: Number of attention heads.
    dropout: Stochastic rate (probability) for dropping an activation value
        when applying dropout within encoder/decoder blocks. The same rate is
        also used for attention dropout in encoder/decoder blocks.
    dropout_shared_axes: Tensor axes on which to share a dropout mask.
        Sharing along batch and sequence axes (``dropout_shared_axes=(0,1)``)
        is a useful way to save memory and apply consistent masks to activation
        vectors at different sequence positions.
    mode: If ``'train'``, each block will include dropout; else, it will
        pass all values through unaltered.
    ff_activation: Type of activation function at the end of each block; must
        be an activation-type subclass of :py:class:`Layer`.

  Returns:
    A list of layers that act in series as a (repeatable) encoder-decoder
    block.
Returns a list of layers implementing a Transformer encoder-decoder block.
Returns a list of layers implementing a Transformer encoder - decoder block .
def _EncoderDecoderBlock(d_model,
                         d_ff,
                         n_heads,
                         dropout,
                         dropout_shared_axes,
                         mode,
                         ff_activation):
  """Returns a list of layers implementing a Transformer encoder-decoder block.

  The block input is a triple (decoder_activations, mask, encoder_activations)
  where the mask was created from the original input token IDs to prevent
  attending to padding positions for that input.

  Args:
    d_model: Last/innermost dimension of activation arrays at most points in
        the model, including the initial embedding output.
    d_ff: Last/innermost dimension of special (typically wider)
        :py:class:`Dense` layer in the feedforward part of each block.
    n_heads: Number of attention heads.
    dropout: Stochastic rate (probability) for dropping an activation value
        when applying dropout within encoder/decoder blocks. The same rate is
        also used for attention dropout in encoder/decoder blocks.
    dropout_shared_axes: Tensor axes on which to share a dropout mask.
        Sharing along batch and sequence axes (``dropout_shared_axes=(0,1)``)
        is a useful way to save memory and apply consistent masks to activation
        vectors at different sequence positions.
    mode: If ``'train'``, each block will include dropout; else, it will
        pass all values through unaltered.
    ff_activation: Type of activation function at the end of each block; must
        be an activation-type subclass of :py:class:`Layer`.

  Returns:
    A list of layers that act in series as a (repeatable) encoder-decoder
    block.
  """
  def _Dropout():
    return tl.Dropout(rate=dropout, shared_axes=dropout_shared_axes, mode=mode)

  def _AttentionQKV():
    return tl.AttentionQKV(d_model, n_heads=n_heads, dropout=dropout,
                           mode=mode, cache_KV_in_predict=True)

  def _CausalAttention():
    return tl.CausalAttention(d_model, n_heads=n_heads, mode=mode)

  def _FFBlock():
    return _FeedForwardBlock(d_model, d_ff, dropout, dropout_shared_axes, mode,
                             ff_activation)

  return [                             # vec_d masks vec_e
      tl.Residual(
          tl.LayerNorm(),
          _CausalAttention(),
          _Dropout(),
      ),
      tl.Residual(
          tl.LayerNorm(),
          tl.Select([0, 2, 2, 1, 2]),  # vec_d vec_e vec_e masks vec_e
          _AttentionQKV(),             # vec_d masks vec_e
          _Dropout(),
      ),
      tl.Residual(
          tl.LayerNorm(),
          _FFBlock(),
          _Dropout(),
      ),
  ]
def _EncoderDecoderBlock ( d_model , d_ff , n_heads , dropout , dropout_shared_axes , mode , ff_activation ) : def _Dropout ( ) : return tl . Dropout ( rate = dropout , shared_axes = dropout_shared_axes , mode = mode ) def _AttentionQKV ( ) : return tl . AttentionQKV ( d_model , n_heads = n_heads , dropout = dropout , mode = mode , cache_KV_in_predict = True ) def _CausalAttention ( ) : return tl . CausalAttention ( d_model , n_heads = n_heads , mode = mode ) def _FFBlock ( ) : return _FeedForwardBlock ( d_model , d_ff , dropout , dropout_shared_axes , mode , ff_activation ) return [ # vec_d masks vec_e tl . Residual ( tl . LayerNorm ( ) , _CausalAttention ( ) , _Dropout ( ) , ) , tl . Residual ( tl . LayerNorm ( ) , tl . Select ( [ 0 , 2 , 2 , 1 , 2 ] ) , # vec_d vec_e vec_e masks vec_e _AttentionQKV ( ) , # vec_d masks vec_e _Dropout ( ) , ) , tl . Residual ( tl . LayerNorm ( ) , _FFBlock ( ) , _Dropout ( ) , ) , ]