(d_model, d_ff, n_heads, dropout, dropout_shared_axes,
                         mode, ff_activation)
return [                             # vec_d masks vec_e
      ResidualZero(
          tl.LayerNorm(),              # vec_d ..... .....
          causal_attention,            # vec_d ..... .....
          _Dropout(),                  # vec_d ..... .....
      ),
      ResidualZero(
          tl.LayerNorm(),              # vec_d ..... .....
          tl.Select([0, 2, 2, 1, 2]),  # vec_d vec_e vec_e masks vec_e
          attention_qkv,               # vec_d masks vec_e
          _Dropout(),                  # vec_d masks vec_e
      ),
      ResidualZero(
          tl.LayerNorm(),
          feed_forward,                # vec_d masks vec_e
          _Dropout(),
      ),
  ]
Returns a list of layers implementing a Transformer encoder-decoder block.

  The input is a triple (decoder_input, mask, encoder) where the mask is
  created from the original source to prevent attending to the padding part
  of the encoder.

  Args:
    d_model: int:  depth of embedding
    d_ff: int: depth of feed-forward layer
    n_heads: int: number of attention heads
    dropout: float: dropout rate (how much to drop out)
    dropout_shared_axes: axes on which to share dropout mask
    mode: str: 'train' or 'eval'
    ff_activation: the non-linearity in feed-forward layer

  Returns:
    A list of layers which maps triples (decoder_activations, mask,
    encoder_activations) to triples of the same sort.
Returns a list of layers implementing a Transformer encoder-decoder block.
Returns a list of layers implementing a Transformer encoder - decoder block .
def _EncoderDecoderBlock(d_model, d_ff, n_heads, dropout, dropout_shared_axes,
                         mode, ff_activation):
  """Returns a list of layers implementing a Transformer encoder-decoder block.

  The input is a triple (decoder_input, mask, encoder) where the mask is
  created from the original source to prevent attending to the padding part
  of the encoder.

  Args:
    d_model: int:  depth of embedding
    d_ff: int: depth of feed-forward layer
    n_heads: int: number of attention heads
    dropout: float: dropout rate (how much to drop out)
    dropout_shared_axes: axes on which to share dropout mask
    mode: str: 'train' or 'eval'
    ff_activation: the non-linearity in feed-forward layer

  Returns:
    A list of layers which maps triples (decoder_activations, mask,
    encoder_activations) to triples of the same sort.
  """
  def _Dropout():
    return tl.Dropout(rate=dropout, shared_axes=dropout_shared_axes, mode=mode)

  attention_qkv = tl.AttentionQKV(
      d_model, n_heads=n_heads, dropout=dropout, mode=mode,
      cache_KV_in_predict=True)

  causal_attention = tl.CausalAttention(
      d_model, n_heads=n_heads, mode=mode)

  feed_forward = _FeedForwardBlock(
      d_model, d_ff, dropout, dropout_shared_axes, mode, ff_activation)

  return [                             # vec_d masks vec_e
      ResidualZero(
          tl.LayerNorm(),              # vec_d ..... .....
          causal_attention,            # vec_d ..... .....
          _Dropout(),                  # vec_d ..... .....
      ),
      ResidualZero(
          tl.LayerNorm(),              # vec_d ..... .....
          tl.Select([0, 2, 2, 1, 2]),  # vec_d vec_e vec_e masks vec_e
          attention_qkv,               # vec_d masks vec_e
          _Dropout(),                  # vec_d masks vec_e
      ),
      ResidualZero(
          tl.LayerNorm(),
          feed_forward,                # vec_d masks vec_e
          _Dropout(),
      ),
  ]
def _EncoderDecoderBlock ( d_model , d_ff , n_heads , dropout , dropout_shared_axes , mode , ff_activation ) : def _Dropout ( ) : return tl . Dropout ( rate = dropout , shared_axes = dropout_shared_axes , mode = mode ) attention_qkv = tl . AttentionQKV ( d_model , n_heads = n_heads , dropout = dropout , mode = mode , cache_KV_in_predict = True ) causal_attention = tl . CausalAttention ( d_model , n_heads = n_heads , mode = mode ) feed_forward = _FeedForwardBlock ( d_model , d_ff , dropout , dropout_shared_axes , mode , ff_activation ) return [ # vec_d masks vec_e ResidualZero ( tl . LayerNorm ( ) , # vec_d ..... ..... causal_attention , # vec_d ..... ..... _Dropout ( ) , # vec_d ..... ..... ) , ResidualZero ( tl . LayerNorm ( ) , # vec_d ..... ..... tl . Select ( [ 0 , 2 , 2 , 1 , 2 ] ) , # vec_d vec_e vec_e masks vec_e attention_qkv , # vec_d masks vec_e _Dropout ( ) , # vec_d masks vec_e ) , ResidualZero ( tl . LayerNorm ( ) , feed_forward , # vec_d masks vec_e _Dropout ( ) , ) , ]