(d_model,
                 d_ff,
                 n_heads,
                 dropout,
                 dropout_shared_axes,
                 mode,
                 ff_activation,
                 ff_dropout,
                 ff_chunk_size,
                 ff_use_sru,
                 ff_sparsity,
                 ff_sparsity_type,
                 attention_chunk_size,
                 attention_type,
                 n_attention_layers=1,
                 n_feedforward_layers=1)
return residual_attentions + feed_forwards
Returns a list of layers that implements a Transformer encoder block.

  The input to the block is a pair, (activations, mask), where the mask was
  created from the original source tokens to prevent attending to the padding
  part of the input.

  Args:
    d_model: Final dimension of tensors at most points in the model, including
      the initial embedding output.
    d_ff: Size of special dense layer in the feed-forward part of each block.
    n_heads: Number of attention heads.
    dropout: Stochastic rate (probability) for dropping an activation value when
      applying dropout within a block.
    dropout_shared_axes: Tensor axes on which to share a dropout mask. Sharing
      along batch and sequence axes (`dropout_shared_axes=(0,1)`) is a useful
      way to save memory and apply consistent masks to activation vectors at
      different sequence positions.
    mode: If `'train'`, each block will include dropout; else, it will pass all
      values through unaltered.
    ff_activation: Type of activation function at the end of each block; must be
      an activation-type subclass of `Layer`.
    ff_dropout: Stochastic rate (probability) for dropping an activation value
      when applying dropout after the FF dense layer.
    ff_chunk_size: int; if > 0, chunk feed-forward into this-sized chunks
    ff_use_sru: int or pair of ints; if > 0, we use this many SRU layers
      in addition to the feed-forward block (second int specifies sru size)
    ff_sparsity: int, if > 0 use sparse feed-forward block with this sparsity
    ff_sparsity_type: string, if ff_sparsity >0,
      use SparseFF if ff_sparsity_type=`'1inN'` and
      use BlockSparseFF if ff_sparsity_type=`'Block'`
    attention_chunk_size: int, if > 0 run attention chunked at this size
    attention_type: The attention layer to use.
    n_attention_layers: how many residual causal attention layers should we
      have before the feed-forward block (default: 1, the standard block)
    n_feedforward_layers: how many FFNN layers should we have (default 1).

  Returns:
    A list of layers that maps (activations, mask) to (activations, mask).
Returns a list of layers that implements a Transformer encoder block.
Returns a list of layers that implements a Transformer encoder block .
def EncoderBlock(d_model,
                 d_ff,
                 n_heads,
                 dropout,
                 dropout_shared_axes,
                 mode,
                 ff_activation,
                 ff_dropout,
                 ff_chunk_size,
                 ff_use_sru,
                 ff_sparsity,
                 ff_sparsity_type,
                 attention_chunk_size,
                 attention_type,
                 n_attention_layers=1,
                 n_feedforward_layers=1):
  """Returns a list of layers that implements a Transformer encoder block.

  The input to the block is a pair, (activations, mask), where the mask was
  created from the original source tokens to prevent attending to the padding
  part of the input.

  Args:
    d_model: Final dimension of tensors at most points in the model, including
      the initial embedding output.
    d_ff: Size of special dense layer in the feed-forward part of each block.
    n_heads: Number of attention heads.
    dropout: Stochastic rate (probability) for dropping an activation value when
      applying dropout within a block.
    dropout_shared_axes: Tensor axes on which to share a dropout mask. Sharing
      along batch and sequence axes (`dropout_shared_axes=(0,1)`) is a useful
      way to save memory and apply consistent masks to activation vectors at
      different sequence positions.
    mode: If `'train'`, each block will include dropout; else, it will pass all
      values through unaltered.
    ff_activation: Type of activation function at the end of each block; must be
      an activation-type subclass of `Layer`.
    ff_dropout: Stochastic rate (probability) for dropping an activation value
      when applying dropout after the FF dense layer.
    ff_chunk_size: int; if > 0, chunk feed-forward into this-sized chunks
    ff_use_sru: int or pair of ints; if > 0, we use this many SRU layers
      in addition to the feed-forward block (second int specifies sru size)
    ff_sparsity: int, if > 0 use sparse feed-forward block with this sparsity
    ff_sparsity_type: string, if ff_sparsity >0,
      use SparseFF if ff_sparsity_type=`'1inN'` and
      use BlockSparseFF if ff_sparsity_type=`'Block'`
    attention_chunk_size: int, if > 0 run attention chunked at this size
    attention_type: The attention layer to use.
    n_attention_layers: how many residual causal attention layers should we
      have before the feed-forward block (default: 1, the standard block)
    n_feedforward_layers: how many FFNN layers should we have (default 1).

  Returns:
    A list of layers that maps (activations, mask) to (activations, mask).
  """
  # `n_attention_layers` number of residuals of attention layer + dropout.
  # pylint: disable=g-complex-comprehension
  residual_attentions = [
      tl.Residual(tl.LayerNorm(),
                  ApplyAttentionLayer(attention_type,
                                      d_model,
                                      n_heads,
                                      d_model // n_heads,
                                      d_model // n_heads,
                                      causal=False,
                                      masked=True,
                                      attention_dropout=dropout,
                                      output_dropout=dropout,
                                      attention_chunk_size=attention_chunk_size,
                                      mode=mode),
                  tl.Dropout(rate=dropout,
                             shared_axes=dropout_shared_axes,
                             mode=mode)
                  )
      for _ in range(n_attention_layers)
  ]

  feed_forwards = [
      tl.Residual(
          FeedForwardWithOptions(d_model, d_ff, dropout,
                                 dropout_shared_axes, ff_activation,
                                 ff_dropout, ff_chunk_size, ff_use_sru,
                                 ff_sparsity, True, mode, False,
                                 ff_sparsity_type)
      )
      for _ in range(n_feedforward_layers)
  ]
  # pylint: enable=g-complex-comprehension

  return residual_attentions + feed_forwards
def EncoderBlock ( d_model , d_ff , n_heads , dropout , dropout_shared_axes , mode , ff_activation , ff_dropout , ff_chunk_size , ff_use_sru , ff_sparsity , ff_sparsity_type , attention_chunk_size , attention_type , n_attention_layers = 1 , n_feedforward_layers = 1 ) : # `n_attention_layers` number of residuals of attention layer + dropout. # pylint: disable=g-complex-comprehension residual_attentions = [ tl . Residual ( tl . LayerNorm ( ) , ApplyAttentionLayer ( attention_type , d_model , n_heads , d_model // n_heads , d_model // n_heads , causal = False , masked = True , attention_dropout = dropout , output_dropout = dropout , attention_chunk_size = attention_chunk_size , mode = mode ) , tl . Dropout ( rate = dropout , shared_axes = dropout_shared_axes , mode = mode ) ) for _ in range ( n_attention_layers ) ] feed_forwards = [ tl . Residual ( FeedForwardWithOptions ( d_model , d_ff , dropout , dropout_shared_axes , ff_activation , ff_dropout , ff_chunk_size , ff_use_sru , ff_sparsity , True , mode , False , ff_sparsity_type ) ) for _ in range ( n_feedforward_layers ) ] # pylint: enable=g-complex-comprehension return residual_attentions + feed_forwards