(input_vocab_size,
             output_vocab_size=None,
             d_model=512,
             d_ff=2048,
             n_encoder_layers=6,
             n_decoder_layers=6,
             n_heads=8,
             dropout=0.1,
             max_len=2048,
             ff_activation=tl.Relu,
             ff_dropout=None,
             mode='train',
             pos_type=None,
             pos_axial_shape=None,
             pos_d_axial_embs=None,
             ff_use_sru=0,
             ff_chunk_size=0,
             ff_sparsity=0)
return tl.Serial(
      # Input: encoder_side_tokens, decoder_side_tokens
      # Copy decoder tokens for use in loss.
      tl.Select([0, 1, 1]),                 # tok_e tok_d tok_d
      tl.Branch([], [tl.PaddingMask(),
                     _RemoveAxes12()]),     # tok_e mask  tok_d .....

      # Encode.
      encoder,                              # vec_e  mask tok_d .....

      # Decode.
      tl.Select([2, 0, 1]),                 # tok_d vec_e mask .....
      tl.ShiftRight(mode=mode),             # tok_d vec_e mask .....
      out_encoder,                          # vec_d vec_e mask .....
      tl.Dup(),                             # vec_d1 vec_d2 vec_e mask .....
      tl.ReversibleSerial(encoder_decoder_blocks),
      _XYAvg(),                             # vec_d vec_e mask .....
      tl.LayerNorm(),                       # vec_d vec_e mask .....

      # Map to output vocab.
      tl.Select([0], n_in=3),               # vec_d .....
      tl.Dense(output_vocab_size),          # vec_d .....
  )
Reversible transformer encoder-decoder model.

  This model expects an input pair: target, source.

  At the moment, this model supports dot-product attention only. For the
  attention types in the Reformer paper, see ReformerLM.

  Args:
    input_vocab_size: int: vocab size of the source.
    output_vocab_size: int (optional): vocab size of the target. If None, the
      source and target are assumed to have the same vocab.
    d_model: int:  depth of embedding
    d_ff: int: depth of feed-forward layer
    n_encoder_layers: int: number of encoder layers
    n_decoder_layers: int: number of decoder layers
    n_heads: int: number of attention heads
    dropout: float: dropout rate (how much to drop out)
    max_len: int: maximum symbol length for positional encoding
    ff_activation: the non-linearity in feed-forward layer
    ff_dropout: float: (optional) separate dropout rate at feed-forward
      nonlinearity. This is called relu_dropout in T2T.
    mode: str: 'train' or 'eval'
    pos_type: string, the type of positional embeddings to use.
    pos_axial_shape: tuple of ints: input shape to use for the axial position
      encoding. If unset, axial position encoding is disabled.
    pos_d_axial_embs: tuple of ints: depth of position embedding for each axis.
      Tuple length must match pos_axial_shape, and values must sum to d_model.
    ff_use_sru: int; if > 0, we use this many SRU layers instead of feed-forward
    ff_chunk_size: int; if > 0, chunk feed-forward into this-sized chunks
    ff_sparsity: int, if > 0 use sparse feed-forward block with this sparsity

  Returns:
    A Reformer model as a layer that maps from a target, source pair to
    activations over a vocab set.
Reversible transformer encoder-decoder model.
Reversible transformer encoder - decoder model .
def Reformer(input_vocab_size,
             output_vocab_size=None,
             d_model=512,
             d_ff=2048,
             n_encoder_layers=6,
             n_decoder_layers=6,
             n_heads=8,
             dropout=0.1,
             max_len=2048,
             ff_activation=tl.Relu,
             ff_dropout=None,
             mode='train',
             pos_type=None,
             pos_axial_shape=None,
             pos_d_axial_embs=None,
             ff_use_sru=0,
             ff_chunk_size=0,
             ff_sparsity=0):
  """Reversible transformer encoder-decoder model.

  This model expects an input pair: target, source.

  At the moment, this model supports dot-product attention only. For the
  attention types in the Reformer paper, see ReformerLM.

  Args:
    input_vocab_size: int: vocab size of the source.
    output_vocab_size: int (optional): vocab size of the target. If None, the
      source and target are assumed to have the same vocab.
    d_model: int:  depth of embedding
    d_ff: int: depth of feed-forward layer
    n_encoder_layers: int: number of encoder layers
    n_decoder_layers: int: number of decoder layers
    n_heads: int: number of attention heads
    dropout: float: dropout rate (how much to drop out)
    max_len: int: maximum symbol length for positional encoding
    ff_activation: the non-linearity in feed-forward layer
    ff_dropout: float: (optional) separate dropout rate at feed-forward
      nonlinearity. This is called relu_dropout in T2T.
    mode: str: 'train' or 'eval'
    pos_type: string, the type of positional embeddings to use.
    pos_axial_shape: tuple of ints: input shape to use for the axial position
      encoding. If unset, axial position encoding is disabled.
    pos_d_axial_embs: tuple of ints: depth of position embedding for each axis.
      Tuple length must match pos_axial_shape, and values must sum to d_model.
    ff_use_sru: int; if > 0, we use this many SRU layers instead of feed-forward
    ff_chunk_size: int; if > 0, chunk feed-forward into this-sized chunks
    ff_sparsity: int, if > 0 use sparse feed-forward block with this sparsity

  Returns:
    A Reformer model as a layer that maps from a target, source pair to
    activations over a vocab set.
  """
  in_encoder, out_encoder, output_vocab_size = (
      ct.EmbeddingAndPositionalEncodings(
          input_vocab_size,
          d_model,
          mode,
          dropout,
          [-2],  # dropout_shared_axes
          max_len,
          output_vocab_size=output_vocab_size,
          pos_type=pos_type,
          pos_axial_shape=pos_axial_shape,
          pos_d_axial_embs=pos_d_axial_embs)
  )

  # pylint: disable=g-complex-comprehension
  encoder_blocks = [
      EncoderBlock(
          d_model, d_ff, n_heads, tl.SelfAttention, dropout, ff_activation,
          ff_dropout, mode=mode, ff_use_sru=ff_use_sru,
          ff_chunk_size=ff_chunk_size, ff_sparsity=ff_sparsity)
      for _ in range(n_encoder_layers)]
  # pylint: enable=g-complex-comprehension

  encoder = tl.Serial([
      in_encoder,
      tl.Dup(),
      tl.ReversibleSerial(encoder_blocks),
      _XYAvg(),
      tl.LayerNorm(),
  ])
  if mode == 'predict':
    encoder = tl.Cache(encoder)

  # pylint: disable=g-complex-comprehension
  encoder_decoder_blocks = [
      EncoderDecoderBlock(
          d_model, d_ff, n_heads, dropout, ff_activation, ff_dropout, mode,
          ff_use_sru=ff_use_sru, ff_chunk_size=ff_chunk_size,
          ff_sparsity=ff_sparsity)
      for _ in range(n_decoder_layers)]
  # pylint: enable=g-complex-comprehension

  # Assemble and return the model.
  return tl.Serial(
      # Input: encoder_side_tokens, decoder_side_tokens
      # Copy decoder tokens for use in loss.
      tl.Select([0, 1, 1]),                 # tok_e tok_d tok_d
      tl.Branch([], [tl.PaddingMask(),
                     _RemoveAxes12()]),     # tok_e mask  tok_d .....

      # Encode.
      encoder,                              # vec_e  mask tok_d .....

      # Decode.
      tl.Select([2, 0, 1]),                 # tok_d vec_e mask .....
      tl.ShiftRight(mode=mode),             # tok_d vec_e mask .....
      out_encoder,                          # vec_d vec_e mask .....
      tl.Dup(),                             # vec_d1 vec_d2 vec_e mask .....
      tl.ReversibleSerial(encoder_decoder_blocks),
      _XYAvg(),                             # vec_d vec_e mask .....
      tl.LayerNorm(),                       # vec_d vec_e mask .....

      # Map to output vocab.
      tl.Select([0], n_in=3),               # vec_d .....
      tl.Dense(output_vocab_size),          # vec_d .....
  )
def Reformer ( input_vocab_size , output_vocab_size = None , d_model = 512 , d_ff = 2048 , n_encoder_layers = 6 , n_decoder_layers = 6 , n_heads = 8 , dropout = 0.1 , max_len = 2048 , ff_activation = tl . Relu , ff_dropout = None , mode = 'train' , pos_type = None , pos_axial_shape = None , pos_d_axial_embs = None , ff_use_sru = 0 , ff_chunk_size = 0 , ff_sparsity = 0 ) : in_encoder , out_encoder , output_vocab_size = ( ct . EmbeddingAndPositionalEncodings ( input_vocab_size , d_model , mode , dropout , [ - 2 ] , # dropout_shared_axes max_len , output_vocab_size = output_vocab_size , pos_type = pos_type , pos_axial_shape = pos_axial_shape , pos_d_axial_embs = pos_d_axial_embs ) ) # pylint: disable=g-complex-comprehension encoder_blocks = [ EncoderBlock ( d_model , d_ff , n_heads , tl . SelfAttention , dropout , ff_activation , ff_dropout , mode = mode , ff_use_sru = ff_use_sru , ff_chunk_size = ff_chunk_size , ff_sparsity = ff_sparsity ) for _ in range ( n_encoder_layers ) ] # pylint: enable=g-complex-comprehension encoder = tl . Serial ( [ in_encoder , tl . Dup ( ) , tl . ReversibleSerial ( encoder_blocks ) , _XYAvg ( ) , tl . LayerNorm ( ) , ] ) if mode == 'predict' : encoder = tl . Cache ( encoder ) # pylint: disable=g-complex-comprehension encoder_decoder_blocks = [ EncoderDecoderBlock ( d_model , d_ff , n_heads , dropout , ff_activation , ff_dropout , mode , ff_use_sru = ff_use_sru , ff_chunk_size = ff_chunk_size , ff_sparsity = ff_sparsity ) for _ in range ( n_decoder_layers ) ] # pylint: enable=g-complex-comprehension # Assemble and return the model. return tl . Serial ( # Input: encoder_side_tokens, decoder_side_tokens # Copy decoder tokens for use in loss. tl . Select ( [ 0 , 1 , 1 ] ) , # tok_e tok_d tok_d tl . Branch ( [ ] , [ tl . PaddingMask ( ) , _RemoveAxes12 ( ) ] ) , # tok_e mask  tok_d ..... # Encode. encoder , # vec_e  mask tok_d ..... # Decode. tl . Select ( [ 2 , 0 , 1 ] ) , # tok_d vec_e mask ..... tl . ShiftRight ( mode = mode ) , # tok_d vec_e mask ..... out_encoder , # vec_d vec_e mask ..... tl . Dup ( ) , # vec_d1 vec_d2 vec_e mask ..... tl . ReversibleSerial ( encoder_decoder_blocks ) , _XYAvg ( ) , # vec_d vec_e mask ..... tl . LayerNorm ( ) , # vec_d vec_e mask ..... # Map to output vocab. tl . Select ( [ 0 ] , n_in = 3 ) , # vec_d ..... tl . Dense ( output_vocab_size ) , # vec_d ..... )