code: def attention_lm_ae_extended():
    
    hparams = attention_lm_moe_base_long_seq()
    hparams.attention_layers = "eeee"
    hparams.attention_local = True
    # hparams.factored_logits=1  # Necessary when the number of expert grow bigger
    hparams.attention_moe_k = 2
    hparams.attention_exp_factor = 4
    # hparams.attention_exp_inputdim = 128
  
    hparams.layer_preprocess_sequence = "n"
    hparams.layer_postprocess_sequence = "da"
    return hparams
docs: Experiment with the exp_factor params.