code: def xmoe2_v1():
    
    hparams = xmoe2_dense(0)
    moe.set_default_moe_hparams(hparams)
    hparams.decoder_layers = (
        ["local_att", "local_att", "drd",
         "att", "drd", "local_att", "local_att", "hmoe"] * 4)[:-1]
    hparams.d_ff = 2048
    hparams.d_kv = 128
    hparams.moe_hidden_size = 32768
    hparams.mesh_shape = "b0:4;b1:8"
    hparams.layout = "outer_batch:b0;inner_batch:b1,expert_x:b1,expert_y:b0"
    hparams.outer_batch_size = 4
    hparams.moe_num_experts = [8, 4]
    hparams.num_heads = 4
    return hparams
docs: Model incorporating mixture-of-experts and local-attention.
  
    ~6B parameters
  
    32 experts in 3 hierarchichal moe layers.
  
    Returns:
      a hparams