code: def transformer_base_vq_ada_32ex_packed():
    
    hparams = transformer_base_v2()
    expert_utils.update_hparams_for_vq_gating(hparams)
    hparams.moe_num_experts = 32
    hparams.gating_type = "vq"
    # this gives us a batch size of 16 because each seq is len 256
    hparams.batch_size = 5072
    hparams.ffn_layer = "local_moe"
    hparams.shared_embedding_and_softmax_weights = False
    hparams.learning_rate_warmup_steps = 10000
    # one epoch for languagemodel_lm1b32k_packed = 27200 steps w/ bsize 128
    hparams.learning_rate_decay_steps = 27200
    hparams.num_heads = 4
    hparams.num_blocks = 1
    hparams.moe_k = 1
    hparams.num_decoder_layers = 6
    hparams.label_smoothing = 0.
    hparams.layer_prepostprocess_dropout = 0.1
    hparams.layer_postprocess_sequence = "dan"
    hparams.layer_preprocess_sequence = "none"
    hparams.weight_decay = 1e-06
    hparams.attention_dropout = 0.1
    hparams.optimizer = "Adafactor"
    hparams.learning_rate_schedule = "linear_warmup*rsqrt_decay*linear_decay"
    hparams.activation_dtype = "float32"
    hparams.learning_rate = 0.1
    hparams.learning_rate_constant = 1.0
    return hparams
docs: Set of hyperparameters for lm1b packed following tpu params.