python3 build.py --ft_model_dir ./llama_7B_sq0.8/1-gpu/ --remove_input_padding --use_gpt_attention_plugin float16 --enable_context_fmha --use_gemm_plugin float16 --output_dir ./tmp/llama/7B/trt_engines/smooth_llama_7B_gamma_sq0.8 --max_batch_size 64 --use_smooth_quant --use_inflight_batching --per_token --per_channel --max_input_len 2000 --max_output_len 2000 --paged_kv_cache
{}
transformers endpoints_compatible region:us