python3 build.py --model_dir ./weights/llama/ --remove_input_padding --use_gpt_attention_plugin float16 --enable_context_fmha --use_gemm_plugin float16 --output_dir ./tmp/llama/7B/trt_engines/llama_7B_kv_4k --max_batch_size 64 --use_inflight_batching --max_input_len 4096 --max_output_len 4096 --paged_kv_cache --int8_kv_cache
{}
transformers endpoints_compatible region:us