run-sailor.sh

#!/bin/sh


printf "Running sail/Sailor-4B-Chat using vLLM OpenAI compatible API Server at port %s\n" "7860"

# Using the 1.8B version because the 4B version cannot be loaded due to memory constraints.
# Reducing max-num-batched-tokens to 7536 because got this error:
# INFO 11-27 15:32:01 model_runner.py:1077] Loading model weights took 7.4150 GB
# INFO 11-27 15:32:09 worker.py:232] Memory profiling results: total_gpu_memory=14.58GiB initial_memory_usage=7.61GiB peak_torch_memory=9.31GiB memory_usage_post_profile=7.62GiB non_torch_memory=0.20GiB kv_cache_size=2.88GiB gpu_memory_utilization=0.85
# INFO 11-27 15:32:10 gpu_executor.py:113] # GPU blocks: 471, # CPU blocks: 655
# INFO 11-27 15:32:10 gpu_executor.py:117] Maximum concurrency for 32768 tokens per request: 0.23x
# ERROR 11-27 15:32:10 engine.py:366] The model's max seq len (32768) is larger than the maximum number of tokens that can be stored in KV cache (7536). Try increasing `gpu_memory_utilization` or decreasing `max_model_len` when initializing the engine.

# After increasing gpu utilization to 0.9, the maximum token for this model is: 9456
# Using NVIDIA 1xL4 (8vCPU 30GB RAM 24GB VRAM) still only support 23712 tokens.
# Using NVIDIA 1xL40S (8vCPU 62GB RAM 48GB VRAM) can support 32768 token. (Increasing RAM not works, only increasing VRAM works).

# 7536tokens÷1.2=6280words.
# 6280words÷500words/page=12.56pages. (For single-spaced)
#
# 9456tokens÷1.2=7880words.
# 7880words÷500words/page=15.76pages. (For single-spaced)
python -u /app/openai_compatible_api_server.py \
    --model sail/Sailor-4B-Chat \
    --revision 89a866a7041e6ec023dd462adeca8e28dd53c83e \
    --seed 42 \
    --host 0.0.0.0 \
    --port 7860 \
    --max-num-batched-tokens 32768 \
    --max-model-len 32768 \
    --dtype float16 \
    --gpu-memory-utilization 0.85