feat(add-model): always download model during build, it will be cached in the consecutive builds

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.12
+FROM python:3.12.7-slim-bookworm
 
 RUN useradd -m -u 1000 user
 USER user
@@ -27,5 +27,7 @@ EXPOSE 7860
 
 #CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]
 
-RUN chmod +x /app/run.sh
-CMD ["/app/run.sh"]
+RUN chmod +x /app/run-llama.sh
+RUN chmod +x /app/run-sailor.sh
+
+CMD ["/app/run-sailor.sh"]

README.md

@@ -21,14 +21,13 @@ poetry export -f requirements.txt --output requirements.txt --without-hashes
 
 > References: https://huggingface.co/spaces/sofianhw/ai/tree/c6527a750644a849b6705bb6fe2fcea4e54a8196
 
-Fixes:
+This `api_server.py` file is exact copy version from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/entrypoints/openai/api_server.py
+
+Changes (use diff tool to see the exact changes of the file):
 
 * [x] change everything route in api_server.py that start (“/v1/xxx”) to (“/api/v1/xxx”).
   and just run the python api_server.py with arguments. https://discuss.huggingface.co/t/run-vllm-docker-on-space/70228/5?u=yusufs
 
-This `api_server.py` file is exact copy version from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/entrypoints/openai/api_server.py
-
-
 
 ## Documentation about config
 

download_model.py

@@ -10,6 +10,16 @@ hf_token = hf_token.strip()
 if hf_token == "":
     raise ValueError("HF_TOKEN is empty")
 
+
+# This is about 2.47 GB
+snapshot_download(
+    repo_id="meta-llama/Llama-3.2-1B-Instruct",
+    revision="9213176726f574b556790deb65791e0c5aa438b6",
+    token=hf_token,
+)
+
+
+# This is about 3.67 GB
 snapshot_download(
     repo_id="sail/Sailor-4B-Chat",
     revision="89a866a7041e6ec023dd462adeca8e28dd53c83e",

run-llama.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+
+
+printf "Running meta-llama/Llama-3.2-1B-Instruct using vLLM OpenAI compatible API Server at port %s\n" "7860"
+
+# Llama-3.2-3B-Instruct max context length is 131072, but we reduce it to 32k.
+# 32k tokens, 3/4 of 32k is 24k words, each page average is 500 or 0.5k words,
+# so that's basically 24k / .5k = 24 x 2 =~48 pages.
+# Because when we use maximum token length, it will be slower and the memory is not enough for T4.
+# https://github.com/vllm-project/vllm/blob/v0.6.4/vllm/config.py#L85-L86
+# https://github.com/vllm-project/vllm/blob/v0.6.4/vllm/config.py#L98-L102
+# [rank0]:     raise ValueError(
+# [rank0]: ValueError: The model's max seq len (131072)
+#   is larger than the maximum number of tokens that can be stored in KV cache (57056).
+#   Try increasing `gpu_memory_utilization` or decreasing `max_model_len` when initializing the engine.
+#
+# Actually, the meta-llama/Llama-3.2-3B-Instruct rev 0cb88a4f764b7a12671c53f0838cd831a0843b95
+# is enough with T4 16GB, but for the sake of the performance and comparing with the same
+# params with the sail/Sailor-1.8B-Chat, I use the
+# meta-llama/Llama-3.2-1B-Instruct rev 9213176726f574b556790deb65791e0c5aa438b6
+python -u /app/openai_compatible_api_server.py \
+    --model meta-llama/Llama-3.2-1B-Instruct \
+    --revision 9213176726f574b556790deb65791e0c5aa438b6 \
+    --host 0.0.0.0 \
+    --port 7860 \
+    --max-num-batched-tokens 32768 \
+    --max-model-len 32768 \
+    --dtype half \
+    --enforce-eager \
+    --gpu-memory-utilization 0.85
+

run.sh → run-sailor.sh

@@ -1,20 +1,9 @@
 #!/bin/sh
 
 
-printf "Running vLLM OpenAI compatible API Server at port %s\n" "7860"
-
-#python -u /app/openai_compatible_api_server.py \
-#    --model meta-llama/Llama-3.2-3B-Instruct \
-#    --revision 0cb88a4f764b7a12671c53f0838cd831a0843b95 \
-#    --host 0.0.0.0 \
-#    --port 7860 \
-#    --max-num-batched-tokens 32768 \
-#    --max-model-len 32768 \
-#    --dtype half \
-#    --enforce-eager \
-#    --gpu-memory-utilization 0.85
-
+printf "Running sail/Sailor-1.8B-Chat using vLLM OpenAI compatible API Server at port %s\n" "7860"
 
+# Using the 1.8B version because the 4B version cannot be loaded due to memory constraints.
 # Reducing max-num-batched-tokens to 7536 because got this error:
 # INFO 11-27 15:32:01 model_runner.py:1077] Loading model weights took 7.4150 GB
 # INFO 11-27 15:32:09 worker.py:232] Memory profiling results: total_gpu_memory=14.58GiB initial_memory_usage=7.61GiB peak_torch_memory=9.31GiB memory_usage_post_profile=7.62GiB non_torch_memory=0.20GiB kv_cache_size=2.88GiB gpu_memory_utilization=0.85
@@ -22,11 +11,12 @@ printf "Running vLLM OpenAI compatible API Server at port %s\n" "7860"
 # INFO 11-27 15:32:10 gpu_executor.py:117] Maximum concurrency for 32768 tokens per request: 0.23x
 # ERROR 11-27 15:32:10 engine.py:366] The model's max seq len (32768) is larger than the maximum number of tokens that can be stored in KV cache (7536). Try increasing `gpu_memory_utilization` or decreasing `max_model_len` when initializing the engine.
 python -u /app/openai_compatible_api_server.py \
-    --model sail/Sailor-4B-Chat \
-    --revision 89a866a7041e6ec023dd462adeca8e28dd53c83e \
+    --model sail/Sailor-1.8B-Chat \
+    --revision 04b86803d4011d4bfd80f3cd3841b005eb899987 \
     --host 0.0.0.0 \
     --port 7860 \
-    --max-num-batched-tokens 7536 \
+    --max-num-batched-tokens 32768 \
+    --max-model-len 32768 \
     --dtype half \
     --enforce-eager \
     --gpu-memory-utilization 0.85