yusufs commited on
Commit
69c6372
·
1 Parent(s): d4b0956

feat(runner.sh): using runner.sh to select llm in the run time

Browse files
Files changed (2) hide show
  1. Dockerfile +2 -1
  2. runner.sh +35 -0
Dockerfile CHANGED
@@ -29,7 +29,8 @@ EXPOSE 7860
29
 
30
  #CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]
31
 
 
32
  RUN chmod +x /app/run-llama.sh
33
  RUN chmod +x /app/run-sailor.sh
34
 
35
- CMD ["/app/run-llama.sh"]
 
29
 
30
  #CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]
31
 
32
+ RUN chmod +x /app/runner.sh
33
  RUN chmod +x /app/run-llama.sh
34
  RUN chmod +x /app/run-sailor.sh
35
 
36
+ CMD ["/app/runner.sh"]
runner.sh ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+
3
+ # Check if MODEL_NAME is set and has a valid value
4
+ if [[ -z "$MODEL_NAME" ]]; then
5
+ echo "Error: MODEL_NAME is not set."
6
+ exit 1
7
+ fi
8
+
9
+ if [[ "$MODEL_NAME" != "meta-llama/Llama-3.2-3B-Instruct" && "$MODEL_NAME" != "sail/Sailor-4B-Chat" ]]; then
10
+ echo "Error: Invalid value for MODEL_NAME. Valid values are:"
11
+ echo " - meta-llama/Llama-3.2-3B-Instruct"
12
+ echo " - sail/Sailor-4B-Chat"
13
+ exit 1
14
+ fi
15
+
16
+ # Check if MODEL_REV is set
17
+ if [[ -z "$MODEL_REV" ]]; then
18
+ echo "Error: MODEL_REV is not set."
19
+ exit 1
20
+ fi
21
+
22
+ printf "Running %s using vLLM OpenAI compatible API Server at port %s\n" $MODEL_NAME "7860"
23
+
24
+ # Run the Python script with the validated environment variables
25
+ python -u /app/openai_compatible_api_server.py \
26
+ --model "${MODEL_NAME}" \
27
+ --revision "${MODEL_REV}" \
28
+ --seed 42 \
29
+ --host 0.0.0.0 \
30
+ --port 7860 \
31
+ --max-num-batched-tokens 32768 \
32
+ --max-model-len 32768 \
33
+ --dtype float16 \
34
+ --enforce-eager \
35
+ --gpu-memory-utilization 0.9