Spaces:
Paused
Paused
feat(runner.sh): using runner.sh to select llm in the run time
Browse files- Dockerfile +2 -1
- runner.sh +35 -0
Dockerfile
CHANGED
@@ -29,7 +29,8 @@ EXPOSE 7860
|
|
29 |
|
30 |
#CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]
|
31 |
|
|
|
32 |
RUN chmod +x /app/run-llama.sh
|
33 |
RUN chmod +x /app/run-sailor.sh
|
34 |
|
35 |
-
CMD ["/app/
|
|
|
29 |
|
30 |
#CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]
|
31 |
|
32 |
+
RUN chmod +x /app/runner.sh
|
33 |
RUN chmod +x /app/run-llama.sh
|
34 |
RUN chmod +x /app/run-sailor.sh
|
35 |
|
36 |
+
CMD ["/app/runner.sh"]
|
runner.sh
ADDED
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#!/bin/bash
|
2 |
+
|
3 |
+
# Check if MODEL_NAME is set and has a valid value
|
4 |
+
if [[ -z "$MODEL_NAME" ]]; then
|
5 |
+
echo "Error: MODEL_NAME is not set."
|
6 |
+
exit 1
|
7 |
+
fi
|
8 |
+
|
9 |
+
if [[ "$MODEL_NAME" != "meta-llama/Llama-3.2-3B-Instruct" && "$MODEL_NAME" != "sail/Sailor-4B-Chat" ]]; then
|
10 |
+
echo "Error: Invalid value for MODEL_NAME. Valid values are:"
|
11 |
+
echo " - meta-llama/Llama-3.2-3B-Instruct"
|
12 |
+
echo " - sail/Sailor-4B-Chat"
|
13 |
+
exit 1
|
14 |
+
fi
|
15 |
+
|
16 |
+
# Check if MODEL_REV is set
|
17 |
+
if [[ -z "$MODEL_REV" ]]; then
|
18 |
+
echo "Error: MODEL_REV is not set."
|
19 |
+
exit 1
|
20 |
+
fi
|
21 |
+
|
22 |
+
printf "Running %s using vLLM OpenAI compatible API Server at port %s\n" $MODEL_NAME "7860"
|
23 |
+
|
24 |
+
# Run the Python script with the validated environment variables
|
25 |
+
python -u /app/openai_compatible_api_server.py \
|
26 |
+
--model "${MODEL_NAME}" \
|
27 |
+
--revision "${MODEL_REV}" \
|
28 |
+
--seed 42 \
|
29 |
+
--host 0.0.0.0 \
|
30 |
+
--port 7860 \
|
31 |
+
--max-num-batched-tokens 32768 \
|
32 |
+
--max-model-len 32768 \
|
33 |
+
--dtype float16 \
|
34 |
+
--enforce-eager \
|
35 |
+
--gpu-memory-utilization 0.9
|