A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
CI: Add ROCm Docker Build (#2886)
Simon Mo committed
Mar 18, 2024 at 12:33 UTC
8c654c045f73198a517becd8b1b23a9b16eae284
3 files changed
+44
.buildkite/run-amd-test.sh
new
+38
@@ -0,0 +1,38 @@
1
+# This script build the ROCm docker image and run the API server inside the container.
2
+# It serves a sanity check for compilation and basic model usage.
3
+set -ex
4
+
5
+# Print ROCm version
6
+rocminfo
7
+
8
+# Try building the docker image
9
+docker build -t rocm -f Dockerfile.rocm .
10
+
11
+# Setup cleanup
12
+remove_docker_container() { docker rm -f rocm || true; }
13
+trap remove_docker_container EXIT
14
+remove_docker_container
15
+
16
+# Run the image
17
+docker run --device /dev/kfd --device /dev/dri --network host --name rocm rocm python3 -m vllm.entrypoints.api_server &
18
+
19
+# Wait for the server to start
20
+wait_for_server_to_start() {
21
+ timeout=300
22
+ counter=0
23
+
24
+ while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8000/health)" != "200" ]; do
25
+ sleep 1
26
+ counter=$((counter + 1))
27
+ if [ $counter -ge $timeout ]; then
28
+ echo "Timeout after $timeout seconds"
29
+ break
30
+ fi
31
+ done
32
+}
33
+wait_for_server_to_start
34
+
35
+# Test a simple prompt
36
+curl -X POST -H "Content-Type: application/json" \
37
+ localhost:8000/generate \
38
+ -d '{"prompt": "San Francisco is a"}'
.buildkite/test-template.j2
+5
@@ -3,6 +3,11 @@
3
{% set default_working_dir = "/vllm-workspace/tests" %}
4
5
steps:
6
+ - label: "AMD Test"
7
+ agents:
8
+ queue: amd
9
+ command: bash .buildkite/run-amd-test.sh
10
+
11
- label: ":docker: build image"
12
commands:
13
- "docker build --build-arg max_jobs=16 --tag {{ docker_image }} --target test --progress plain ."
requirements-rocm.txt
+1
@@ -11,3 +11,4 @@ fastapi
11
uvicorn[standard]
12
pydantic >= 2.0 # Required for OpenAI server.
13
prometheus_client >= 0.18.0
14
+outlines == 0.0.34
\ No newline at end of file