@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8c654c045f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

CI: Add ROCm Docker Build (#2886)

Simon Mo committed Mar 18, 2024 at 12:33 UTC 8c654c045f73198a517becd8b1b23a9b16eae284
3 files changed +44
.buildkite/run-amd-test.sh new
+38
@@ -0,0 +1,38 @@
1 +# This script build the ROCm docker image and run the API server inside the container.
2 +# It serves a sanity check for compilation and basic model usage.
3 +set -ex
4 +
5 +# Print ROCm version
6 +rocminfo
7 +
8 +# Try building the docker image
9 +docker build -t rocm -f Dockerfile.rocm .
10 +
11 +# Setup cleanup
12 +remove_docker_container() { docker rm -f rocm || true; }
13 +trap remove_docker_container EXIT
14 +remove_docker_container
15 +
16 +# Run the image
17 +docker run --device /dev/kfd --device /dev/dri --network host --name rocm rocm python3 -m vllm.entrypoints.api_server &
18 +
19 +# Wait for the server to start
20 +wait_for_server_to_start() {
21 + timeout=300
22 + counter=0
23 +
24 + while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8000/health)" != "200" ]; do
25 + sleep 1
26 + counter=$((counter + 1))
27 + if [ $counter -ge $timeout ]; then
28 + echo "Timeout after $timeout seconds"
29 + break
30 + fi
31 + done
32 +}
33 +wait_for_server_to_start
34 +
35 +# Test a simple prompt
36 +curl -X POST -H "Content-Type: application/json" \
37 + localhost:8000/generate \
38 + -d '{"prompt": "San Francisco is a"}'
.buildkite/test-template.j2
+5
@@ -3,6 +3,11 @@
3 {% set default_working_dir = "/vllm-workspace/tests" %}
4
5 steps:
6 + - label: "AMD Test"
7 + agents:
8 + queue: amd
9 + command: bash .buildkite/run-amd-test.sh
10 +
11 - label: ":docker: build image"
12 commands:
13 - "docker build --build-arg max_jobs=16 --tag {{ docker_image }} --target test --progress plain ."
requirements-rocm.txt
+1
@@ -11,3 +11,4 @@ fastapi
11 uvicorn[standard]
12 pydantic >= 2.0 # Required for OpenAI server.
13 prometheus_client >= 0.18.0
14 +outlines == 0.0.34
\ No newline at end of file