A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[CI/Build] AMD CI pipeline with extended set of tests. (#4267)
Co-authored-by: simon-mo <simon.mo@hey.com>
Alexei-V-Ivanov-AMD committed
May 2, 2024 at 14:29 UTC
9b5c9f9484858279a937498ebf9239a9df67f61f
5 files changed
+67
-45
.buildkite/run-amd-test.sh
+25
-33
@@ -1,10 +1,11 @@
1
-# This script build the ROCm docker image and run the API server inside the container.
2
-# It serves a sanity check for compilation and basic model usage.
1
+# This script build the ROCm docker image and runs test inside it.
2
set -ex
3
4
# Print ROCm version
5
+echo "--- ROCm info"
6
rocminfo
7
8
+echo "--- Resetting GPUs"
9
10
echo "reset" > /opt/amdgpu/etc/gpu_state
11
@@ -16,37 +17,28 @@ while true; do
17
fi
18
done
19
20
+echo "--- Building container"
21
+sha=$(git rev-parse --short HEAD)
22
+container_name=rocm_${sha}
23
+docker build \
24
+ -t ${container_name} \
25
+ -f Dockerfile.rocm \
26
+ --progress plain \
27
+ .
28
+
29
+remove_docker_container() {
30
+ docker rm -f ${container_name} || docker image rm -f ${container_name} || true
31
+}
32
+trap remove_docker_container EXIT
33
34
+echo "--- Running container"
35
21
-# Try building the docker image
22
-docker build -t rocm -f Dockerfile.rocm .
23
-
24
-# Setup cleanup
25
-remove_docker_container() { docker rm -f rocm || true; }
26
-trap remove_docker_container EXIT
27
-remove_docker_container
28
-
29
-# Run the image
30
-export HIP_VISIBLE_DEVICES=1
31
-docker run --device /dev/kfd --device /dev/dri --network host -e HIP_VISIBLE_DEVICES --name rocm rocm python3 -m vllm.entrypoints.api_server &
32
-
33
-# Wait for the server to start
34
-wait_for_server_to_start() {
35
- timeout=300
36
- counter=0
37
-
38
- while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8000/health)" != "200" ]; do
39
- sleep 1
40
- counter=$((counter + 1))
41
- if [ $counter -ge $timeout ]; then
42
- echo "Timeout after $timeout seconds"
43
- break
44
- fi
45
- done
46
-}
47
-wait_for_server_to_start
36
+docker run \
37
+ --device /dev/kfd --device /dev/dri \
38
+ --network host \
39
+ --rm \
40
+ -e HF_TOKEN \
41
+ --name ${container_name} \
42
+ ${container_name} \
43
+ /bin/bash -c $(echo $1 | sed "s/^'//" | sed "s/'$//")
44
49
-# Test a simple prompt
50
-curl -X POST -H "Content-Type: application/json" \
51
- localhost:8000/generate \
52
- -d '{"prompt": "San Francisco is a"}'
.buildkite/run-benchmarks.sh
+5
@@ -53,6 +53,11 @@ echo '```' >> benchmark_results.md
53
tail -n 20 benchmark_serving.txt >> benchmark_results.md # last 20 lines
54
echo '```' >> benchmark_results.md
55
56
+# if the agent binary is not found, skip uploading the results, exit 0
57
+if [ ! -f /workspace/buildkite-agent ]; then
58
+ exit 0
59
+fi
60
+
61
# upload the results to buildkite
62
/workspace/buildkite-agent annotate --style "info" --context "benchmark-results" < benchmark_results.md
63
.buildkite/test-pipeline.yaml
+14
-1
@@ -20,6 +20,7 @@ steps:
20
- VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1 pytest -v -s basic_correctness/test_preemption.py
21
22
- label: Core Test
23
+ mirror_hardwares: [amd]
24
command: pytest -v -s core
25
26
- label: Distributed Comm Ops Test
@@ -29,7 +30,10 @@ steps:
30
31
- label: Distributed Tests
32
working_dir: "/vllm-workspace/tests/distributed"
32
- num_gpus: 2
33
+
34
+ num_gpus: 2 # only support 1 or 2 for now.
35
+ mirror_hardwares: [amd]
36
+
37
commands:
38
- pytest -v -s test_pynccl_library.py
39
- TEST_DIST_MODEL=facebook/opt-125m pytest -v -s test_basic_distributed_correctness.py
@@ -44,6 +48,7 @@ steps:
48
- pytest -v -s test_pynccl.py
49
50
- label: Engine Test
51
+ mirror_hardwares: [amd]
52
command: pytest -v -s engine tokenization test_sequence.py test_config.py test_logger.py
53
54
- label: Entrypoints Test
@@ -54,6 +59,7 @@ steps:
59
60
- label: Examples Test
61
working_dir: "/vllm-workspace/examples"
62
+ mirror_hardwares: [amd]
63
commands:
64
# install aws cli for llava_example.py
65
- pip install awscli
@@ -67,16 +73,19 @@ steps:
73
parallelism: 4
74
75
- label: Models Test
76
+ mirror_hardwares: [amd]
77
commands:
78
- bash ../.buildkite/download-images.sh
79
- pytest -v -s models --ignore=models/test_llava.py --ignore=models/test_mistral.py
80
81
- label: Llava Test
82
+ mirror_hardwares: [amd]
83
commands:
84
- bash ../.buildkite/download-images.sh
85
- pytest -v -s models/test_llava.py
86
87
- label: Prefix Caching Test
88
+ mirror_hardwares: [amd]
89
commands:
90
- pytest -v -s prefix_caching
91
@@ -84,12 +93,15 @@ steps:
93
command: pytest -v -s samplers
94
95
- label: LogitsProcessor Test
96
+ mirror_hardwares: [amd]
97
command: pytest -v -s test_logits_processor.py
98
99
- label: Worker Test
100
+ mirror_hardwares: [amd]
101
command: pytest -v -s worker
102
103
- label: Speculative decoding tests
104
+ mirror_hardwares: [amd]
105
command: pytest -v -s spec_decode
106
107
- label: LoRA Test %N
@@ -107,6 +119,7 @@ steps:
119
120
- label: Benchmarks
121
working_dir: "/vllm-workspace/.buildkite"
122
+ mirror_hardwares: [amd]
123
commands:
124
- pip install aiohttp
125
- bash run-benchmarks.sh
.buildkite/test-template.j2
+16
-5
@@ -16,18 +16,29 @@ steps:
16
limit: 5
17
- wait
18
19
- - label: "AMD Test"
20
- agents:
21
- queue: amd
22
- command: bash .buildkite/run-amd-test.sh
19
+ - group: "AMD Tests"
20
+ depends_on: ~
21
+ steps:
22
+ {% for step in steps %}
23
+ {% if step.mirror_hardwares and "amd" in step.mirror_hardwares %}
24
+ - label: "AMD: {{ step.label }}"
25
+ agents:
26
+ queue: amd
27
+ command: bash .buildkite/run-amd-test.sh "'cd {{ (step.working_dir or default_working_dir) | safe }} && {{ step.command or (step.commands | join(' && ')) | safe }}'"
28
+ env:
29
+ DOCKER_BUILDKIT: "1"
30
+ {% endif %}
31
+ {% endfor %}
32
33
- label: "Neuron Test"
34
+ depends_on: ~
35
agents:
36
queue: neuron
37
command: bash .buildkite/run-neuron-test.sh
38
soft_fail: true
39
30
- - label: "CPU Test"
40
+ - label: "Intel Test"
41
+ depends_on: ~
42
command: bash .buildkite/run-cpu-test.sh
43
44
{% for step in steps %}
Dockerfile.rocm
+7
-6
@@ -46,7 +46,7 @@ RUN apt-get update && apt-get install -y \
46
47
### Mount Point ###
48
# When launching the container, mount the code directory to /app
49
-ARG APP_MOUNT=/app
49
+ARG APP_MOUNT=/vllm-workspace
50
VOLUME [ ${APP_MOUNT} ]
51
WORKDIR ${APP_MOUNT}
52
@@ -89,15 +89,16 @@ RUN if [ "$BUILD_TRITON" = "1" ]; then \
89
&& cd ../..; \
90
fi
91
92
-COPY ./ /app/vllm
92
+WORKDIR /vllm-workspace
93
+COPY . .
94
95
RUN python3 -m pip install --upgrade pip numba
96
96
-RUN cd /app \
97
- && cd vllm \
98
- && pip install -U -r requirements-rocm.txt \
99
- && patch /opt/rocm/include/hip/amd_detail/amd_hip_bf16.h /app/vllm/rocm_patch/rocm_bf16.patch \
97
+RUN --mount=type=cache,target=/root/.cache/pip \
98
+ pip install -U -r requirements-rocm.txt \
99
+ && patch /opt/rocm/include/hip/amd_detail/amd_hip_bf16.h ./rocm_patch/rocm_bf16.patch \
100
&& python3 setup.py install \
101
+ && cp build/lib.linux-x86_64-cpython-39/vllm/_C.cpython-39-x86_64-linux-gnu.so vllm/ \
102
&& cd ..
103
104
RUN python3 -m pip install --upgrade pip