@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9b5c9f9484
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[CI/Build] AMD CI pipeline with extended set of tests. (#4267)

Co-authored-by: simon-mo <simon.mo@hey.com>

Alexei-V-Ivanov-AMD committed May 2, 2024 at 14:29 UTC 9b5c9f9484858279a937498ebf9239a9df67f61f
5 files changed +67 -45
.buildkite/run-amd-test.sh
+25 -33
@@ -1,10 +1,11 @@
1 -# This script build the ROCm docker image and run the API server inside the container.
2 -# It serves a sanity check for compilation and basic model usage.
1 +# This script build the ROCm docker image and runs test inside it.
2 set -ex
3
4 # Print ROCm version
5 +echo "--- ROCm info"
6 rocminfo
7
8 +echo "--- Resetting GPUs"
9
10 echo "reset" > /opt/amdgpu/etc/gpu_state
11
@@ -16,37 +17,28 @@ while true; do
17 fi
18 done
19
20 +echo "--- Building container"
21 +sha=$(git rev-parse --short HEAD)
22 +container_name=rocm_${sha}
23 +docker build \
24 + -t ${container_name} \
25 + -f Dockerfile.rocm \
26 + --progress plain \
27 + .
28 +
29 +remove_docker_container() {
30 + docker rm -f ${container_name} || docker image rm -f ${container_name} || true
31 +}
32 +trap remove_docker_container EXIT
33
34 +echo "--- Running container"
35
21 -# Try building the docker image
22 -docker build -t rocm -f Dockerfile.rocm .
23 -
24 -# Setup cleanup
25 -remove_docker_container() { docker rm -f rocm || true; }
26 -trap remove_docker_container EXIT
27 -remove_docker_container
28 -
29 -# Run the image
30 -export HIP_VISIBLE_DEVICES=1
31 -docker run --device /dev/kfd --device /dev/dri --network host -e HIP_VISIBLE_DEVICES --name rocm rocm python3 -m vllm.entrypoints.api_server &
32 -
33 -# Wait for the server to start
34 -wait_for_server_to_start() {
35 - timeout=300
36 - counter=0
37 -
38 - while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8000/health)" != "200" ]; do
39 - sleep 1
40 - counter=$((counter + 1))
41 - if [ $counter -ge $timeout ]; then
42 - echo "Timeout after $timeout seconds"
43 - break
44 - fi
45 - done
46 -}
47 -wait_for_server_to_start
36 +docker run \
37 + --device /dev/kfd --device /dev/dri \
38 + --network host \
39 + --rm \
40 + -e HF_TOKEN \
41 + --name ${container_name} \
42 + ${container_name} \
43 + /bin/bash -c $(echo $1 | sed "s/^'//" | sed "s/'$//")
44
49 -# Test a simple prompt
50 -curl -X POST -H "Content-Type: application/json" \
51 - localhost:8000/generate \
52 - -d '{"prompt": "San Francisco is a"}'
.buildkite/run-benchmarks.sh
+5
@@ -53,6 +53,11 @@ echo '```' >> benchmark_results.md
53 tail -n 20 benchmark_serving.txt >> benchmark_results.md # last 20 lines
54 echo '```' >> benchmark_results.md
55
56 +# if the agent binary is not found, skip uploading the results, exit 0
57 +if [ ! -f /workspace/buildkite-agent ]; then
58 + exit 0
59 +fi
60 +
61 # upload the results to buildkite
62 /workspace/buildkite-agent annotate --style "info" --context "benchmark-results" < benchmark_results.md
63
.buildkite/test-pipeline.yaml
+14 -1
@@ -20,6 +20,7 @@ steps:
20 - VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1 pytest -v -s basic_correctness/test_preemption.py
21
22 - label: Core Test
23 + mirror_hardwares: [amd]
24 command: pytest -v -s core
25
26 - label: Distributed Comm Ops Test
@@ -29,7 +30,10 @@ steps:
30
31 - label: Distributed Tests
32 working_dir: "/vllm-workspace/tests/distributed"
32 - num_gpus: 2
33 +
34 + num_gpus: 2 # only support 1 or 2 for now.
35 + mirror_hardwares: [amd]
36 +
37 commands:
38 - pytest -v -s test_pynccl_library.py
39 - TEST_DIST_MODEL=facebook/opt-125m pytest -v -s test_basic_distributed_correctness.py
@@ -44,6 +48,7 @@ steps:
48 - pytest -v -s test_pynccl.py
49
50 - label: Engine Test
51 + mirror_hardwares: [amd]
52 command: pytest -v -s engine tokenization test_sequence.py test_config.py test_logger.py
53
54 - label: Entrypoints Test
@@ -54,6 +59,7 @@ steps:
59
60 - label: Examples Test
61 working_dir: "/vllm-workspace/examples"
62 + mirror_hardwares: [amd]
63 commands:
64 # install aws cli for llava_example.py
65 - pip install awscli
@@ -67,16 +73,19 @@ steps:
73 parallelism: 4
74
75 - label: Models Test
76 + mirror_hardwares: [amd]
77 commands:
78 - bash ../.buildkite/download-images.sh
79 - pytest -v -s models --ignore=models/test_llava.py --ignore=models/test_mistral.py
80
81 - label: Llava Test
82 + mirror_hardwares: [amd]
83 commands:
84 - bash ../.buildkite/download-images.sh
85 - pytest -v -s models/test_llava.py
86
87 - label: Prefix Caching Test
88 + mirror_hardwares: [amd]
89 commands:
90 - pytest -v -s prefix_caching
91
@@ -84,12 +93,15 @@ steps:
93 command: pytest -v -s samplers
94
95 - label: LogitsProcessor Test
96 + mirror_hardwares: [amd]
97 command: pytest -v -s test_logits_processor.py
98
99 - label: Worker Test
100 + mirror_hardwares: [amd]
101 command: pytest -v -s worker
102
103 - label: Speculative decoding tests
104 + mirror_hardwares: [amd]
105 command: pytest -v -s spec_decode
106
107 - label: LoRA Test %N
@@ -107,6 +119,7 @@ steps:
119
120 - label: Benchmarks
121 working_dir: "/vllm-workspace/.buildkite"
122 + mirror_hardwares: [amd]
123 commands:
124 - pip install aiohttp
125 - bash run-benchmarks.sh
.buildkite/test-template.j2
+16 -5
@@ -16,18 +16,29 @@ steps:
16 limit: 5
17 - wait
18
19 - - label: "AMD Test"
20 - agents:
21 - queue: amd
22 - command: bash .buildkite/run-amd-test.sh
19 + - group: "AMD Tests"
20 + depends_on: ~
21 + steps:
22 + {% for step in steps %}
23 + {% if step.mirror_hardwares and "amd" in step.mirror_hardwares %}
24 + - label: "AMD: {{ step.label }}"
25 + agents:
26 + queue: amd
27 + command: bash .buildkite/run-amd-test.sh "'cd {{ (step.working_dir or default_working_dir) | safe }} && {{ step.command or (step.commands | join(' && ')) | safe }}'"
28 + env:
29 + DOCKER_BUILDKIT: "1"
30 + {% endif %}
31 + {% endfor %}
32
33 - label: "Neuron Test"
34 + depends_on: ~
35 agents:
36 queue: neuron
37 command: bash .buildkite/run-neuron-test.sh
38 soft_fail: true
39
30 - - label: "CPU Test"
40 + - label: "Intel Test"
41 + depends_on: ~
42 command: bash .buildkite/run-cpu-test.sh
43
44 {% for step in steps %}
Dockerfile.rocm
+7 -6
@@ -46,7 +46,7 @@ RUN apt-get update && apt-get install -y \
46
47 ### Mount Point ###
48 # When launching the container, mount the code directory to /app
49 -ARG APP_MOUNT=/app
49 +ARG APP_MOUNT=/vllm-workspace
50 VOLUME [ ${APP_MOUNT} ]
51 WORKDIR ${APP_MOUNT}
52
@@ -89,15 +89,16 @@ RUN if [ "$BUILD_TRITON" = "1" ]; then \
89 && cd ../..; \
90 fi
91
92 -COPY ./ /app/vllm
92 +WORKDIR /vllm-workspace
93 +COPY . .
94
95 RUN python3 -m pip install --upgrade pip numba
96
96 -RUN cd /app \
97 - && cd vllm \
98 - && pip install -U -r requirements-rocm.txt \
99 - && patch /opt/rocm/include/hip/amd_detail/amd_hip_bf16.h /app/vllm/rocm_patch/rocm_bf16.patch \
97 +RUN --mount=type=cache,target=/root/.cache/pip \
98 + pip install -U -r requirements-rocm.txt \
99 + && patch /opt/rocm/include/hip/amd_detail/amd_hip_bf16.h ./rocm_patch/rocm_bf16.patch \
100 && python3 setup.py install \
101 + && cp build/lib.linux-x86_64-cpython-39/vllm/_C.cpython-39-x86_64-linux-gnu.so vllm/ \
102 && cd ..
103
104 RUN python3 -m pip install --upgrade pip