@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 067c5d9ad1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 7, 19:00 UTC

[ROCm][CI] Added MI325 mirrors (#34923)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>

Andreas Karatzas committed Feb 24, 2026 at 15:37 UTC 067c5d9ad1be71a36f7878631d97d7239dd097ab
7 files changed +299 -168
.buildkite/scripts/hardware_ci/run-amd-test.sh
+237 -163
@@ -1,25 +1,37 @@
1 #!/bin/bash
2
3 -# This script runs test inside the corresponding ROCm docker container.
3 +# This script runs tests inside the corresponding ROCm docker container.
4 +# It handles both single-node and multi-node test configurations.
5 +#
6 +# Multi-node detection: Instead of matching on fragile group names, we detect
7 +# multi-node jobs structurally by looking for the bracket command syntax
8 +# "[node0_cmds] && [node1_cmds]" or via the NUM_NODES environment variable.
9 set -o pipefail
10
11 # Export Python path
12 export PYTHONPATH=".."
13
9 -# Print ROCm version
10 -echo "--- Confirming Clean Initial State"
11 -while true; do
12 - sleep 3
13 - if grep -q clean /opt/amdgpu/etc/gpu_state; then
14 - echo "GPUs state is \"clean\""
15 - break
16 - fi
17 -done
18 -
19 -echo "--- ROCm info"
20 -rocminfo
14 +###############################################################################
15 +# Helper Functions
16 +###############################################################################
17 +
18 +wait_for_clean_gpus() {
19 + local timeout=${1:-300}
20 + local start=$SECONDS
21 + echo "--- Waiting for clean GPU state (timeout: ${timeout}s)"
22 + while true; do
23 + if grep -q clean /opt/amdgpu/etc/gpu_state; then
24 + echo "GPUs state is \"clean\""
25 + return
26 + fi
27 + if (( SECONDS - start >= timeout )); then
28 + echo "Error: GPUs did not reach clean state within ${timeout}s" >&2
29 + exit 1
30 + fi
31 + sleep 3
32 + done
33 +}
34
22 -# cleanup older docker images
35 cleanup_docker() {
36 # Get Docker's root directory
37 docker_root=$(docker info -f '{{.DockerRootDir}}')
@@ -28,15 +40,12 @@ cleanup_docker() {
40 exit 1
41 fi
42 echo "Docker root directory: $docker_root"
31 - # Check disk usage of the filesystem where Docker's root directory is located
43 +
44 disk_usage=$(df "$docker_root" | tail -1 | awk '{print $5}' | sed 's/%//')
33 - # Define the threshold
45 threshold=70
46 if [ "$disk_usage" -gt "$threshold" ]; then
47 echo "Disk usage is above $threshold%. Cleaning up Docker images and volumes..."
37 - # Remove dangling images (those that are not tagged and not used by any container)
48 docker image prune -f
39 - # Remove unused volumes / force the system prune for old images as well.
49 docker volume prune -f && docker system prune --force --filter "until=72h" --all
50 echo "Docker images and volumes cleanup completed."
51 else
@@ -45,193 +54,258 @@ cleanup_docker() {
54 }
55
56 cleanup_network() {
48 - for node in $(seq 0 $((NUM_NODES-1))); do
49 - if docker pr -a -q -f name="node${node}" | grep -q .; then
50 - docker stop "node${node}"
57 + local max_nodes=${NUM_NODES:-2}
58 + for node in $(seq 0 $((max_nodes - 1))); do
59 + if docker ps -a -q -f name="node${node}" | grep -q .; then
60 + docker stop "node${node}" || true
61 fi
62 done
53 - if docker network ls | grep docker-net; then
54 - docker network rm docker-net
63 + if docker network ls | grep -q docker-net; then
64 + docker network rm docker-net || true
65 fi
66 }
67
58 -# Call the cleanup docker function
68 +is_multi_node() {
69 + local cmds="$1"
70 + # Primary signal: NUM_NODES environment variable set by the pipeline
71 + if [[ "${NUM_NODES:-1}" -gt 1 ]]; then
72 + return 0
73 + fi
74 + # Fallback: detect the bracket syntax structurally
75 + # Pattern: [...] && [...] (per-node command arrays)
76 + if [[ "$cmds" =~ \[.*\].*\&\&.*\[.*\] ]]; then
77 + return 0
78 + fi
79 + return 1
80 +}
81 +
82 +###############################################################################
83 +# Pytest marker re-quoting
84 +#
85 +# When commands are passed through Buildkite -> shell -> $* -> bash -c,
86 +# quotes around pytest -m marker expressions get stripped:
87 +# pytest -v -s -m 'not cpu_test' v1/core
88 +# becomes:
89 +# pytest -v -s -m not cpu_test v1/core
90 +#
91 +# pytest then interprets "cpu_test" as a file path, not part of the marker.
92 +# This function detects unquoted multi-word marker expressions and re-quotes
93 +# them so they survive the final bash -c expansion.
94 +###############################################################################
95 +
96 +re_quote_pytest_markers() {
97 + local cmds="$1"
98 + # Pattern: -m not <identifier> -> -m 'not <identifier>'
99 + # Handles the common cases: 'not cpu_test', 'not slow_test', etc.
100 + cmds=$(echo "$cmds" | sed -E "s/-m not ([a-zA-Z_][a-zA-Z0-9_]*)/-m 'not \1'/g")
101 + echo "$cmds"
102 +}
103 +
104 +###############################################################################
105 +# ROCm-specific pytest command rewrites
106 +#
107 +# These apply ignore flags and environment overrides for tests that are not
108 +# yet supported or behave differently on ROCm hardware. Kept as a single
109 +# function so new exclusions are easy to add in one place.
110 +###############################################################################
111 +
112 +apply_rocm_test_overrides() {
113 + local cmds="$1"
114 +
115 + # --- Model registry filter ---
116 + if [[ $cmds == *"pytest -v -s models/test_registry.py"* ]]; then
117 + cmds=${cmds//"pytest -v -s models/test_registry.py"/"pytest -v -s models/test_registry.py -k 'not BambaForCausalLM and not GritLM and not Mamba2ForCausalLM and not Zamba2ForCausalLM'"}
118 + fi
119 +
120 + # --- LoRA: disable custom paged attention ---
121 + if [[ $cmds == *"pytest -v -s lora"* ]]; then
122 + cmds=${cmds//"pytest -v -s lora"/"VLLM_ROCM_CUSTOM_PAGED_ATTN=0 pytest -v -s lora"}
123 + fi
124 +
125 + # --- Kernel ignores ---
126 + if [[ $cmds == *" kernels/core"* ]]; then
127 + cmds="${cmds} \
128 + --ignore=kernels/core/test_fused_quant_layernorm.py \
129 + --ignore=kernels/core/test_permute_cols.py"
130 + fi
131 +
132 + if [[ $cmds == *" kernels/attention"* ]]; then
133 + cmds="${cmds} \
134 + --ignore=kernels/attention/test_attention_selector.py \
135 + --ignore=kernels/attention/test_encoder_decoder_attn.py \
136 + --ignore=kernels/attention/test_flash_attn.py \
137 + --ignore=kernels/attention/test_flashinfer.py \
138 + --ignore=kernels/attention/test_prefix_prefill.py \
139 + --ignore=kernels/attention/test_cascade_flash_attn.py \
140 + --ignore=kernels/attention/test_mha_attn.py \
141 + --ignore=kernels/attention/test_lightning_attn.py \
142 + --ignore=kernels/attention/test_attention.py"
143 + fi
144 +
145 + if [[ $cmds == *" kernels/quantization"* ]]; then
146 + cmds="${cmds} \
147 + --ignore=kernels/quantization/test_int8_quant.py \
148 + --ignore=kernels/quantization/test_machete_mm.py \
149 + --ignore=kernels/quantization/test_block_fp8.py \
150 + --ignore=kernels/quantization/test_block_int8.py \
151 + --ignore=kernels/quantization/test_marlin_gemm.py \
152 + --ignore=kernels/quantization/test_cutlass_scaled_mm.py \
153 + --ignore=kernels/quantization/test_int8_kernel.py"
154 + fi
155 +
156 + if [[ $cmds == *" kernels/mamba"* ]]; then
157 + cmds="${cmds} \
158 + --ignore=kernels/mamba/test_mamba_mixer2.py \
159 + --ignore=kernels/mamba/test_causal_conv1d.py \
160 + --ignore=kernels/mamba/test_mamba_ssm_ssd.py"
161 + fi
162 +
163 + if [[ $cmds == *" kernels/moe"* ]]; then
164 + cmds="${cmds} \
165 + --ignore=kernels/moe/test_moe.py \
166 + --ignore=kernels/moe/test_cutlass_moe.py \
167 + --ignore=kernels/moe/test_triton_moe_ptpc_fp8.py"
168 + fi
169 +
170 + # --- Entrypoint ignores ---
171 + if [[ $cmds == *" entrypoints/openai "* ]]; then
172 + cmds=${cmds//" entrypoints/openai "/" entrypoints/openai \
173 + --ignore=entrypoints/openai/test_audio.py \
174 + --ignore=entrypoints/openai/test_shutdown.py \
175 + --ignore=entrypoints/openai/test_completion.py \
176 + --ignore=entrypoints/openai/test_models.py \
177 + --ignore=entrypoints/openai/test_lora_adapters.py \
178 + --ignore=entrypoints/openai/test_return_tokens_as_ids.py \
179 + --ignore=entrypoints/openai/test_root_path.py \
180 + --ignore=entrypoints/openai/test_tokenization.py \
181 + --ignore=entrypoints/openai/test_prompt_validation.py "}
182 + fi
183 +
184 + if [[ $cmds == *" entrypoints/llm "* ]]; then
185 + cmds=${cmds//" entrypoints/llm "/" entrypoints/llm \
186 + --ignore=entrypoints/llm/test_chat.py \
187 + --ignore=entrypoints/llm/test_accuracy.py \
188 + --ignore=entrypoints/llm/test_init.py \
189 + --ignore=entrypoints/llm/test_prompt_validation.py "}
190 + fi
191 +
192 + # Clean up escaped newlines from --ignore appends
193 + cmds=$(echo "$cmds" | sed 's/ \\ / /g')
194 +
195 + echo "$cmds"
196 +}
197 +
198 +###############################################################################
199 +# Main
200 +###############################################################################
201 +
202 +# --- GPU initialization ---
203 +echo "--- Confirming Clean Initial State"
204 +wait_for_clean_gpus
205 +
206 +echo "--- ROCm info"
207 +rocminfo
208 +
209 +# --- Docker housekeeping ---
210 cleanup_docker
211
212 echo "--- Resetting GPUs"
62 -
213 echo "reset" > /opt/amdgpu/etc/gpu_state
214 +wait_for_clean_gpus
215
65 -while true; do
66 - sleep 3
67 - if grep -q clean /opt/amdgpu/etc/gpu_state; then
68 - echo "GPUs state is \"clean\""
69 - break
70 - fi
71 -done
72 -
216 +# --- Pull test image ---
217 echo "--- Pulling container"
218 image_name="rocm/vllm-ci:${BUILDKITE_COMMIT}"
219 container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)"
220 docker pull "${image_name}"
221
222 remove_docker_container() {
79 - docker rm -f "${container_name}" || docker image rm -f "${image_name}" || true
223 + docker rm -f "${container_name}" || docker image rm -f "${image_name}" || true
224 }
225 trap remove_docker_container EXIT
226
227 +# --- Prepare commands ---
228 echo "--- Running container"
229
230 HF_CACHE="$(realpath ~)/huggingface"
231 mkdir -p "${HF_CACHE}"
232 HF_MOUNT="/root/.cache/huggingface"
233
89 -commands=$@
234 +commands="$*"
235 echo "Raw commands: $commands"
236
92 -commands=${commands//"pytest -v -s basic_correctness/test_basic_correctness.py"/"pytest -v -s basic_correctness/test_basic_correctness.py"}
93 -
94 -if [[ $commands == *"pytest -v -s models/test_registry.py"* ]]; then
95 - commands=${commands//"pytest -v -s models/test_registry.py"/"pytest -v -s models/test_registry.py -k 'not BambaForCausalLM and not GritLM and not Mamba2ForCausalLM and not Zamba2ForCausalLM'"}
96 -fi
97 -
98 -commands=${commands//"pytest -v -s compile/test_basic_correctness.py"/"pytest -v -s compile/test_basic_correctness.py"}
99 -
100 -if [[ $commands == *"pytest -v -s lora"* ]]; then
101 - commands=${commands//"pytest -v -s lora"/"VLLM_ROCM_CUSTOM_PAGED_ATTN=0 pytest -v -s lora"}
102 -fi
103 -
104 -#ignore certain kernels tests
105 -if [[ $commands == *" kernels/core"* ]]; then
106 - commands="${commands} \
107 - --ignore=kernels/core/test_fused_quant_layernorm.py \
108 - --ignore=kernels/core/test_permute_cols.py"
109 -fi
110 -
111 -if [[ $commands == *" kernels/attention"* ]]; then
112 - commands="${commands} \
113 - --ignore=kernels/attention/test_attention_selector.py \
114 - --ignore=kernels/attention/test_encoder_decoder_attn.py \
115 - --ignore=kernels/attention/test_flash_attn.py \
116 - --ignore=kernels/attention/test_flashinfer.py \
117 - --ignore=kernels/attention/test_prefix_prefill.py \
118 - --ignore=kernels/attention/test_cascade_flash_attn.py \
119 - --ignore=kernels/attention/test_mha_attn.py \
120 - --ignore=kernels/attention/test_lightning_attn.py \
121 - --ignore=kernels/attention/test_attention.py"
122 -fi
123 -
124 -if [[ $commands == *" kernels/quantization"* ]]; then
125 - commands="${commands} \
126 - --ignore=kernels/quantization/test_int8_quant.py \
127 - --ignore=kernels/quantization/test_machete_mm.py \
128 - --ignore=kernels/quantization/test_block_fp8.py \
129 - --ignore=kernels/quantization/test_block_int8.py \
130 - --ignore=kernels/quantization/test_marlin_gemm.py \
131 - --ignore=kernels/quantization/test_cutlass_scaled_mm.py \
132 - --ignore=kernels/quantization/test_int8_kernel.py"
133 -fi
134 -
135 -if [[ $commands == *" kernels/mamba"* ]]; then
136 - commands="${commands} \
137 - --ignore=kernels/mamba/test_mamba_mixer2.py \
138 - --ignore=kernels/mamba/test_causal_conv1d.py \
139 - --ignore=kernels/mamba/test_mamba_ssm_ssd.py"
140 -fi
141 -
142 -if [[ $commands == *" kernels/moe"* ]]; then
143 - commands="${commands} \
144 - --ignore=kernels/moe/test_moe.py \
145 - --ignore=kernels/moe/test_cutlass_moe.py \
146 - --ignore=kernels/moe/test_triton_moe_ptpc_fp8.py"
147 -fi
148 -
149 -#ignore certain Entrypoints/openai tests
150 -if [[ $commands == *" entrypoints/openai "* ]]; then
151 - commands=${commands//" entrypoints/openai "/" entrypoints/openai \
152 - --ignore=entrypoints/openai/test_audio.py \
153 - --ignore=entrypoints/openai/test_shutdown.py \
154 - --ignore=entrypoints/openai/test_completion.py \
155 - --ignore=entrypoints/openai/test_models.py \
156 - --ignore=entrypoints/openai/test_lora_adapters.py \
157 - --ignore=entrypoints/openai/test_return_tokens_as_ids.py \
158 - --ignore=entrypoints/openai/test_root_path.py \
159 - --ignore=entrypoints/openai/test_tokenization.py \
160 - --ignore=entrypoints/openai/test_prompt_validation.py "}
161 -fi
162 -
163 -#ignore certain Entrypoints/llm tests
164 -if [[ $commands == *" entrypoints/llm "* ]]; then
165 - commands=${commands//" entrypoints/llm "/" entrypoints/llm \
166 - --ignore=entrypoints/llm/test_chat.py \
167 - --ignore=entrypoints/llm/test_accuracy.py \
168 - --ignore=entrypoints/llm/test_init.py \
169 - --ignore=entrypoints/llm/test_prompt_validation.py "}
170 -fi
171 -
172 -commands=$(echo "$commands" | sed 's/ \\ / /g')
237 +# Fix quoting before ROCm overrides (so overrides see correct structure)
238 +commands=$(re_quote_pytest_markers "$commands")
239 +commands=$(apply_rocm_test_overrides "$commands")
240 echo "Final commands: $commands"
241
175 -# --ignore=entrypoints/openai/test_encoder_decoder.py \
176 -# --ignore=entrypoints/openai/test_embedding.py \
177 -# --ignore=entrypoints/openai/test_oot_registration.py
178 -# --ignore=entrypoints/openai/test_accuracy.py \
179 -# --ignore=entrypoints/openai/test_models.py <= Fails on MI250 but passes on MI300 as of 2025-03-13
180 -
181 -
242 MYPYTHONPATH=".."
243
184 -# Test that we're launching on the machine that has
185 -# proper access to GPUs
244 +# Verify GPU access
245 render_gid=$(getent group render | cut -d: -f3)
246 if [[ -z "$render_gid" ]]; then
247 echo "Error: 'render' group not found. This is required for GPU access." >&2
248 exit 1
249 fi
250
192 -if [[ $commands == *"VLLM_TEST_GROUP_NAME=mi325_4-2-node-tests-4-gpus-in-total"* ]]; then
193 -
251 +# --- Route: multi-node vs single-node ---
252 +if is_multi_node "$commands"; then
253 + echo "--- Multi-node job detected"
254 export DCKR_VER=$(docker --version | sed 's/Docker version \(.*\), build .*/\1/')
255
196 - if [[ "$commands" =~ ^(.*)"["(.*)"] && ["(.*)"]"$ ]]; then
197 - prefix=$( echo "${BASH_REMATCH[1]}" | sed 's/;//g')
198 - echo "PREFIX: ${prefix}"
199 - export composite_command="(command rocm-smi || true)"
200 - myIFS=$IFS
201 - IFS=','
202 - read -ra node0 <<< ${BASH_REMATCH[2]}
203 - read -ra node1 <<< ${BASH_REMATCH[3]}
204 - IFS=$myIFS
205 - for i in "${!node0[@]}";do
206 - command_node_0=$(echo ${node0[i]} | sed 's/\"//g')
207 - command_node_1=$(echo ${node1[i]} | sed 's/\"//g')
208 -
209 - export commands="./.buildkite/scripts/run-multi-node-test.sh /vllm-workspace/tests 2 2 ${image_name} '${command_node_0}' '${command_node_1}'"
210 - echo "COMMANDS: ${commands}"
211 - composite_command=$(echo "${composite_command} && ${commands}")
212 - done
213 - /bin/bash -c "${composite_command}"
214 - cleanup_network
256 + # Parse the bracket syntax: prefix ; [node0_cmds] && [node1_cmds]
257 + # BASH_REMATCH[1] = prefix (everything before first bracket)
258 + # BASH_REMATCH[2] = comma-separated node0 commands
259 + # BASH_REMATCH[3] = comma-separated node1 commands
260 + if [[ "$commands" =~ ^(.*)\[(.*)"] && ["(.*)\]$ ]]; then
261 + prefix=$(echo "${BASH_REMATCH[1]}" | sed 's/;//g')
262 + echo "PREFIX: ${prefix}"
263 +
264 + export composite_command="(command rocm-smi || true)"
265 + saved_IFS=$IFS
266 + IFS=','
267 + read -ra node0 <<< "${BASH_REMATCH[2]}"
268 + read -ra node1 <<< "${BASH_REMATCH[3]}"
269 + IFS=$saved_IFS
270 +
271 + if [[ ${#node0[@]} -ne ${#node1[@]} ]]; then
272 + echo "Warning: node0 has ${#node0[@]} commands, node1 has ${#node1[@]}. They will be paired by index."
273 + fi
274 +
275 + for i in "${!node0[@]}"; do
276 + command_node_0=$(echo "${node0[i]}" | sed 's/\"//g')
277 + command_node_1=$(echo "${node1[i]}" | sed 's/\"//g')
278 +
279 + step_cmd="./.buildkite/scripts/run-multi-node-test.sh /vllm-workspace/tests 2 2 ${image_name} '${command_node_0}' '${command_node_1}'"
280 + echo "COMMANDS: ${step_cmd}"
281 + composite_command="${composite_command} && ${step_cmd}"
282 + done
283 +
284 + /bin/bash -c "${composite_command}"
285 + cleanup_network
286 else
216 - echo "Failed to parse node commands! Exiting."
217 - cleanup_network
218 - exit 111
287 + echo "Multi-node job detected but failed to parse bracket command syntax."
288 + echo "Expected format: prefix ; [node0_cmd1, node0_cmd2] && [node1_cmd1, node1_cmd2]"
289 + echo "Got: $commands"
290 + cleanup_network
291 + exit 111
292 fi
293 else
294 + echo "--- Single-node job"
295 echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES"
296 docker run \
223 - --device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \
224 - --network=host \
225 - --shm-size=16gb \
226 - --group-add "$render_gid" \
227 - --rm \
228 - -e HF_TOKEN \
229 - -e AWS_ACCESS_KEY_ID \
230 - -e AWS_SECRET_ACCESS_KEY \
231 - -v "${HF_CACHE}:${HF_MOUNT}" \
232 - -e "HF_HOME=${HF_MOUNT}" \
233 - -e "PYTHONPATH=${MYPYTHONPATH}" \
234 - --name "${container_name}" \
235 - "${image_name}" \
236 - /bin/bash -c "${commands}"
297 + --device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \
298 + --network=host \
299 + --shm-size=16gb \
300 + --group-add "$render_gid" \
301 + --rm \
302 + -e HF_TOKEN \
303 + -e AWS_ACCESS_KEY_ID \
304 + -e AWS_SECRET_ACCESS_KEY \
305 + -v "${HF_CACHE}:${HF_MOUNT}" \
306 + -e "HF_HOME=${HF_MOUNT}" \
307 + -e "PYTHONPATH=${MYPYTHONPATH}" \
308 + --name "${container_name}" \
309 + "${image_name}" \
310 + /bin/bash -c "${commands}"
311 fi
.buildkite/test_areas/engine.yaml
+8
@@ -28,3 +28,11 @@ steps:
28 - pytest -v -s v1/engine/test_preprocess_error_handling.py
29 # Run the rest of v1/engine tests
30 - pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py
31 + mirror:
32 + amd:
33 + device: mi325_8
34 + depends_on:
35 + - image-build-amd
36 + commands:
37 + - pytest -v -s v1/e2e
38 + - pytest -v -s v1/engine
.buildkite/test_areas/entrypoints.yaml
+5 -5
@@ -24,11 +24,6 @@ steps:
24 - pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_generate.py --ignore=entrypoints/llm/test_collective_rpc.py
25 - pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process
26 - pytest -v -s entrypoints/offline_mode # Needs to avoid interference with other tests
27 - mirror:
28 - amd:
29 - device: mi325_1
30 - depends_on:
31 - - image-build-amd
27
28 - label: Entrypoints Integration (API Server 1)
29 timeout_in_minutes: 130
@@ -65,6 +60,11 @@ steps:
60 commands:
61 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
62 - pytest -v -s entrypoints/pooling
63 + mirror:
64 + amd:
65 + device: mi325_1
66 + depends_on:
67 + - image-build-amd
68
69 - label: Entrypoints Integration (Responses API)
70 timeout_in_minutes: 50
.buildkite/test_areas/misc.yaml
+6
@@ -16,6 +16,7 @@ steps:
16 - pytest -v -s v1/sample
17 - pytest -v -s v1/logits_processors
18 - pytest -v -s v1/worker
19 + # TODO: create another `optional` test group for slow tests
20 - pytest -v -s -m 'not slow_test' v1/spec_decode
21 - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit
22 - pytest -v -s -m 'not cpu_test' v1/metrics
@@ -25,6 +26,11 @@ steps:
26 # Integration test for streaming correctness (requires special branch).
27 - pip install -U git+https://github.com/robertgshaw2-redhat/lm-evaluation-harness.git@streaming-api
28 - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
29 + mirror:
30 + amd:
31 + device: mi325_1
32 + depends_on:
33 + - image-build-amd
34
35 - label: V1 Others (CPU)
36 depends_on:
.buildkite/test_areas/models_language.yaml
+14
@@ -55,6 +55,15 @@ steps:
55 - uv pip install --system --no-build-isolation 'git+https://github.com/state-spaces/mamba@v2.3.0'
56 - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.5.2'
57 - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)'
58 + mirror:
59 + amd:
60 + device: mi325_1
61 + depends_on:
62 + - image-build-amd
63 + commands:
64 + - uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr'
65 + - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.5.2'
66 + - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)'
67
68 - label: Language Models Test (PPL)
69 timeout_in_minutes: 110
@@ -73,6 +82,11 @@ steps:
82 - tests/models/language/pooling
83 commands:
84 - pytest -v -s models/language/pooling -m 'not core_model'
85 + mirror:
86 + amd:
87 + device: mi325_1
88 + depends_on:
89 + - image-build-amd
90
91 - label: Language Models Test (MTEB)
92 timeout_in_minutes: 110
docker/Dockerfile.rocm
+13
@@ -305,6 +305,14 @@ RUN --mount=type=bind,from=export_vllm,src=/,target=/install \
305 RUN --mount=type=bind,from=build_rixl,src=/app/install,target=/rixl_install \
306 uv pip install --system /rixl_install/*.whl
307
308 +# RIXL/MoRIIO runtime dependencies (RDMA userspace libraries)
309 +RUN apt-get update -q -y && apt-get install -q -y \
310 + librdmacm1 \
311 + libibverbs1 \
312 + ibverbs-providers \
313 + ibverbs-utils \
314 + && rm -rf /var/lib/apt/lists/*
315 +
316 WORKDIR /vllm-workspace
317 ARG COMMON_WORKDIR
318 COPY --from=build_vllm ${COMMON_WORKDIR}/vllm /vllm-workspace
@@ -330,6 +338,11 @@ RUN bash /tmp/install_torchcodec.sh \
338 # Copy in the v1 package (for python-only install test group)
339 COPY --from=export_vllm /vllm_v1 /usr/local/lib/python${PYTHON_VERSION}/dist-packages/vllm/v1
340
341 +# Set MIOPEN ENVS to resolve performance regressions in MIOpen 3D convolution kernel
342 +# See: https://github.com/pytorch/pytorch/issues/169857
343 +ENV MIOPEN_DEBUG_CONV_DIRECT=0
344 +ENV MIOPEN_DEBUG_CONV_GEMM=0
345 +
346 # Source code is used in the `python_only_compile.sh` test
347 # We hide it inside `src/` so that this source code
348 # will not be imported by other tests
tests/v1/kv_connector/unit/test_moriio_connector.py
+16
@@ -2,6 +2,7 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 import importlib.util
4 import os
5 +import subprocess
6 from unittest.mock import MagicMock, patch
7
8 import msgspec
@@ -40,6 +41,19 @@ from .utils import create_request, create_scheduler
41
42 aiter_available = importlib.util.find_spec("aiter") is not None
43 mori_available = importlib.util.find_spec("mori") is not None
44 +
45 +
46 +def _rdma_available() -> bool:
47 + """Check if RDMA devices are available."""
48 + try:
49 + result = subprocess.run(["ibv_devinfo"], capture_output=True, text=True)
50 + return "No IB devices found" not in result.stderr
51 + except FileNotFoundError:
52 + return False
53 +
54 +
55 +rdma_available = _rdma_available()
56 +
57 pytestmark = pytest.mark.skipif(
58 not (current_platform.is_rocm() and mori_available),
59 reason="MoRIIOs are only available on ROCm with aiter package installed",
@@ -393,6 +407,7 @@ def test_read_mode_loads_remote_block_ids(moriio_read_mode):
407 @pytest.mark.skipif(
408 not aiter_available, reason="Requires aiter package for ROCm FlashAttention backend"
409 )
410 +@pytest.mark.skipif(not rdma_available, reason="No RDMA devices available")
411 def test_register_kv_caches(mock_parallel_groups):
412 """Test that MoRIIOConnector.register_kv_caches correctly registers kv caches."""
413 ROLE = "kv_consumer"
@@ -488,6 +503,7 @@ def test_register_kv_caches(mock_parallel_groups):
503 @pytest.mark.skipif(
504 not aiter_available, reason="Requires aiter package for ROCm FlashAttention backend"
505 )
506 +@pytest.mark.skipif(not rdma_available, reason="No RDMA devices available")
507 def test_moriio_handshake_returns_metadata(mock_parallel_groups):
508 """MoRIIO handshake socket returns valid agent metadata over ZMQ."""
509