A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 14:00 UTC
[ROCm][CI] Adding infiniband mappings for moriio tests (#35170)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
Feb 27, 2026 at 23:53 UTC
88e8525f2ed7d234937190ae523841a523a49f99
1 file changed
+183
-11
.buildkite/scripts/hardware_ci/run-amd-test.sh
+183
-11
@@ -6,6 +6,26 @@
6
# Multi-node detection: Instead of matching on fragile group names, we detect
7
# multi-node jobs structurally by looking for the bracket command syntax
8
# "[node0_cmds] && [node1_cmds]" or via the NUM_NODES environment variable.
9
+#
10
+###############################################################################
11
+# QUOTING / COMMAND PASSING
12
+#
13
+# Passing commands as positional arguments ($*) is fragile when the command
14
+# string itself contains double quotes, e.g.:
15
+#
16
+# bash run-amd-test.sh "export FLAGS="value" && pytest -m "not slow""
17
+#
18
+# The outer shell resolves the nested quotes *before* this script runs, so
19
+# the script receives mangled input it cannot fully recover.
20
+#
21
+# Preferred: pass commands via the VLLM_TEST_COMMANDS environment variable:
22
+#
23
+# export VLLM_TEST_COMMANDS='export FLAGS="value" && pytest -m "not slow"'
24
+# bash run-amd-test.sh
25
+#
26
+# Single-quoted assignment preserves all inner double quotes verbatim.
27
+# The $* path is kept for backward compatibility but callers should migrate.
28
+###############################################################################
29
set -o pipefail
30
31
# Export Python path
@@ -80,25 +100,140 @@ is_multi_node() {
100
}
101
102
###############################################################################
83
-# Pytest marker re-quoting
103
+# Pytest marker/keyword re-quoting
104
#
105
# When commands are passed through Buildkite -> shell -> $* -> bash -c,
86
-# quotes around pytest -m marker expressions get stripped:
106
+# quotes around multi-word pytest -m/-k expressions get stripped:
107
# pytest -v -s -m 'not cpu_test' v1/core
108
# becomes:
109
# pytest -v -s -m not cpu_test v1/core
110
#
111
# pytest then interprets "cpu_test" as a file path, not part of the marker.
92
-# This function detects unquoted multi-word marker expressions and re-quotes
93
-# them so they survive the final bash -c expansion.
112
+#
113
+# This function detects unquoted expressions after -m/-k and re-quotes them
114
+# by collecting tokens until a recognizable boundary is reached:
115
+# - test path (contains '/')
116
+# - test file (ends with '.py')
117
+# - another pytest flag (--xxx or -x single-char flags)
118
+# - command separator (&& || ; |)
119
+# - environment variable assignment (FOO=bar)
120
+#
121
+# Single-word markers (e.g. -m cpu_test, -m hybrid_model) pass through
122
+# unquoted since they have no spaces and work fine.
123
+#
124
+# Already-quoted expressions (containing literal single quotes) are passed
125
+# through untouched to avoid double-quoting values injected by
126
+# apply_rocm_test_overrides.
127
+#
128
+# NOTE: This ONLY fixes -m/-k flags. It cannot recover arbitrary inner
129
+# double-quotes stripped by the calling shell (see header comment).
130
+# Use VLLM_TEST_COMMANDS to avoid the problem entirely.
131
###############################################################################
95
-
132
re_quote_pytest_markers() {
97
- local cmds="$1"
98
- # Pattern: -m not <identifier> -> -m 'not <identifier>'
99
- # Handles the common cases: 'not cpu_test', 'not slow_test', etc.
100
- cmds=$(echo "$cmds" | sed -E "s/-m not ([a-zA-Z_][a-zA-Z0-9_]*)/-m 'not \1'/g")
101
- echo "$cmds"
133
+ local input="$1"
134
+ local output=""
135
+ local collecting=false
136
+ local marker_buf=""
137
+
138
+ # Flatten newlines for consistent tokenization
139
+ local flat="${input//$'\n'/ }"
140
+
141
+ # Disable globbing to prevent *.py etc. from expanding during read -ra
142
+ local restore_glob
143
+ restore_glob="$(shopt -p -o noglob 2>/dev/null || true)"
144
+ set -o noglob
145
+ local -a words
146
+ read -ra words <<< "$flat"
147
+ eval "$restore_glob"
148
+
149
+ for word in "${words[@]}"; do
150
+ if $collecting; then
151
+ # If the token we're about to collect already contains a literal
152
+ # single quote, the expression was already quoted upstream.
153
+ # Flush and stop collecting.
154
+ if [[ "$word" == *"'"* ]]; then
155
+ if [[ -n "$marker_buf" ]]; then
156
+ # Should not normally happen (partial buf + quote), flush raw
157
+ output+="${marker_buf} "
158
+ marker_buf=""
159
+ fi
160
+ output+="${word} "
161
+ collecting=false
162
+ continue
163
+ fi
164
+
165
+ local is_boundary=false
166
+ case "$word" in
167
+ # Command separators
168
+ "&&"|"||"|";"|"|")
169
+ is_boundary=true ;;
170
+ # Long flags (--ignore, --shard-id, etc.)
171
+ --*)
172
+ is_boundary=true ;;
173
+ # Short flags (-v, -s, -x, etc.) but NOT negative marker tokens
174
+ # like "not" which don't start with "-". Also skip -k/-m which
175
+ # would start a new marker (handled below).
176
+ -[a-zA-Z])
177
+ is_boundary=true ;;
178
+ # Test path (contains /)
179
+ */*)
180
+ is_boundary=true ;;
181
+ # Test file (ends with .py, possibly with ::method)
182
+ *.py|*.py::*)
183
+ is_boundary=true ;;
184
+ # Environment variable assignment preceding a command (FOO=bar)
185
+ *=*)
186
+ # Only treat as boundary if it looks like VAR=value, not
187
+ # pytest filter expressions like num_gpus=2 inside markers
188
+ if [[ "$word" =~ ^[A-Z_][A-Z0-9_]*= ]]; then
189
+ is_boundary=true
190
+ fi
191
+ ;;
192
+ esac
193
+
194
+ if $is_boundary; then
195
+ # Flush the collected marker expression
196
+ if [[ "$marker_buf" == *" "* || "$marker_buf" == *"("* ]]; then
197
+ output+="'${marker_buf}' "
198
+ else
199
+ output+="${marker_buf} "
200
+ fi
201
+ collecting=false
202
+ marker_buf=""
203
+ # Check if this boundary word itself starts a new -m/-k
204
+ if [[ "$word" == "-m" || "$word" == "-k" ]]; then
205
+ output+="${word} "
206
+ collecting=true
207
+ else
208
+ output+="${word} "
209
+ fi
210
+ else
211
+ # Accumulate into marker buffer
212
+ if [[ -n "$marker_buf" ]]; then
213
+ marker_buf+=" ${word}"
214
+ else
215
+ marker_buf="${word}"
216
+ fi
217
+ fi
218
+ elif [[ "$word" == "-m" || "$word" == "-k" ]]; then
219
+ output+="${word} "
220
+ collecting=true
221
+ marker_buf=""
222
+ else
223
+ output+="${word} "
224
+ fi
225
+ done
226
+
227
+ # Flush any trailing marker expression (marker at end of command)
228
+ if $collecting && [[ -n "$marker_buf" ]]; then
229
+ if [[ "$marker_buf" == *" "* || "$marker_buf" == *"("* ]]; then
230
+ output+="'${marker_buf}'"
231
+ else
232
+ output+="${marker_buf}"
233
+ fi
234
+ fi
235
+
236
+ echo "${output% }"
237
}
238
239
###############################################################################
@@ -231,11 +366,35 @@ HF_CACHE="$(realpath ~)/huggingface"
366
mkdir -p "${HF_CACHE}"
367
HF_MOUNT="/root/.cache/huggingface"
368
234
-commands="$*"
369
+# ---- Command source selection ----
370
+# Prefer VLLM_TEST_COMMANDS (preserves all inner quoting intact).
371
+# Fall back to $* for backward compatibility, but warn that inner
372
+# double-quotes will have been stripped by the calling shell.
373
+if [[ -n "${VLLM_TEST_COMMANDS:-}" ]]; then
374
+ commands="${VLLM_TEST_COMMANDS}"
375
+ echo "Commands sourced from VLLM_TEST_COMMANDS (quoting preserved)"
376
+else
377
+ commands="$*"
378
+ if [[ -z "$commands" ]]; then
379
+ echo "Error: No test commands provided." >&2
380
+ echo "Usage:" >&2
381
+ echo " Preferred: VLLM_TEST_COMMANDS='...' bash $0" >&2
382
+ echo " Legacy: bash $0 \"commands here\"" >&2
383
+ exit 1
384
+ fi
385
+ echo "Commands sourced from positional args (legacy mode)"
386
+ echo "WARNING: Inner double-quotes in the command string may have been"
387
+ echo " stripped by the calling shell. If you see syntax errors, switch to:"
388
+ echo " export VLLM_TEST_COMMANDS='your commands here'"
389
+ echo " bash $0"
390
+fi
391
+
392
echo "Raw commands: $commands"
393
394
# Fix quoting before ROCm overrides (so overrides see correct structure)
395
commands=$(re_quote_pytest_markers "$commands")
396
+echo "After re-quoting: $commands"
397
+
398
commands=$(apply_rocm_test_overrides "$commands")
399
echo "Final commands: $commands"
400
@@ -248,6 +407,18 @@ if [[ -z "$render_gid" ]]; then
407
exit 1
408
fi
409
410
+# --- RDMA device passthrough (conditional) ---
411
+# If the host has RDMA devices, pass them through so tests like
412
+# test_moriio_connector can access ibverbs. On hosts without RDMA
413
+# hardware the tests will gracefully skip via _rdma_available().
414
+RDMA_FLAGS=""
415
+if [ -d /dev/infiniband ]; then
416
+ echo "RDMA devices detected on host, enabling passthrough"
417
+ RDMA_FLAGS="--device /dev/infiniband --cap-add=IPC_LOCK"
418
+else
419
+ echo "No RDMA devices found on host, RDMA tests will be skipped"
420
+fi
421
+
422
# --- Route: multi-node vs single-node ---
423
if is_multi_node "$commands"; then
424
echo "--- Multi-node job detected"
@@ -295,6 +466,7 @@ else
466
echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES"
467
docker run \
468
--device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \
469
+ $RDMA_FLAGS \
470
--network=host \
471
--shm-size=16gb \
472
--group-add "$render_gid" \