|
1
|
#!/bin/bash |
|
2
|
|
|
3
|
# shellcheck disable=SC2329 # Every function in this file is only ever reached |
|
4
|
# transitively through handle_amd_runner_exit, the EXIT trap handler - shellcheck's |
|
5
|
# unused-function check doesn't do full reachability analysis from an indirectly |
|
6
|
# invoked entry point, so it flags the whole diagnostic-collection call chain as |
|
7
|
# dead code even though it runs live on every nonzero exit. |
|
8
|
|
|
9
|
# This script runs ROCm tests either directly in a native CI pod or inside the |
|
10
|
# corresponding Docker container. Multi-node tests continue to use Docker. |
|
11
|
# |
|
12
|
# Multi-node detection: Instead of matching on fragile group names, we detect |
|
13
|
# multi-node jobs structurally by looking for the bracket command syntax |
|
14
|
# "[node0_cmds] && [node1_cmds]" or via the NUM_NODES environment variable. |
|
15
|
# |
|
16
|
############################################################################### |
|
17
|
# QUOTING / COMMAND PASSING |
|
18
|
# |
|
19
|
# Passing commands as positional arguments ($*) is fragile when the command |
|
20
|
# string itself contains double quotes, e.g.: |
|
21
|
# |
|
22
|
# bash run-amd-test.sh "export FLAGS="value" && pytest -m "not slow"" |
|
23
|
# |
|
24
|
# The outer shell resolves the nested quotes *before* this script runs, so |
|
25
|
# the script receives mangled input it cannot fully recover. |
|
26
|
# |
|
27
|
# Preferred: pass commands via the VLLM_TEST_COMMANDS environment variable: |
|
28
|
# |
|
29
|
# export VLLM_TEST_COMMANDS='export FLAGS="value" && pytest -m "not slow"' |
|
30
|
# bash run-amd-test.sh |
|
31
|
# |
|
32
|
# Single-quoted assignment preserves all inner double quotes verbatim. |
|
33
|
# The $* path is kept for backward compatibility but callers should migrate. |
|
34
|
############################################################################### |
|
35
|
set -o pipefail |
|
36
|
|
|
37
|
# shellcheck source=.buildkite/scripts/rocm/build-config.sh |
|
38
|
source "$(dirname "${BASH_SOURCE[0]}")/../rocm/build-config.sh" || exit $? |
|
39
|
|
|
40
|
: "${BUILDKIT_PROGRESS:=plain}" |
|
41
|
: "${TERM:=xterm-256color}" |
|
42
|
: "${FORCE_COLOR:=1}" |
|
43
|
: "${CLICOLOR_FORCE:=1}" |
|
44
|
: "${PY_COLORS:=1}" |
|
45
|
: "${ROCM_DOCKER_TTY:=1}" |
|
46
|
: "${PYTHONFAULTHANDLER:=1}" |
|
47
|
: "${PYTEST_TIMEOUT:=2400}" |
|
48
|
|
|
49
|
# Preserve runner-only values before clear_ci_orchestration_env removes them |
|
50
|
# from the environment inherited by the test process. |
|
51
|
amd_diagnostics_expected_gpu_count="${VLLM_CI_EXPECTED_GPU_COUNT:-1}" |
|
52
|
amd_diagnostics_dir="${VLLM_CI_DIAGNOSTICS_DIR:-artifacts/amd-gpu-diagnostics}" |
|
53
|
amd_diagnostics_checkout_root="${BUILDKITE_BUILD_CHECKOUT_PATH:-$(pwd -P)}" |
|
54
|
amd_diagnostics_execution_mode="${VLLM_CI_EXECUTION_MODE:-single-node}" |
|
55
|
amd_diagnostics_test_group="${VLLM_TEST_GROUP_NAME:-${BUILDKITE_STEP_KEY:-unknown}}" |
|
56
|
amd_diagnostics_workspace="${VLLM_CI_WORKSPACE:-/vllm-workspace}" |
|
57
|
amd_diagnostics_pod_name="${VLLM_CI_K8S_POD_NAME:-${POD_NAME:-${HOSTNAME:-}}}" |
|
58
|
amd_diagnostics_k8s_namespace="${VLLM_CI_K8S_NAMESPACE:-${POD_NAMESPACE:-unknown}}" |
|
59
|
amd_diagnostics_k8s_node_name="${VLLM_CI_K8S_NODE_NAME:-${NODE_NAME:-}}" |
|
60
|
amd_diagnostics_collected=0 |
|
61
|
amd_diagnostics_memory_events_path="" |
|
62
|
amd_diagnostics_probe_budget_seconds=25 |
|
63
|
amd_diagnostics_command_timeout_seconds=5 |
|
64
|
amd_diagnostics_upload_timeout_seconds=20 |
|
65
|
if [[ " ${PYTEST_ADDOPTS:-} " != *" --color"* ]]; then |
|
66
|
PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--color=yes" |
|
67
|
fi |
|
68
|
if [[ " ${PYTEST_ADDOPTS:-} " != *" --durations="* ]]; then |
|
69
|
PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--durations=25" |
|
70
|
fi |
|
71
|
if [[ " ${PYTEST_ADDOPTS:-} " != *" --durations-min="* ]]; then |
|
72
|
PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--durations-min=1.0" |
|
73
|
fi |
|
74
|
# Dump stacks after 25 minutes, then stop an individual test after 40 minutes. |
|
75
|
if [[ " ${PYTEST_ADDOPTS:-} " != *" faulthandler_timeout="* ]]; then |
|
76
|
PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }-o faulthandler_timeout=1500" |
|
77
|
fi |
|
78
|
if [[ " ${PYTEST_ADDOPTS:-} " != *" --timeout-method="* && |
|
79
|
" ${PYTEST_ADDOPTS:-} " != *" --timeout-method "* ]]; then |
|
80
|
PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--timeout-method=thread" |
|
81
|
fi |
|
82
|
export BUILDKIT_PROGRESS TERM FORCE_COLOR CLICOLOR_FORCE PY_COLORS PYTEST_ADDOPTS PYTEST_TIMEOUT ROCM_DOCKER_TTY |
|
83
|
export PYTHONFAULTHANDLER |
|
84
|
|
|
85
|
# Export Python path for commands that run directly on the host. Containerized |
|
86
|
# tests set this to /vllm-workspace below so spawned Python processes do not |
|
87
|
# depend on their current working directory. |
|
88
|
export PYTHONPATH="${PYTHONPATH:-..}" |
|
89
|
|
|
90
|
ci_started_at=$SECONDS |
|
91
|
|
|
92
|
############################################################################### |
|
93
|
# Helper Functions |
|
94
|
############################################################################### |
|
95
|
|
|
96
|
report_docker_usage() { |
|
97
|
echo "--- Docker usage" |
|
98
|
docker system df || true |
|
99
|
} |
|
100
|
|
|
101
|
clear_ci_orchestration_env() { |
|
102
|
unset -v \ |
|
103
|
VLLM_TEST_GROUP_NAME \ |
|
104
|
VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE \ |
|
105
|
VLLM_CI_ARTIFACT_STEP \ |
|
106
|
VLLM_TEST_CACHE \ |
|
107
|
VLLM_CI_EXECUTION_MODE \ |
|
108
|
VLLM_CI_WORKSPACE \ |
|
109
|
VLLM_CI_REQUIRE_WORKSPACE_MOUNT \ |
|
110
|
VLLM_TEST_COMMANDS \ |
|
111
|
VLLM_CI_BRANCH \ |
|
112
|
CI_ROCM_DOCKERFILE_BASE \ |
|
113
|
CI_ROCM_DOCKERFILE \ |
|
114
|
ROCM_BASE_DOCKERFILE \ |
|
115
|
CI_BASE_DOCKERFILE \ |
|
116
|
VLLM_CI_BASE_IMAGE \ |
|
117
|
VLLM_CI_FALLBACK_IMAGE \ |
|
118
|
VLLM_CI_DOCKER_DISABLED \ |
|
119
|
VLLM_CI_DIAGNOSTICS_DIR \ |
|
120
|
VLLM_CI_ARTIFACT_GLOB \ |
|
121
|
VLLM_CI_ARTIFACT_CHECKSUM_GLOB \ |
|
122
|
VLLM_CI_EXPECTED_GPU_COUNT \ |
|
123
|
VLLM_CI_K8S_POD_NAME \ |
|
124
|
VLLM_CI_K8S_NAMESPACE \ |
|
125
|
VLLM_CI_K8S_NODE_NAME \ |
|
126
|
VLLM_CI_USE_ARTIFACTS \ |
|
127
|
VLLM_CI_RESULTS_ROOT \ |
|
128
|
VLLM_ALLOW_DEPRECATED_BEAM_SEARCH |
|
129
|
} |
|
130
|
|
|
131
|
amd_ci_teardown_log() { |
|
132
|
local event=$1 |
|
133
|
shift |
|
134
|
|
|
135
|
printf '[amd-ci-teardown] event=%s' "${event}" >&2 |
|
136
|
if (($#)); then |
|
137
|
printf ' %s' "$@" >&2 |
|
138
|
fi |
|
139
|
printf '\n' >&2 |
|
140
|
} |
|
141
|
|
|
142
|
run_docker_with_ci_timeout() { |
|
143
|
local raw_timeout=${CONTAINER_TIMEOUT_S:-0} |
|
144
|
local configured_timeout=0 |
|
145
|
local elapsed=0 |
|
146
|
local remaining_timeout=0 |
|
147
|
local status=0 |
|
148
|
|
|
149
|
if [[ ! "${raw_timeout}" =~ ^(0|[1-9][0-9]{0,5})$ ]]; then |
|
150
|
amd_ci_teardown_log configuration_error \ |
|
151
|
"reason=invalid_timeout" "expected=integer_0_to_604800" |
|
152
|
return 2 |
|
153
|
fi |
|
154
|
configured_timeout=$((10#${raw_timeout})) |
|
155
|
if ((configured_timeout > 604800)); then |
|
156
|
amd_ci_teardown_log configuration_error \ |
|
157
|
"reason=invalid_timeout" "expected=integer_0_to_604800" |
|
158
|
return 2 |
|
159
|
fi |
|
160
|
|
|
161
|
remaining_timeout=${configured_timeout} |
|
162
|
if ((configured_timeout > 0)); then |
|
163
|
if ! command -v timeout >/dev/null 2>&1; then |
|
164
|
amd_ci_teardown_log configuration_error \ |
|
165
|
"reason=timeout_command_not_found" |
|
166
|
return 127 |
|
167
|
fi |
|
168
|
elapsed=$((SECONDS - ci_started_at)) |
|
169
|
if ((elapsed >= configured_timeout)); then |
|
170
|
amd_ci_teardown_log deadline_exhausted \ |
|
171
|
"mode=docker" "configured_s=${configured_timeout}" "elapsed_s=${elapsed}" |
|
172
|
return 124 |
|
173
|
fi |
|
174
|
remaining_timeout=$((configured_timeout - elapsed)) |
|
175
|
fi |
|
176
|
|
|
177
|
amd_ci_teardown_log workload_started \ |
|
178
|
"mode=docker" "timeout_s=${remaining_timeout}" |
|
179
|
if ((remaining_timeout == 0)); then |
|
180
|
"$@" || status=$? |
|
181
|
else |
|
182
|
# Docker runs interactively, so keep it in the foreground process group. |
|
183
|
# --verbose records both TERM and any KILL escalation caused by timeout. |
|
184
|
timeout --verbose --foreground --signal=TERM --kill-after=10s \ |
|
185
|
"${remaining_timeout}s" "$@" || status=$? |
|
186
|
fi |
|
187
|
amd_ci_teardown_log workload_finished "mode=docker" "status=${status}" |
|
188
|
return "${status}" |
|
189
|
} |
|
190
|
|
|
191
|
prepare_artifact_image() { |
|
192
|
if [[ "${VLLM_CI_USE_ARTIFACTS:-0}" != "1" ]]; then |
|
193
|
return 1 |
|
194
|
fi |
|
195
|
if ! command -v buildkite-agent >/dev/null 2>&1; then |
|
196
|
echo "buildkite-agent not found; cannot download ROCm wheel artifact" |
|
197
|
return 1 |
|
198
|
fi |
|
199
|
|
|
200
|
local artifact_glob="${VLLM_CI_ARTIFACT_GLOB:-artifacts/vllm-rocm-install/vllm-rocm-install.tar.gz}" |
|
201
|
local archive="" |
|
202
|
local metadata_file="" |
|
203
|
local base_image="${VLLM_CI_BASE_IMAGE:-rocm/vllm-dev:ci_base}" |
|
204
|
local artifact_image="" |
|
205
|
local artifact_key="" |
|
206
|
local base_digest="" |
|
207
|
local wheel_dir="" |
|
208
|
local context_dir="" |
|
209
|
local workspace_dir="" |
|
210
|
|
|
211
|
artifact_work_dir=$(mktemp -d -t vllm-rocm-artifact.XXXXXX) |
|
212
|
wheel_dir="${artifact_work_dir}/wheels" |
|
213
|
context_dir="${artifact_work_dir}/context" |
|
214
|
workspace_dir="${context_dir}/workspace" |
|
215
|
mkdir -p "${wheel_dir}" "${context_dir}/wheels" "${workspace_dir}" |
|
216
|
|
|
217
|
echo "--- Downloading ROCm wheel artifact" |
|
218
|
if ! buildkite-agent artifact download "${artifact_glob}" "${artifact_work_dir}"; then |
|
219
|
echo "Failed to download ${artifact_glob}" |
|
220
|
return 1 |
|
221
|
fi |
|
222
|
buildkite-agent artifact download \ |
|
223
|
"artifacts/vllm-rocm-install/ci-base-image.txt" \ |
|
224
|
"${artifact_work_dir}" >/dev/null 2>&1 || true |
|
225
|
|
|
226
|
archive=$(find "${artifact_work_dir}" -name "vllm-rocm-install.tar.gz" -type f | head -1) |
|
227
|
if [[ -z "${archive}" || ! -f "${archive}" ]]; then |
|
228
|
echo "ROCm wheel artifact archive was not found" |
|
229
|
return 1 |
|
230
|
fi |
|
231
|
|
|
232
|
metadata_file=$(find "${artifact_work_dir}" -name "ci-base-image.txt" -type f | head -1) |
|
233
|
if [[ -n "${metadata_file}" && -s "${metadata_file}" ]]; then |
|
234
|
base_image=$(tr -d '[:space:]' < "${metadata_file}") |
|
235
|
elif using_custom_rocm_dockerfiles; then |
|
236
|
echo "Custom ROCm ci_base metadata is missing; using the full CI image" |
|
237
|
return 1 |
|
238
|
fi |
|
239
|
|
|
240
|
echo "--- Preparing local ROCm test image" |
|
241
|
echo "Base image: ${base_image}" |
|
242
|
docker pull "${base_image}" || return 1 |
|
243
|
base_digest=$( |
|
244
|
docker image inspect \ |
|
245
|
--format='{{if .RepoDigests}}{{index .RepoDigests 0}}{{else}}{{.Id}}{{end}}' \ |
|
246
|
"${base_image}" 2>/dev/null || printf '%s' "${base_image}" |
|
247
|
) |
|
248
|
|
|
249
|
artifact_key=$( |
|
250
|
{ |
|
251
|
printf 'base-image:%s\n' "${base_digest}" |
|
252
|
sha256sum "${archive}" |
|
253
|
} | sha256sum | cut -c1-24 |
|
254
|
) |
|
255
|
artifact_image="rocm/vllm-ci-artifact:${artifact_key}" |
|
256
|
|
|
257
|
if docker image inspect "${artifact_image}" >/dev/null 2>&1; then |
|
258
|
echo "Using existing local ROCm artifact image: ${artifact_image}" |
|
259
|
image_name="${artifact_image}" |
|
260
|
return 0 |
|
261
|
fi |
|
262
|
|
|
263
|
tar -xzf "${archive}" -C "${wheel_dir}" || return 1 |
|
264
|
if ! ls "${wheel_dir}"/*.whl >/dev/null 2>&1; then |
|
265
|
echo "ROCm wheel artifact did not contain a wheel" |
|
266
|
return 1 |
|
267
|
fi |
|
268
|
if [[ ! -d "${wheel_dir}/tests" ]]; then |
|
269
|
echo "ROCm wheel artifact did not contain the test workspace" |
|
270
|
return 1 |
|
271
|
fi |
|
272
|
|
|
273
|
cp "${wheel_dir}"/*.whl "${context_dir}/wheels/" || return 1 |
|
274
|
tar -C "${wheel_dir}" \ |
|
275
|
--exclude='*.whl' \ |
|
276
|
--exclude='.vllm-ci-artifact' \ |
|
277
|
--exclude='./.vllm-ci-artifact' \ |
|
278
|
-cf - . \ |
|
279
|
| tar -C "${workspace_dir}" -xf - || return 1 |
|
280
|
cat > "${context_dir}/Dockerfile" <<'EOF' |
|
281
|
ARG BASE_IMAGE |
|
282
|
FROM ${BASE_IMAGE} |
|
283
|
COPY wheels/ /tmp/vllm-wheels/ |
|
284
|
COPY workspace/ /vllm-workspace/ |
|
285
|
RUN python3 -m pip install --no-deps --force-reinstall /tmp/vllm-wheels/*.whl \ |
|
286
|
&& rm -rf /tmp/vllm-wheels |
|
287
|
WORKDIR /vllm-workspace |
|
288
|
EOF |
|
289
|
|
|
290
|
echo "--- Building local ROCm test image" |
|
291
|
docker build \ |
|
292
|
--pull=false \ |
|
293
|
--progress "${BUILDKIT_PROGRESS}" \ |
|
294
|
--build-arg "BASE_IMAGE=${base_image}" \ |
|
295
|
-t "${artifact_image}" \ |
|
296
|
"${context_dir}" || return 1 |
|
297
|
image_name="${artifact_image}" |
|
298
|
return 0 |
|
299
|
} |
|
300
|
|
|
301
|
is_native_runtime() { |
|
302
|
[[ "${AMD_CI_RUNTIME:-}" == "native" || "${NATIVE_CI:-}" == "true" ]] |
|
303
|
} |
|
304
|
|
|
305
|
validate_native_workspace() { |
|
306
|
local workspace_dir="${VLLM_CI_WORKSPACE:-/vllm-workspace}" |
|
307
|
local workspace_real="" |
|
308
|
local checkout_real="" |
|
309
|
local workspace_mount="" |
|
310
|
|
|
311
|
mkdir -p "${workspace_dir}" || return 1 |
|
312
|
workspace_real=$(readlink -m "${workspace_dir}") || return 1 |
|
313
|
if [[ -n "${BUILDKITE_BUILD_CHECKOUT_PATH:-}" ]]; then |
|
314
|
checkout_real=$(readlink -m "${BUILDKITE_BUILD_CHECKOUT_PATH}") || return 1 |
|
315
|
if [[ "${checkout_real}" == "${workspace_real}" \ |
|
316
|
|| "${checkout_real}" == "${workspace_real}/"* \ |
|
317
|
|| "${workspace_real}" == "${checkout_real}/"* ]]; then |
|
318
|
echo "Refusing to replace ${workspace_real}; it overlaps the Buildkite checkout ${checkout_real}" >&2 |
|
319
|
return 1 |
|
320
|
fi |
|
321
|
fi |
|
322
|
if [[ "${VLLM_CI_REQUIRE_WORKSPACE_MOUNT:-1}" == "1" ]]; then |
|
323
|
if ! command -v findmnt >/dev/null 2>&1; then |
|
324
|
echo "findmnt is required to verify the native workspace mount" >&2 |
|
325
|
return 1 |
|
326
|
fi |
|
327
|
workspace_mount=$(findmnt -n -T "${workspace_real}" -o TARGET 2>/dev/null || true) |
|
328
|
if [[ "$(readlink -m "${workspace_mount:-/}")" != "${workspace_real}" ]]; then |
|
329
|
echo "Native CI requires a dedicated volume mounted at ${workspace_real}" >&2 |
|
330
|
return 1 |
|
331
|
fi |
|
332
|
fi |
|
333
|
} |
|
334
|
|
|
335
|
prepare_native_workspace() { |
|
336
|
local test_commands="${1:-}" |
|
337
|
|
|
338
|
if [[ "${VLLM_CI_USE_ARTIFACTS:-0}" != "1" ]]; then |
|
339
|
echo "Native CI requires VLLM_CI_USE_ARTIFACTS=1" |
|
340
|
return 1 |
|
341
|
fi |
|
342
|
if ! command -v buildkite-agent >/dev/null 2>&1; then |
|
343
|
echo "buildkite-agent not found; cannot download ROCm wheel artifact" |
|
344
|
return 1 |
|
345
|
fi |
|
346
|
validate_native_workspace || return 1 |
|
347
|
|
|
348
|
local artifact_glob="${VLLM_CI_ARTIFACT_GLOB:-artifacts/vllm-rocm-install/vllm-rocm-install.tar.gz}" |
|
349
|
local artifact_checksum_glob="${VLLM_CI_ARTIFACT_CHECKSUM_GLOB:-${artifact_glob}.sha256}" |
|
350
|
local artifact_step="${VLLM_CI_ARTIFACT_STEP:-image-build-amd}" |
|
351
|
local archive="" |
|
352
|
local checksum="" |
|
353
|
local download_dir="" |
|
354
|
local metadata_dir="" |
|
355
|
local recorded_base="" |
|
356
|
local recorded_commit="" |
|
357
|
local recorded_wheel="" |
|
358
|
local checkout="" |
|
359
|
local checkout_commit="" |
|
360
|
local workspace_dir="${VLLM_CI_WORKSPACE:-/vllm-workspace}" |
|
361
|
local wheel_dir="" |
|
362
|
local attempt=0 |
|
363
|
local attempt_dir="" |
|
364
|
local -a archives=() |
|
365
|
local -a checksums=() |
|
366
|
local -a wheels=() |
|
367
|
|
|
368
|
artifact_work_dir=$(mktemp -d -t vllm-rocm-artifact.XXXXXX) || return 1 |
|
369
|
wheel_dir="${artifact_work_dir}/wheels" |
|
370
|
mkdir -p "${wheel_dir}" || return 1 |
|
371
|
|
|
372
|
echo "--- Downloading ROCm wheel artifact from ${artifact_step} (native in-pod)" |
|
373
|
for attempt in 1 2 3; do |
|
374
|
attempt_dir="${artifact_work_dir}/download-${attempt}" |
|
375
|
rm -rf "${attempt_dir}" || return 1 |
|
376
|
mkdir -p "${attempt_dir}" || return 1 |
|
377
|
if buildkite-agent artifact download \ |
|
378
|
"${artifact_glob}" "${attempt_dir}" --step "${artifact_step}" \ |
|
379
|
&& buildkite-agent artifact download \ |
|
380
|
"${artifact_checksum_glob}" "${attempt_dir}" --step "${artifact_step}"; then |
|
381
|
download_dir="${attempt_dir}" |
|
382
|
break |
|
383
|
fi |
|
384
|
echo "Artifact download attempt ${attempt}/3 failed" |
|
385
|
if [[ "${attempt}" -lt 3 ]]; then |
|
386
|
sleep $((attempt * 2)) |
|
387
|
fi |
|
388
|
done |
|
389
|
if [[ -z "${download_dir}" ]]; then |
|
390
|
echo "Failed to download ${artifact_glob} and ${artifact_checksum_glob} from ${artifact_step}" |
|
391
|
return 1 |
|
392
|
fi |
|
393
|
|
|
394
|
mapfile -t archives < <( |
|
395
|
find "${download_dir}" -name "vllm-rocm-install.tar.gz" -type f -print |
|
396
|
) |
|
397
|
mapfile -t checksums < <( |
|
398
|
find "${download_dir}" -name "vllm-rocm-install.tar.gz.sha256" -type f -print |
|
399
|
) |
|
400
|
if [[ ${#archives[@]} -ne 1 || ${#checksums[@]} -ne 1 ]]; then |
|
401
|
echo "Expected exactly one ROCm archive and checksum; found ${#archives[@]} archive(s) and ${#checksums[@]} checksum(s)" >&2 |
|
402
|
return 1 |
|
403
|
fi |
|
404
|
archive="${archives[0]}" |
|
405
|
checksum="${checksums[0]}" |
|
406
|
if [[ "$(dirname "${archive}")" != "$(dirname "${checksum}")" ]]; then |
|
407
|
echo "ROCm archive and checksum were downloaded to different directories" >&2 |
|
408
|
return 1 |
|
409
|
fi |
|
410
|
( |
|
411
|
cd "$(dirname "${archive}")" |
|
412
|
sha256sum -c "$(basename "${checksum}")" |
|
413
|
) || return 1 |
|
414
|
|
|
415
|
tar --no-same-owner -xzf "${archive}" -C "${wheel_dir}" || return 1 |
|
416
|
mapfile -t wheels < <( |
|
417
|
find "${wheel_dir}" -maxdepth 1 -type f -name '*.whl' -print |
|
418
|
) |
|
419
|
if [[ ${#wheels[@]} -ne 1 ]]; then |
|
420
|
echo "ROCm artifact must contain exactly one top-level wheel; found ${#wheels[@]}" >&2 |
|
421
|
return 1 |
|
422
|
fi |
|
423
|
metadata_dir="${wheel_dir}/.vllm-ci-artifact" |
|
424
|
for metadata_file in commit.txt native-base-image.txt wheel-filename.txt; do |
|
425
|
if [[ ! -s "${metadata_dir}/${metadata_file}" ]]; then |
|
426
|
echo "ROCm artifact metadata is missing ${metadata_file}" >&2 |
|
427
|
return 1 |
|
428
|
fi |
|
429
|
done |
|
430
|
for metadata_file in ci-base-image.txt fallback-image.txt; do |
|
431
|
if [[ ! -f "${metadata_dir}/${metadata_file}" ]]; then |
|
432
|
echo "ROCm artifact metadata is missing ${metadata_file}" >&2 |
|
433
|
return 1 |
|
434
|
fi |
|
435
|
done |
|
436
|
|
|
437
|
recorded_commit=$(tr -d '\r\n' < "${metadata_dir}/commit.txt") |
|
438
|
recorded_base=$(tr -d '\r\n' < "${metadata_dir}/native-base-image.txt") |
|
439
|
recorded_wheel=$(tr -d '\r\n' < "${metadata_dir}/wheel-filename.txt") |
|
440
|
if [[ -z "${BUILDKITE_COMMIT:-}" || "${recorded_commit}" != "${BUILDKITE_COMMIT}" ]]; then |
|
441
|
echo "ROCm artifact commit ${recorded_commit} does not match ${BUILDKITE_COMMIT:-unset}" >&2 |
|
442
|
return 1 |
|
443
|
fi |
|
444
|
if [[ -z "${BUILDKITE_BUILD_ID:-}" \ |
|
445
|
|| "${recorded_base}" != *":ci_base-build-${BUILDKITE_BUILD_ID}" ]]; then |
|
446
|
echo "ROCm artifact base is not scoped to Buildkite build ${BUILDKITE_BUILD_ID:-unset}: ${recorded_base}" >&2 |
|
447
|
return 1 |
|
448
|
fi |
|
449
|
if [[ -z "${VLLM_CI_BASE_IMAGE:-}" || "${recorded_base}" != "${VLLM_CI_BASE_IMAGE}" ]]; then |
|
450
|
echo "ROCm artifact base ${recorded_base} does not match ${VLLM_CI_BASE_IMAGE:-unset}" >&2 |
|
451
|
return 1 |
|
452
|
fi |
|
453
|
if [[ "${recorded_wheel}" != "$(basename "${wheels[0]}")" ]]; then |
|
454
|
echo "ROCm artifact wheel manifest ${recorded_wheel} does not match $(basename "${wheels[0]}")" >&2 |
|
455
|
return 1 |
|
456
|
fi |
|
457
|
for required_dir in tests .buildkite requirements; do |
|
458
|
if [[ ! -d "${wheel_dir}/${required_dir}" ]]; then |
|
459
|
echo "ROCm wheel artifact did not contain ${required_dir}/" >&2 |
|
460
|
return 1 |
|
461
|
fi |
|
462
|
done |
|
463
|
|
|
464
|
echo "--- Installing ROCm wheel into pod environment" |
|
465
|
python3 -m pip install --no-deps --force-reinstall "${wheels[0]}" || return 1 |
|
466
|
|
|
467
|
echo "--- Preparing ${workspace_dir} from artifact" |
|
468
|
find "${workspace_dir}" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + || return 1 |
|
469
|
tar -C "${wheel_dir}" \ |
|
470
|
--exclude='*.whl' \ |
|
471
|
--exclude='.vllm-ci-artifact' \ |
|
472
|
--exclude='./.vllm-ci-artifact' \ |
|
473
|
-cf - . | tar --no-same-owner -C "${workspace_dir}" -xf - || return 1 |
|
474
|
if [[ ! -d "${workspace_dir}/tests" ]]; then |
|
475
|
echo "Failed to stage the native test workspace" >&2 |
|
476
|
return 1 |
|
477
|
fi |
|
478
|
|
|
479
|
# The ROCm artifact intentionally contains only the installed wheel and the |
|
480
|
# test workspace. The Python-only compilation job also needs setup.py and the |
|
481
|
# vllm source tree, so overlay the matching Buildkite checkout for that job. |
|
482
|
if [[ "${test_commands}" == *python_only_compile.sh* ]]; then |
|
483
|
checkout="${BUILDKITE_BUILD_CHECKOUT_PATH:-}" |
|
484
|
if [[ -z "${checkout}" || ! -d "${checkout}" ]]; then |
|
485
|
echo "Python-only native CI requires BUILDKITE_BUILD_CHECKOUT_PATH" >&2 |
|
486
|
return 1 |
|
487
|
fi |
|
488
|
if ! git -c "safe.directory=${checkout}" -C "${checkout}" \ |
|
489
|
rev-parse --is-inside-work-tree >/dev/null 2>&1; then |
|
490
|
echo "Buildkite checkout is not a Git worktree: ${checkout}" >&2 |
|
491
|
return 1 |
|
492
|
fi |
|
493
|
checkout_commit=$( |
|
494
|
git -c "safe.directory=${checkout}" -C "${checkout}" rev-parse HEAD |
|
495
|
) || return 1 |
|
496
|
if [[ "${checkout_commit}" != "${recorded_commit}" ]]; then |
|
497
|
echo "Buildkite checkout ${checkout_commit} does not match ROCm artifact ${recorded_commit}" >&2 |
|
498
|
return 1 |
|
499
|
fi |
|
500
|
|
|
501
|
# setup.py normally derives this from .git via setuptools-scm. The native |
|
502
|
# source overlay deliberately excludes Git metadata, so preserve the exact |
|
503
|
# version from the already installed, artifact-matched wheel. |
|
504
|
VLLM_VERSION_OVERRIDE=$( |
|
505
|
python3 -c 'import importlib.metadata as m; print(m.version("vllm"))' |
|
506
|
) || return 1 |
|
507
|
export VLLM_VERSION_OVERRIDE |
|
508
|
VLLM_PRECOMPILED_WHEEL_LOCATION="${wheels[0]}" |
|
509
|
export VLLM_PRECOMPILED_WHEEL_LOCATION |
|
510
|
echo "INFO: native Python-only wheel=${VLLM_PRECOMPILED_WHEEL_LOCATION}" |
|
511
|
|
|
512
|
echo "--- Overlaying full source checkout for Python-only compilation" |
|
513
|
# Archive the verified commit instead of copying the worktree so dirty or |
|
514
|
# untracked agent files cannot contaminate the artifact-matched workspace. |
|
515
|
git -c "safe.directory=${checkout}" -C "${checkout}" \ |
|
516
|
archive --format=tar "${recorded_commit}" \ |
|
517
|
| tar --no-same-owner -C "${workspace_dir}" -xf - || return 1 |
|
518
|
for required_source in setup.py pyproject.toml vllm; do |
|
519
|
if [[ ! -e "${workspace_dir}/${required_source}" ]]; then |
|
520
|
echo "Full source checkout is missing ${required_source}" >&2 |
|
521
|
return 1 |
|
522
|
fi |
|
523
|
done |
|
524
|
fi |
|
525
|
|
|
526
|
return 0 |
|
527
|
} |
|
528
|
|
|
529
|
initialize_native_environment() { |
|
530
|
local job_id="${BUILDKITE_JOB_ID:-${BUILDKITE_PARALLEL_JOB:-local}}" |
|
531
|
local job_id_suffix="" |
|
532
|
local native_root="" |
|
533
|
local hf_fstype="" |
|
534
|
local hf_mount="" |
|
535
|
|
|
536
|
if [[ "$(id -u)" -ne 0 ]]; then |
|
537
|
echo "Native ROCm CI currently requires the ci_base container to run as root" >&2 |
|
538
|
return 1 |
|
539
|
fi |
|
540
|
|
|
541
|
job_id="${job_id//[^A-Za-z0-9_.-]/_}" |
|
542
|
job_id_suffix="${job_id##*-}" |
|
543
|
job_id_suffix="${job_id_suffix:0:12}" |
|
544
|
native_root="/tmp/vllm-native-${job_id}" |
|
545
|
TMPDIR="/tmp/vllm-${job_id_suffix}/tmp" |
|
546
|
VLLM_RPC_BASE_PATH="/tmp" |
|
547
|
TORCHINDUCTOR_CACHE_DIR="${native_root}/cache/torchinductor" |
|
548
|
TRITON_CACHE_DIR="${native_root}/cache/triton" |
|
549
|
VLLM_CACHE_ROOT="${native_root}/cache/vllm" |
|
550
|
XDG_CACHE_HOME="${native_root}/cache/xdg" |
|
551
|
: "${HF_HOME:=/home/buildkite-agent/huggingface}" |
|
552
|
# datasets uses POSIX locks that are unsupported by the shared HF NFS cache. |
|
553
|
# Keep processed datasets job-local while retaining the persistent Hub cache. |
|
554
|
HF_DATASETS_CACHE="${native_root}/cache/huggingface/datasets" |
|
555
|
# openai-harmony downloads its tiktoken vocab on first use and caches it under |
|
556
|
# $TMPDIR by default, which is job-local; keep it with the persistent Hub cache. |
|
557
|
TIKTOKEN_RS_CACHE_DIR="${HF_HOME}/tiktoken-rs-cache" |
|
558
|
: "${HF_HUB_DOWNLOAD_TIMEOUT:=300}" |
|
559
|
: "${HF_HUB_ETAG_TIMEOUT:=60}" |
|
560
|
if [[ "${VLLM_CI_EXPECTED_GPU_COUNT:-1}" == "0" ]]; then |
|
561
|
# CPU-only native jobs intentionally reuse the ROCm wheel. Make that target |
|
562
|
# explicit so platform selection does not depend on wheel metadata. |
|
563
|
VLLM_TARGET_DEVICE=cpu |
|
564
|
export VLLM_TARGET_DEVICE |
|
565
|
fi |
|
566
|
export TMPDIR VLLM_RPC_BASE_PATH |
|
567
|
export TORCHINDUCTOR_CACHE_DIR TRITON_CACHE_DIR VLLM_CACHE_ROOT XDG_CACHE_HOME |
|
568
|
export HF_HOME HF_DATASETS_CACHE HF_HUB_DOWNLOAD_TIMEOUT HF_HUB_ETAG_TIMEOUT |
|
569
|
export TIKTOKEN_RS_CACHE_DIR |
|
570
|
export PYTORCH_ROCM_ARCH="" |
|
571
|
|
|
572
|
mkdir -p "${TMPDIR}" \ |
|
573
|
"${TORCHINDUCTOR_CACHE_DIR}" \ |
|
574
|
"${TRITON_CACHE_DIR}" \ |
|
575
|
"${VLLM_CACHE_ROOT}" \ |
|
576
|
"${XDG_CACHE_HOME}" \ |
|
577
|
"${HF_HOME}" \ |
|
578
|
"${TIKTOKEN_RS_CACHE_DIR}" \ |
|
579
|
"${HF_DATASETS_CACHE}" || return 1 |
|
580
|
|
|
581
|
echo "Native compile caches: VLLM_CACHE_ROOT=${VLLM_CACHE_ROOT} TORCHINDUCTOR_CACHE_DIR=${TORCHINDUCTOR_CACHE_DIR}" |
|
582
|
|
|
583
|
if [[ "${VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE:-0}" == "1" ]]; then |
|
584
|
if ! command -v findmnt >/dev/null 2>&1; then |
|
585
|
echo "findmnt is required to verify the native Hugging Face cache mount" >&2 |
|
586
|
return 1 |
|
587
|
fi |
|
588
|
hf_mount=$(findmnt -n -T "${HF_HOME}" -o TARGET 2>/dev/null || true) |
|
589
|
if [[ -z "${hf_mount}" || "${hf_mount}" == "/" ]]; then |
|
590
|
echo "Native CI requires a persistent volume mounted at or above ${HF_HOME}" >&2 |
|
591
|
return 1 |
|
592
|
fi |
|
593
|
fi |
|
594
|
|
|
595
|
if command -v findmnt >/dev/null 2>&1; then |
|
596
|
hf_fstype=$(findmnt -n -T "${HF_HOME}" -o FSTYPE 2>/dev/null || true) |
|
597
|
fi |
|
598
|
if [[ "${hf_fstype}" == nfs || "${hf_fstype}" == nfs4 ]]; then |
|
599
|
# Keep hf-xet state local and avoid vectored writes on shared NFS. |
|
600
|
export HF_XET_CACHE="${native_root}/cache/hf-xet" |
|
601
|
export HF_XET_HIGH_PERFORMANCE=0 |
|
602
|
export HF_XET_RECONSTRUCTION_USE_VECTORED_WRITE=0 |
|
603
|
mkdir -p "${HF_XET_CACHE}" || return 1 |
|
604
|
echo "Configured hf-xet for shared ${hf_fstype} cache at ${HF_HOME}" |
|
605
|
fi |
|
606
|
} |
|
607
|
|
|
608
|
check_dpx_gpu_exclusivity() { |
|
609
|
local devices=(/dev/dri/renderD*) |
|
610
|
local device_id lock_status |
|
611
|
|
|
612
|
if [[ "${#devices[@]}" -ne 1 || ! -c "${devices[0]}" ]]; then |
|
613
|
echo "DPX guard requires exactly one render device; refusing to start tests." >&2 |
|
614
|
return 1 |
|
615
|
fi |
|
616
|
# This queue mounts the same host-local HF cache into every pod on the node. |
|
617
|
if ! mountpoint -q "${HF_HOME}"; then |
|
618
|
echo "DPX guard requires the shared HF cache mount; refusing to start tests." >&2 |
|
619
|
return 1 |
|
620
|
fi |
|
621
|
device_id=$(stat -Lc '%t-%T' "${devices[0]}") || return 1 |
|
622
|
mkdir -p "${HF_HOME}/.vllm-dpx-locks" || return 1 |
|
623
|
# Hold the descriptor through workload and teardown; never delete the file. |
|
624
|
exec {dpx_gpu_lock_fd}>>"${HF_HOME}/.vllm-dpx-locks/${device_id}.lock" || return 1 |
|
625
|
flock -n -E 75 "${dpx_gpu_lock_fd}" && return 0 |
|
626
|
lock_status=$? |
|
627
|
if [[ "${lock_status}" -eq 75 ]]; then |
|
628
|
echo "DPX GPU collision: ${devices[0]} (${device_id}) is already locked by another CI job; refusing to start tests." >&2 |
|
629
|
else |
|
630
|
echo "DPX GPU lock failed (status ${lock_status}); refusing to start tests." >&2 |
|
631
|
fi |
|
632
|
return 1 |
|
633
|
} |
|
634
|
|
|
635
|
run_native_preflight() { |
|
636
|
local expected_gpus="${VLLM_CI_EXPECTED_GPU_COUNT:-1}" |
|
637
|
|
|
638
|
if [[ ! "${expected_gpus}" =~ ^[0-9]+$ ]]; then |
|
639
|
echo "Invalid VLLM_CI_EXPECTED_GPU_COUNT=${expected_gpus}" >&2 |
|
640
|
return 1 |
|
641
|
fi |
|
642
|
|
|
643
|
python3 -c "import encodings, importlib.metadata as im, importlib.util as iu; [im.version(d) for d in ('transformers', 'torch', 'ray', 'sympy', 'markupsafe', 'vllm')]; missing=[m for m in ('torch.utils.model_zoo', 'transformers.models.nomic_bert', 'ray.dag', 'sympy.physics', 'markupsafe._speedups') if iu.find_spec(m) is None]; assert not missing, missing" || return 1 |
|
644
|
|
|
645
|
if [[ "${expected_gpus}" == "0" ]]; then |
|
646
|
echo "Native CPU-only AMD job: skipping ROCm device validation" |
|
647
|
return 0 |
|
648
|
fi |
|
649
|
|
|
650
|
echo "--- ROCm info" |
|
651
|
rocminfo || return 1 |
|
652
|
VLLM_CI_EXPECTED_GPU_COUNT="${expected_gpus}" python3 - <<'PY' |
|
653
|
import os |
|
654
|
|
|
655
|
import torch |
|
656
|
|
|
657
|
expected = int(os.environ["VLLM_CI_EXPECTED_GPU_COUNT"]) |
|
658
|
assert torch.version.hip, "PyTorch is not a ROCm build" |
|
659
|
assert torch.cuda.is_available(), "ROCm GPU is not available to PyTorch" |
|
660
|
actual = torch.cuda.device_count() |
|
661
|
assert actual == expected, f"Expected {expected} ROCm GPU(s), found {actual}" |
|
662
|
PY |
|
663
|
} |
|
664
|
|
|
665
|
is_multi_node() { |
|
666
|
local cmds="$1" |
|
667
|
# Primary signal: NUM_NODES environment variable set by the pipeline |
|
668
|
if [[ "${NUM_NODES:-1}" -gt 1 ]]; then |
|
669
|
return 0 |
|
670
|
fi |
|
671
|
# Fallback: detect the bracket syntax structurally |
|
672
|
# Pattern: [...] && [...] (per-node command arrays) |
|
673
|
if [[ "$cmds" == *'] && ['* ]]; then |
|
674
|
return 0 |
|
675
|
fi |
|
676
|
return 1 |
|
677
|
} |
|
678
|
|
|
679
|
append_failure_diagnostic_section() { |
|
680
|
local log_file=$1 |
|
681
|
local title=$2 |
|
682
|
|
|
683
|
printf '\n===============================================================================\n' \ |
|
684
|
>> "${log_file}" |
|
685
|
printf '%s\n' "${title}" >> "${log_file}" |
|
686
|
printf '===============================================================================\n' \ |
|
687
|
>> "${log_file}" |
|
688
|
} |
|
689
|
|
|
690
|
append_failure_diagnostic_note() { |
|
691
|
local log_file=$1 |
|
692
|
shift |
|
693
|
printf '%s\n' "$@" >> "${log_file}" |
|
694
|
} |
|
695
|
|
|
696
|
run_failure_diagnostic() { |
|
697
|
local log_file=$1 |
|
698
|
local probe_deadline=$2 |
|
699
|
shift 2 |
|
700
|
local command_status=0 |
|
701
|
local command_timeout_seconds=${amd_diagnostics_command_timeout_seconds} |
|
702
|
local remaining_seconds=$((probe_deadline - SECONDS)) |
|
703
|
|
|
704
|
{ |
|
705
|
printf '\n$' |
|
706
|
printf ' %q' "$@" |
|
707
|
printf '\n' |
|
708
|
} >> "${log_file}" |
|
709
|
|
|
710
|
if ! command -v timeout >/dev/null 2>&1; then |
|
711
|
echo "[timeout unavailable; bounded diagnostic command skipped]" \ |
|
712
|
>> "${log_file}" |
|
713
|
return 0 |
|
714
|
fi |
|
715
|
if [[ "${remaining_seconds}" -le 0 ]]; then |
|
716
|
echo "[diagnostic probe budget exhausted; command skipped]" \ |
|
717
|
>> "${log_file}" |
|
718
|
return 0 |
|
719
|
fi |
|
720
|
if [[ "${remaining_seconds}" -lt "${command_timeout_seconds}" ]]; then |
|
721
|
command_timeout_seconds=${remaining_seconds} |
|
722
|
fi |
|
723
|
|
|
724
|
timeout --kill-after=1s "${command_timeout_seconds}s" "$@" \ |
|
725
|
>> "${log_file}" 2>&1 |
|
726
|
command_status=$? |
|
727
|
if [[ "${command_status}" -ne 0 ]]; then |
|
728
|
echo "[diagnostic command exited with status ${command_status}]" \ |
|
729
|
>> "${log_file}" |
|
730
|
fi |
|
731
|
return 0 |
|
732
|
} |
|
733
|
|
|
734
|
upload_failure_diagnostics_artifact() { |
|
735
|
local upload_root=$1 upload_path=$2 agent_bin="" |
|
736
|
local agent_upload_help="" |
|
737
|
local -a upload_options=() |
|
738
|
|
|
739
|
if [[ -n "${BUILDKITE_BIN_PATH:-}" \ |
|
740
|
&& -x "${BUILDKITE_BIN_PATH}/buildkite-agent" ]]; then |
|
741
|
agent_bin="${BUILDKITE_BIN_PATH}/buildkite-agent" |
|
742
|
else |
|
743
|
agent_bin=$(command -v buildkite-agent 2>/dev/null || true) |
|
744
|
fi |
|
745
|
if [[ -z "${agent_bin}" && -x /workspace/buildkite-agent ]]; then |
|
746
|
agent_bin=/workspace/buildkite-agent |
|
747
|
fi |
|
748
|
[[ -n "${agent_bin}" && -n "${BUILDKITE_JOB_ID:-}" ]] || return 1 |
|
749
|
command -v timeout >/dev/null 2>&1 || return 1 |
|
750
|
|
|
751
|
# Older AMD runners predate the literal upload options. A restricted path |
|
752
|
# has no glob or delimiter characters, so it is safe with either agent. |
|
753
|
if [[ ! "${upload_path}" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]*(/[A-Za-z0-9_][A-Za-z0-9_.-]*)*$ ]]; then |
|
754
|
agent_upload_help=$("${agent_bin}" artifact upload --help 2>&1 || true) |
|
755
|
if [[ "${agent_upload_help}" != *"--literal"* \ |
|
756
|
|| "${agent_upload_help}" != *"--delimiter"* ]]; then |
|
757
|
return 1 |
|
758
|
fi |
|
759
|
upload_options=(--literal --delimiter "") |
|
760
|
fi |
|
761
|
|
|
762
|
( |
|
763
|
cd "${upload_root}" || exit 1 |
|
764
|
BUILDKITE_AGENT_DEBUG=false \ |
|
765
|
BUILDKITE_AGENT_DEBUG_HTTP=false \ |
|
766
|
BUILDKITE_AGENT_TRACE_HTTP=false \ |
|
767
|
BUILDKITE_AGENT_LOG_LEVEL=error \ |
|
768
|
timeout --kill-after=2s "${amd_diagnostics_upload_timeout_seconds}s" \ |
|
769
|
"${agent_bin}" artifact upload \ |
|
770
|
"${upload_options[@]}" "./${upload_path}" |
|
771
|
) >/dev/null 2>&1 |
|
772
|
} |
|
773
|
|
|
774
|
append_failure_diagnostic_file() { |
|
775
|
local log_file=$1 |
|
776
|
local label=$2 |
|
777
|
local path=$3 |
|
778
|
local command_status=0 |
|
779
|
|
|
780
|
printf '\n%s:\n' "${label}" >> "${log_file}" |
|
781
|
printf '$ cat %q\n' "${path}" >> "${log_file}" |
|
782
|
if [[ ! -r "${path}" ]]; then |
|
783
|
echo "[file unavailable]" >> "${log_file}" |
|
784
|
return 0 |
|
785
|
fi |
|
786
|
|
|
787
|
cat "${path}" >> "${log_file}" 2>&1 |
|
788
|
command_status=$? |
|
789
|
if [[ "${command_status}" -ne 0 ]]; then |
|
790
|
echo "[diagnostic file read exited with status ${command_status}]" \ |
|
791
|
>> "${log_file}" |
|
792
|
fi |
|
793
|
return 0 |
|
794
|
} |
|
795
|
|
|
796
|
decode_mountinfo_path() { |
|
797
|
local value=$1 |
|
798
|
|
|
799
|
value=${value//\\040/ } |
|
800
|
value=${value//\\011/$'\t'} |
|
801
|
value=${value//\\012/$'\n'} |
|
802
|
value=${value//\\134/\\} |
|
803
|
printf '%s\n' "${value}" |
|
804
|
} |
|
805
|
|
|
806
|
resolve_current_cgroup_v2_dir() { |
|
807
|
local cgroup_path="" |
|
808
|
local mount_root="" |
|
809
|
local mount_point="" |
|
810
|
local relative_path="" |
|
811
|
local candidate="" |
|
812
|
|
|
813
|
cgroup_path=$(awk -F: '$2 == "" { print $3; exit }' \ |
|
814
|
/proc/self/cgroup 2>/dev/null) |
|
815
|
[[ -n "${cgroup_path}" ]] || return 1 |
|
816
|
|
|
817
|
while IFS=$'\t' read -r mount_root mount_point; do |
|
818
|
mount_root=$(decode_mountinfo_path "${mount_root}") |
|
819
|
mount_point=$(decode_mountinfo_path "${mount_point}") |
|
820
|
if [[ "${cgroup_path}" == "/" ]]; then |
|
821
|
relative_path="" |
|
822
|
elif [[ "${mount_root}" == "/" ]]; then |
|
823
|
relative_path="${cgroup_path}" |
|
824
|
elif [[ "${cgroup_path}" == "${mount_root}" ]]; then |
|
825
|
relative_path="" |
|
826
|
elif [[ "${cgroup_path}" == "${mount_root}/"* ]]; then |
|
827
|
relative_path="/${cgroup_path#"${mount_root}/"}" |
|
828
|
else |
|
829
|
continue |
|
830
|
fi |
|
831
|
|
|
832
|
candidate="${mount_point%/}${relative_path}" |
|
833
|
if [[ -d "${candidate}" ]]; then |
|
834
|
printf '%s\n' "${candidate}" |
|
835
|
return 0 |
|
836
|
fi |
|
837
|
done < <( |
|
838
|
awk ' |
|
839
|
{ |
|
840
|
for (i = 6; i <= NF; i++) { |
|
841
|
if ($i == "-" && $(i + 1) == "cgroup2") { |
|
842
|
print $4 "\t" $5 |
|
843
|
break |
|
844
|
} |
|
845
|
} |
|
846
|
} |
|
847
|
' /proc/self/mountinfo 2>/dev/null |
|
848
|
) |
|
849
|
return 1 |
|
850
|
} |
|
851
|
|
|
852
|
collect_cgroup_files() { |
|
853
|
local log_file=$1 |
|
854
|
local title=$2 |
|
855
|
local cgroup_dir=$3 |
|
856
|
shift 3 |
|
857
|
local cgroup_file="" |
|
858
|
local files_collected=0 |
|
859
|
|
|
860
|
append_failure_diagnostic_section "${log_file}" "${title}" |
|
861
|
if [[ -z "${cgroup_dir}" || ! -d "${cgroup_dir}" ]]; then |
|
862
|
append_failure_diagnostic_note "${log_file}" \ |
|
863
|
"resolved_cgroup_path=unavailable" |
|
864
|
return 0 |
|
865
|
fi |
|
866
|
|
|
867
|
append_failure_diagnostic_note "${log_file}" \ |
|
868
|
"resolved_cgroup_path=${cgroup_dir}" |
|
869
|
for cgroup_file in "$@"; do |
|
870
|
if [[ -r "${cgroup_dir}/${cgroup_file}" ]]; then |
|
871
|
append_failure_diagnostic_file "${log_file}" "${cgroup_file}" \ |
|
872
|
"${cgroup_dir}/${cgroup_file}" |
|
873
|
files_collected=$((files_collected + 1)) |
|
874
|
fi |
|
875
|
done |
|
876
|
if [[ "${files_collected}" -eq 0 ]]; then |
|
877
|
append_failure_diagnostic_note "${log_file}" \ |
|
878
|
"[no requested cgroup files were readable]" |
|
879
|
fi |
|
880
|
} |
|
881
|
|
|
882
|
collect_cgroup_diagnostics() { |
|
883
|
local log_file=$1 |
|
884
|
local cgroup_dir="" |
|
885
|
|
|
886
|
append_failure_diagnostic_section "${log_file}" "Cgroup membership" |
|
887
|
append_failure_diagnostic_file "${log_file}" "current_process" \ |
|
888
|
/proc/self/cgroup |
|
889
|
append_failure_diagnostic_file "${log_file}" "pid_namespace_init_process" \ |
|
890
|
/proc/1/cgroup |
|
891
|
|
|
892
|
cgroup_dir=$(resolve_current_cgroup_v2_dir 2>/dev/null || true) |
|
893
|
collect_cgroup_files "${log_file}" \ |
|
894
|
"Cgroup v2 current-process resource state" "${cgroup_dir}" \ |
|
895
|
cgroup.type cgroup.events \ |
|
896
|
memory.current memory.peak memory.max memory.high memory.low memory.min \ |
|
897
|
memory.oom.group memory.swap.current memory.swap.max \ |
|
898
|
memory.events memory.events.local memory.swap.events memory.stat \ |
|
899
|
memory.pressure \ |
|
900
|
cpu.max cpu.max.burst cpu.weight cpu.stat cpu.pressure \ |
|
901
|
cpuset.cpus cpuset.cpus.effective cpuset.mems cpuset.mems.effective \ |
|
902
|
pids.current pids.max pids.events pids.events.local \ |
|
903
|
io.stat io.max io.weight io.pressure |
|
904
|
if [[ -n "${cgroup_dir}" && -r "${cgroup_dir}/memory.events" ]]; then |
|
905
|
amd_diagnostics_memory_events_path="${cgroup_dir}/memory.events" |
|
906
|
fi |
|
907
|
} |
|
908
|
|
|
909
|
collect_process_diagnostics() { |
|
910
|
local log_file=$1 |
|
911
|
local probe_deadline=$2 |
|
912
|
local shell_proc_dir="/proc/$$" |
|
913
|
|
|
914
|
append_failure_diagnostic_section "${log_file}" \ |
|
915
|
"Runner process constraints and PID-namespace-visible processes" |
|
916
|
append_failure_diagnostic_file "${log_file}" "diagnostic_shell_limits" \ |
|
917
|
"${shell_proc_dir}/limits" |
|
918
|
if command -v awk >/dev/null 2>&1; then |
|
919
|
# shellcheck disable=SC2016 # The expression is evaluated by awk. |
|
920
|
run_failure_diagnostic "${log_file}" "${probe_deadline}" awk ' |
|
921
|
$1 ~ /^(Pid:|PPid:|Threads:|VmPeak:|VmSize:|VmHWM:|VmRSS:|RssAnon:|RssFile:|RssShmem:|Cpus_allowed_list:|Mems_allowed_list:|voluntary_ctxt_switches:|nonvoluntary_ctxt_switches:)$/ { |
|
922
|
print |
|
923
|
} |
|
924
|
' "${shell_proc_dir}/status" |
|
925
|
fi |
|
926
|
if command -v nproc >/dev/null 2>&1; then |
|
927
|
run_failure_diagnostic "${log_file}" "${probe_deadline}" nproc |
|
928
|
fi |
|
929
|
run_failure_diagnostic "${log_file}" "${probe_deadline}" \ |
|
930
|
/bin/bash -c 'ulimit -a' |
|
931
|
if command -v ps >/dev/null 2>&1 && command -v head >/dev/null 2>&1; then |
|
932
|
append_failure_diagnostic_note "${log_file}" \ |
|
933
|
"process_snapshot=top_100_by_rss" |
|
934
|
run_failure_diagnostic "${log_file}" "${probe_deadline}" \ |
|
935
|
/bin/bash -c \ |
|
936
|
'ps -eo pid,ppid,stat,etimes,nlwp,pcpu,rss,vsz,comm --sort=-rss | head -n 101' |
|
937
|
fi |
|
938
|
return 0 |
|
939
|
} |
|
940
|
|
|
941
|
collect_mount_diagnostic() { |
|
942
|
local log_file=$1 |
|
943
|
local probe_deadline=$2 |
|
944
|
local label=$3 |
|
945
|
local path=$4 |
|
946
|
|
|
947
|
printf '\n%s (%s):\n' "${label}" "${path}" >> "${log_file}" |
|
948
|
if [[ ! -e "${path}" ]]; then |
|
949
|
echo "[path unavailable]" >> "${log_file}" |
|
950
|
return 0 |
|
951
|
fi |
|
952
|
if command -v findmnt >/dev/null 2>&1; then |
|
953
|
run_failure_diagnostic "${log_file}" "${probe_deadline}" findmnt \ |
|
954
|
-n -T "${path}" -o TARGET,FSTYPE,VFS-OPTIONS |
|
955
|
fi |
|
956
|
if command -v df >/dev/null 2>&1; then |
|
957
|
run_failure_diagnostic "${log_file}" "${probe_deadline}" df -h \ |
|
958
|
--output=fstype,size,used,avail,pcent,itotal,iused,iavail,ipcent,target \ |
|
959
|
-- "${path}" |
|
960
|
fi |
|
961
|
return 0 |
|
962
|
} |
|
963
|
|
|
964
|
collect_execution_resource_diagnostics() { |
|
965
|
local log_file=$1 |
|
966
|
local probe_deadline=$2 |
|
967
|
|
|
968
|
# free and /proc/meminfo generally describe the node from a pod. Cgroups are |
|
969
|
# the live source for this container's resource limits, usage, and events. |
|
970
|
collect_cgroup_diagnostics "${log_file}" |
|
971
|
collect_process_diagnostics "${log_file}" "${probe_deadline}" |
|
972
|
|
|
973
|
append_failure_diagnostic_section "${log_file}" \ |
|
974
|
"Mount-visible filesystem capacity" |
|
975
|
append_failure_diagnostic_note "${log_file}" \ |
|
976
|
"These values are filesystem views, not Kubernetes ephemeral-storage quotas." |
|
977
|
collect_mount_diagnostic "${log_file}" "${probe_deadline}" \ |
|
978
|
"checkout" "${amd_diagnostics_checkout_root}" |
|
979
|
if is_native_runtime; then |
|
980
|
collect_mount_diagnostic "${log_file}" "${probe_deadline}" \ |
|
981
|
"native workspace emptyDir" "${amd_diagnostics_workspace}" |
|
982
|
else |
|
983
|
collect_mount_diagnostic "${log_file}" "${probe_deadline}" \ |
|
984
|
"test workspace (if mounted)" "${amd_diagnostics_workspace}" |
|
985
|
fi |
|
986
|
collect_mount_diagnostic "${log_file}" "${probe_deadline}" \ |
|
987
|
"temporary directory" "${TMPDIR:-/tmp}" |
|
988
|
if [[ "${TMPDIR:-/tmp}" != "/tmp" ]]; then |
|
989
|
collect_mount_diagnostic "${log_file}" "${probe_deadline}" \ |
|
990
|
"system temporary directory" /tmp |
|
991
|
fi |
|
992
|
if is_native_runtime; then |
|
993
|
collect_mount_diagnostic "${log_file}" "${probe_deadline}" \ |
|
994
|
"shared-memory emptyDir" /dev/shm |
|
995
|
else |
|
996
|
collect_mount_diagnostic "${log_file}" "${probe_deadline}" \ |
|
997
|
"shared-memory mount" /dev/shm |
|
998
|
fi |
|
999
|
if [[ -n "${HF_HOME:-}" ]]; then |
|
1000
|
collect_mount_diagnostic "${log_file}" "${probe_deadline}" \ |
|
1001
|
"Hugging Face cache" "${HF_HOME}" |
|
1002
|
fi |
|
1003
|
return 0 |
|
1004
|
} |
|
1005
|
|
|
1006
|
collect_amd_device_nodes() { |
|
1007
|
local log_file=$1 |
|
1008
|
local probe_deadline=$2 |
|
1009
|
local device="" |
|
1010
|
local device_count=0 |
|
1011
|
|
|
1012
|
for device in /dev/kfd /dev/dri/renderD*; do |
|
1013
|
if [[ ! -e "${device}" ]]; then |
|
1014
|
continue |
|
1015
|
fi |
|
1016
|
device_count=$((device_count + 1)) |
|
1017
|
if command -v stat >/dev/null 2>&1; then |
|
1018
|
run_failure_diagnostic "${log_file}" "${probe_deadline}" stat -Lc \ |
|
1019
|
'%n type=%F mode=%a owner=%u:%g device=%t:%T' -- "${device}" |
|
1020
|
else |
|
1021
|
append_failure_diagnostic_note "${log_file}" "device=${device}" |
|
1022
|
fi |
|
1023
|
done |
|
1024
|
if [[ "${device_count}" -eq 0 ]]; then |
|
1025
|
append_failure_diagnostic_note "${log_file}" \ |
|
1026
|
"No /dev/kfd or /dev/dri/renderD* device nodes are visible." |
|
1027
|
fi |
|
1028
|
return 0 |
|
1029
|
} |
|
1030
|
|
|
1031
|
collect_rocm_failure_diagnostics() { |
|
1032
|
local exit_code=$1 |
|
1033
|
local job_id="${BUILDKITE_JOB_ID:-local}" |
|
1034
|
local retry_count="${BUILDKITE_RETRY_COUNT:-0}" |
|
1035
|
local parallel_job="${BUILDKITE_PARALLEL_JOB:-0}" |
|
1036
|
local diagnostics_relative_path="" |
|
1037
|
local diagnostics_path="" |
|
1038
|
local diagnostics_parent="" |
|
1039
|
local diagnostics_fallback_root="" |
|
1040
|
local diagnostics_storage="checkout" |
|
1041
|
local diagnostics_artifact_summary="" |
|
1042
|
local checkout_real="" |
|
1043
|
local diagnostics_parent_real="" |
|
1044
|
local exit_signal="" |
|
1045
|
local probe_deadline=0 |
|
1046
|
local runtime="single-node-docker" |
|
1047
|
local diagnostics_scope="outer-runner-after-test-container-exit" |
|
1048
|
local identity_label="runner" |
|
1049
|
local identity_value="${HOSTNAME:-unknown}" |
|
1050
|
local k8s_pod="unknown" |
|
1051
|
local k8s_namespace="${amd_diagnostics_k8s_namespace}" |
|
1052
|
local oom_kill_count="" |
|
1053
|
local summary_identity_label="Runner" |
|
1054
|
local -a summary_rows=() |
|
1055
|
|
|
1056
|
if [[ "${amd_diagnostics_collected}" == "1" ]]; then |
|
1057
|
return 0 |
|
1058
|
fi |
|
1059
|
amd_diagnostics_collected=1 |
|
1060
|
|
|
1061
|
if is_native_runtime; then |
|
1062
|
runtime="native-kubernetes" |
|
1063
|
diagnostics_scope="current-container-cgroup-and-namespaces" |
|
1064
|
if [[ -n "${amd_diagnostics_pod_name}" ]]; then |
|
1065
|
identity_label="pod" |
|
1066
|
identity_value="${amd_diagnostics_pod_name}" |
|
1067
|
k8s_pod="${amd_diagnostics_pod_name}" |
|
1068
|
else |
|
1069
|
identity_label="container_hostname" |
|
1070
|
fi |
|
1071
|
elif [[ "${amd_diagnostics_execution_mode}" == "multi-node" ]]; then |
|
1072
|
runtime="multi-node-docker" |
|
1073
|
fi |
|
1074
|
if [[ "${k8s_namespace}" == "unknown" \ |
|
1075
|
&& -r /var/run/secrets/kubernetes.io/serviceaccount/namespace ]]; then |
|
1076
|
k8s_namespace=$( |
|
1077
|
tr -d '\r\n' \ |
|
1078
|
< /var/run/secrets/kubernetes.io/serviceaccount/namespace 2>/dev/null |
|
1079
|
) |
|
1080
|
fi |
|
1081
|
|
|
1082
|
job_id="${job_id//[^A-Za-z0-9_.-]/_}" |
|
1083
|
retry_count="${retry_count//[^A-Za-z0-9_.-]/_}" |
|
1084
|
parallel_job="${parallel_job//[^A-Za-z0-9_.-]/_}" |
|
1085
|
|
|
1086
|
if [[ "${exit_code}" -gt 128 && "${exit_code}" -le 192 ]]; then |
|
1087
|
exit_signal=$(kill -l "$((exit_code - 128))" 2>/dev/null || true) |
|
1088
|
fi |
|
1089
|
|
|
1090
|
# Buildkite keeps the supplied artifact path. Restrict the configurable |
|
1091
|
# directory to a checkout-relative path so the UI shows a clean artifact key. |
|
1092
|
if [[ -z "${amd_diagnostics_dir}" \ |
|
1093
|
|| "${amd_diagnostics_dir}" == /* \ |
|
1094
|
|| "${amd_diagnostics_dir}" == *".."* ]]; then |
|
1095
|
echo "WARNING: ignoring unsafe VLLM_CI_DIAGNOSTICS_DIR" |
|
1096
|
amd_diagnostics_dir="artifacts/amd-gpu-diagnostics" |
|
1097
|
fi |
|
1098
|
diagnostics_relative_path="${amd_diagnostics_dir}/${job_id}/diagnostics.log" |
|
1099
|
diagnostics_path="${amd_diagnostics_checkout_root}/${diagnostics_relative_path}" |
|
1100
|
diagnostics_parent=$(dirname "${diagnostics_path}") |
|
1101
|
checkout_real=$(readlink -m "${amd_diagnostics_checkout_root}" 2>/dev/null || true) |
|
1102
|
diagnostics_parent_real=$(readlink -m "${diagnostics_parent}" 2>/dev/null || true) |
|
1103
|
if [[ -z "${checkout_real}" || -z "${diagnostics_parent_real}" \ |
|
1104
|
|| ("${diagnostics_parent_real}" != "${checkout_real}" \ |
|
1105
|
&& "${diagnostics_parent_real}" != "${checkout_real}/"*) ]] \ |
|
1106
|
|| ! mkdir -p "${diagnostics_parent}" \ |
|
1107
|
|| [[ -L "${diagnostics_path}" ]] \ |
|
1108
|
|| ! (set -o noclobber; : > "${diagnostics_path}") 2>/dev/null; then |
|
1109
|
echo "WARNING: unable to create AMD CI diagnostics in the checkout; using temporary storage." |
|
1110
|
diagnostics_path="" |
|
1111
|
if diagnostics_fallback_root=$(mktemp -d -t \ |
|
1112
|
vllm-amd-diagnostics.XXXXXX 2>/dev/null); then |
|
1113
|
diagnostics_path="${diagnostics_fallback_root}/${diagnostics_relative_path}" |
|
1114
|
diagnostics_parent=$(dirname "${diagnostics_path}") |
|
1115
|
if (umask 077; mkdir -p "${diagnostics_parent}" \ |
|
1116
|
&& (set -o noclobber; : > "${diagnostics_path}")) 2>/dev/null; then |
|
1117
|
diagnostics_storage="temporary-fallback" |
|
1118
|
else |
|
1119
|
rm -rf -- "${diagnostics_fallback_root}" || true |
|
1120
|
diagnostics_fallback_root="" |
|
1121
|
diagnostics_path="" |
|
1122
|
fi |
|
1123
|
fi |
|
1124
|
fi |
|
1125
|
if [[ -z "${diagnostics_path}" ]]; then |
|
1126
|
echo "WARNING: unable to create AMD CI diagnostics at ${diagnostics_relative_path}." |
|
1127
|
printf '\nAMD CI failure summary\n' |
|
1128
|
printf '%-22s | %s\n' \ |
|
1129
|
"Field" "Value" \ |
|
1130
|
"----------------------" "-----" \ |
|
1131
|
"Exit code" "${exit_code}" \ |
|
1132
|
"Signal" "${exit_signal:-none}" \ |
|
1133
|
"Runtime" "${runtime}" \ |
|
1134
|
"Pod/container" "${identity_value}" \ |
|
1135
|
"Diagnostics artifact" "unavailable: ${diagnostics_relative_path}" |
|
1136
|
return 0 |
|
1137
|
fi |
|
1138
|
|
|
1139
|
{ |
|
1140
|
echo "AMD CI failure diagnostics" |
|
1141
|
echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" |
|
1142
|
echo "exit_code=${exit_code}" |
|
1143
|
echo "exit_signal=${exit_signal:-none}" |
|
1144
|
echo "build_url=${BUILDKITE_BUILD_URL:-unknown}" |
|
1145
|
echo "job_id=${BUILDKITE_JOB_ID:-unknown}" |
|
1146
|
echo "step_key=${BUILDKITE_STEP_KEY:-unknown}" |
|
1147
|
echo "step_label=${BUILDKITE_LABEL:-unknown}" |
|
1148
|
echo "test_group=${amd_diagnostics_test_group}" |
|
1149
|
echo "retry_count=${retry_count}" |
|
1150
|
echo "parallel_job=${parallel_job}" |
|
1151
|
echo "execution_mode=${amd_diagnostics_execution_mode}" |
|
1152
|
echo "runtime=${runtime}" |
|
1153
|
echo "diagnostics_scope=${diagnostics_scope}" |
|
1154
|
echo "expected_gpu_count=${amd_diagnostics_expected_gpu_count}" |
|
1155
|
echo "probe_budget_seconds=${amd_diagnostics_probe_budget_seconds}" |
|
1156
|
echo "command_timeout_seconds=${amd_diagnostics_command_timeout_seconds}" |
|
1157
|
echo "diagnostics_storage=${diagnostics_storage}" |
|
1158
|
echo "agent_name=${BUILDKITE_AGENT_NAME:-unknown}" |
|
1159
|
echo "container_hostname=${HOSTNAME:-unknown}" |
|
1160
|
echo "k8s_pod=${k8s_pod}" |
|
1161
|
echo "k8s_namespace=${k8s_namespace:-unknown}" |
|
1162
|
echo "k8s_node=${amd_diagnostics_k8s_node_name:-unknown}" |
|
1163
|
echo "kernel=$(uname -srmo 2>/dev/null || echo unknown)" |
|
1164
|
} > "${diagnostics_path}" |
|
1165
|
|
|
1166
|
probe_deadline=$((SECONDS + amd_diagnostics_probe_budget_seconds)) |
|
1167
|
|
|
1168
|
# Cgroup, process, and mount state is fast and most useful for explaining OOM, |
|
1169
|
# throttling, PID exhaustion, and emptyDir pressure. Capture it before external |
|
1170
|
# probes can consume the shared deadline. |
|
1171
|
collect_execution_resource_diagnostics \ |
|
1172
|
"${diagnostics_path}" "${probe_deadline}" |
|
1173
|
|
|
1174
|
append_failure_diagnostic_section "${diagnostics_path}" \ |
|
1175
|
"AMD GPU diagnostics (tool-visible devices)" |
|
1176
|
if [[ "${amd_diagnostics_expected_gpu_count}" == "0" ]]; then |
|
1177
|
append_failure_diagnostic_note "${diagnostics_path}" \ |
|
1178
|
"CPU-only job; AMD GPU probes skipped." |
|
1179
|
else |
|
1180
|
if [[ "${runtime}" == "native-kubernetes" ]]; then |
|
1181
|
append_failure_diagnostic_note "${diagnostics_path}" \ |
|
1182
|
"SMI visibility may be broader than the Kubernetes device allocation." |
|
1183
|
else |
|
1184
|
append_failure_diagnostic_note "${diagnostics_path}" \ |
|
1185
|
"Outer-runner SMI visibility may be broader than the test container." |
|
1186
|
fi |
|
1187
|
collect_amd_device_nodes "${diagnostics_path}" "${probe_deadline}" |
|
1188
|
fi |
|
1189
|
if [[ "${amd_diagnostics_expected_gpu_count}" != "0" ]] \ |
|
1190
|
&& command -v amd-smi >/dev/null 2>&1; then |
|
1191
|
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \ |
|
1192
|
amd-smi version |
|
1193
|
# Bus data identifies a card within the public node without publishing its |
|
1194
|
# persistent UUID, serial number, or process list. |
|
1195
|
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \ |
|
1196
|
amd-smi static --bus --gpu all |
|
1197
|
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \ |
|
1198
|
amd-smi metric --ecc --ecc-blocks --pcie --gpu all |
|
1199
|
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \ |
|
1200
|
amd-smi static --ras --gpu all |
|
1201
|
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \ |
|
1202
|
amd-smi metric --power --temperature --usage --mem-usage --gpu all |
|
1203
|
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" \ |
|
1204
|
amd-smi xgmi --link-status --gpu all |
|
1205
|
elif [[ "${amd_diagnostics_expected_gpu_count}" != "0" ]] \ |
|
1206
|
&& command -v rocm-smi >/dev/null 2>&1; then |
|
1207
|
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" rocm-smi \ |
|
1208
|
--showbus --showreplaycount --showrasinfo --showpagesinfo |
|
1209
|
elif [[ "${amd_diagnostics_expected_gpu_count}" != "0" ]]; then |
|
1210
|
append_failure_diagnostic_note "${diagnostics_path}" \ |
|
1211
|
"Neither amd-smi nor rocm-smi is available." |
|
1212
|
fi |
|
1213
|
|
|
1214
|
if [[ "${runtime}" == "native-kubernetes" ]]; then |
|
1215
|
append_failure_diagnostic_section "${diagnostics_path}" \ |
|
1216
|
"Unauthenticated pod network reachability" |
|
1217
|
else |
|
1218
|
append_failure_diagnostic_section "${diagnostics_path}" \ |
|
1219
|
"Unauthenticated outer-runner network reachability" |
|
1220
|
fi |
|
1221
|
if command -v curl >/dev/null 2>&1; then |
|
1222
|
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" curl -q \ |
|
1223
|
--silent --location --proto '=https' --proto-redir '=https' \ |
|
1224
|
--max-redirs 3 --output /dev/null --connect-timeout 2 --max-time 4 \ |
|
1225
|
--write-out 'target=huggingface http_code=%{http_code} dns_done_s=%{time_namelookup} connect_done_s=%{time_connect} tls_done_s=%{time_appconnect} first_byte_s=%{time_starttransfer} total_s=%{time_total}\n' \ |
|
1226
|
https://huggingface.co/api/models/gpt2 |
|
1227
|
run_failure_diagnostic "${diagnostics_path}" "${probe_deadline}" curl -q \ |
|
1228
|
--silent --location --proto '=https' --proto-redir '=https' \ |
|
1229
|
--max-redirs 3 --output /dev/null --connect-timeout 2 --max-time 4 \ |
|
1230
|
--write-out 'target=github_git_smart_http http_code=%{http_code} dns_done_s=%{time_namelookup} connect_done_s=%{time_connect} tls_done_s=%{time_appconnect} first_byte_s=%{time_starttransfer} total_s=%{time_total}\n' \ |
|
1231
|
'https://github.com/vllm-project/ci-infra.git/info/refs?service=git-upload-pack' |
|
1232
|
else |
|
1233
|
append_failure_diagnostic_note "${diagnostics_path}" \ |
|
1234
|
"curl unavailable; reachability probes skipped." |
|
1235
|
fi |
|
1236
|
|
|
1237
|
if [[ "${runtime}" == "native-kubernetes" \ |
|
1238
|
&& -n "${amd_diagnostics_memory_events_path}" \ |
|
1239
|
&& -r "${amd_diagnostics_memory_events_path}" ]]; then |
|
1240
|
oom_kill_count=$( |
|
1241
|
awk '$1 == "oom_kill" { print $2; exit }' \ |
|
1242
|
"${amd_diagnostics_memory_events_path}" 2>/dev/null |
|
1243
|
) |
|
1244
|
if [[ ! "${oom_kill_count}" =~ ^[0-9]+$ ]]; then |
|
1245
|
oom_kill_count="" |
|
1246
|
fi |
|
1247
|
fi |
|
1248
|
|
|
1249
|
case "${identity_label}" in |
|
1250
|
pod) |
|
1251
|
summary_identity_label="Kubernetes pod" |
|
1252
|
;; |
|
1253
|
container_hostname) |
|
1254
|
summary_identity_label="Container hostname" |
|
1255
|
;; |
|
1256
|
esac |
|
1257
|
|
|
1258
|
summary_rows=( |
|
1259
|
"Field" "Value" |
|
1260
|
"----------------------" "-----" |
|
1261
|
"Exit code" "${exit_code}" |
|
1262
|
"Signal" "${exit_signal:-none}" |
|
1263
|
"Runtime" "${runtime}" |
|
1264
|
"${summary_identity_label}" "${identity_value}" |
|
1265
|
) |
|
1266
|
if [[ -n "${amd_diagnostics_k8s_node_name}" ]]; then |
|
1267
|
summary_rows+=("Kubernetes node" "${amd_diagnostics_k8s_node_name}") |
|
1268
|
fi |
|
1269
|
if [[ -n "${oom_kill_count}" ]]; then |
|
1270
|
summary_rows+=("Cgroup OOM kills" "${oom_kill_count}") |
|
1271
|
fi |
|
1272
|
diagnostics_artifact_summary="${diagnostics_relative_path}" |
|
1273
|
if [[ -n "${diagnostics_fallback_root}" ]]; then |
|
1274
|
if upload_failure_diagnostics_artifact \ |
|
1275
|
"${diagnostics_fallback_root}" "${diagnostics_relative_path}"; then |
|
1276
|
echo "Uploaded AMD CI diagnostics artifact from temporary storage: ${diagnostics_relative_path}" |
|
1277
|
diagnostics_artifact_summary="${diagnostics_relative_path} (direct upload)" |
|
1278
|
else |
|
1279
|
echo "WARNING: failed to upload temporary AMD CI diagnostics artifact: ${diagnostics_relative_path}" |
|
1280
|
echo "--- AMD CI diagnostics (artifact upload failed)" |
|
1281
|
cat "${diagnostics_path}" || true |
|
1282
|
diagnostics_artifact_summary="job log only: direct upload failed" |
|
1283
|
fi |
|
1284
|
rm -rf -- "${diagnostics_fallback_root}" || \ |
|
1285
|
echo "WARNING: unable to remove temporary AMD CI diagnostics storage." |
|
1286
|
fi |
|
1287
|
summary_rows+=("Diagnostics artifact" "${diagnostics_artifact_summary}") |
|
1288
|
|
|
1289
|
printf '\nAMD CI failure summary\n' |
|
1290
|
printf '%-22s | %s\n' "${summary_rows[@]}" |
|
1291
|
|
|
1292
|
return 0 |
|
1293
|
} |
|
1294
|
|
|
1295
|
handle_pytest_exit() { |
|
1296
|
local exit_code=$1 |
|
1297
|
if [ "$exit_code" -eq 5 ]; then |
|
1298
|
echo "Pytest exit code 5 (no tests collected) - treating as success." |
|
1299
|
exit 0 |
|
1300
|
fi |
|
1301
|
exit "$exit_code" |
|
1302
|
} |
|
1303
|
|
|
1304
|
############################################################################### |
|
1305
|
# Pytest marker/keyword re-quoting |
|
1306
|
# |
|
1307
|
# When commands are passed through Buildkite -> shell -> $* -> bash -c, |
|
1308
|
# quotes around multi-word pytest -m/-k expressions get stripped: |
|
1309
|
# pytest -v -s -m 'not cpu_test' v1/core |
|
1310
|
# becomes: |
|
1311
|
# pytest -v -s -m not cpu_test v1/core |
|
1312
|
# |
|
1313
|
# pytest then interprets "cpu_test" as a file path, not part of the marker. |
|
1314
|
# |
|
1315
|
# This function detects unquoted expressions after -m/-k and re-quotes them |
|
1316
|
# by collecting tokens until a recognizable boundary is reached: |
|
1317
|
# - test path (contains '/') |
|
1318
|
# - test file (ends with '.py') |
|
1319
|
# - another pytest flag (--xxx or -x single-char flags) |
|
1320
|
# - command separator (&& || ; |) |
|
1321
|
# - environment variable assignment (FOO=bar) |
|
1322
|
# |
|
1323
|
# Single-word markers (e.g. -m cpu_test, -m hybrid_model) pass through |
|
1324
|
# unquoted since they have no spaces and work fine. |
|
1325
|
# |
|
1326
|
# Already-quoted expressions (containing literal single quotes) are passed |
|
1327
|
# through untouched to avoid double-quoting well-formed shell fragments. |
|
1328
|
# |
|
1329
|
# NOTE: This ONLY fixes -m/-k flags. It cannot recover arbitrary inner |
|
1330
|
# double-quotes stripped by the calling shell (see header comment). |
|
1331
|
# Use VLLM_TEST_COMMANDS to avoid the problem entirely. |
|
1332
|
############################################################################### |
|
1333
|
re_quote_pytest_markers() { |
|
1334
|
local input="$1" |
|
1335
|
local output="" |
|
1336
|
local collecting=false |
|
1337
|
local marker_buf="" |
|
1338
|
|
|
1339
|
# Strip backslash-newline continuations, then flatten remaining newlines |
|
1340
|
local flat="${input//$'\\\n'/ }" |
|
1341
|
flat="${flat//$'\n'/ }" |
|
1342
|
|
|
1343
|
# Disable globbing to prevent *.py etc. from expanding during read -ra |
|
1344
|
local restore_glob |
|
1345
|
restore_glob="$(shopt -p -o noglob 2>/dev/null || true)" |
|
1346
|
set -o noglob |
|
1347
|
local -a words |
|
1348
|
read -ra words <<< "$flat" |
|
1349
|
eval "$restore_glob" |
|
1350
|
|
|
1351
|
for word in "${words[@]}"; do |
|
1352
|
if $collecting; then |
|
1353
|
# If the token we're about to collect already contains a literal |
|
1354
|
# single quote, the expression was already quoted upstream. |
|
1355
|
# Flush and stop collecting. |
|
1356
|
if [[ "$word" == *"'"* ]]; then |
|
1357
|
if [[ -n "$marker_buf" ]]; then |
|
1358
|
# Should not normally happen (partial buf + quote), flush raw |
|
1359
|
output+="${marker_buf} " |
|
1360
|
marker_buf="" |
|
1361
|
fi |
|
1362
|
output+="${word} " |
|
1363
|
collecting=false |
|
1364
|
continue |
|
1365
|
fi |
|
1366
|
|
|
1367
|
local is_boundary=false |
|
1368
|
case "$word" in |
|
1369
|
# Line-continuation artifact |
|
1370
|
"\\") |
|
1371
|
is_boundary=true ;; |
|
1372
|
# Command separators |
|
1373
|
"&&"|"||"|";"|"|") |
|
1374
|
is_boundary=true ;; |
|
1375
|
# Long flags (--ignore, --shard-id, etc.) |
|
1376
|
--*) |
|
1377
|
is_boundary=true ;; |
|
1378
|
# Short flags (-v, -s, -x, etc.) but NOT negative marker tokens |
|
1379
|
# like "not" which don't start with "-". Also skip -k/-m which |
|
1380
|
# would start a new marker (handled below). |
|
1381
|
-[a-zA-Z]) |
|
1382
|
is_boundary=true ;; |
|
1383
|
# Test path (contains /) |
|
1384
|
*/*) |
|
1385
|
is_boundary=true ;; |
|
1386
|
# Test file (ends with .py, possibly with ::method) |
|
1387
|
*.py|*.py::*) |
|
1388
|
is_boundary=true ;; |
|
1389
|
# Environment variable assignment preceding a command (FOO=bar) |
|
1390
|
*=*) |
|
1391
|
# Only treat as boundary if it looks like VAR=value, not |
|
1392
|
# pytest filter expressions like num_gpus=2 inside markers |
|
1393
|
if [[ "$word" =~ ^[A-Z_][A-Z0-9_]*= ]]; then |
|
1394
|
is_boundary=true |
|
1395
|
fi |
|
1396
|
;; |
|
1397
|
esac |
|
1398
|
|
|
1399
|
if $is_boundary; then |
|
1400
|
# Strip surrounding double quotes if present (from upstream |
|
1401
|
# single-to-double conversion); without this, wrapping below |
|
1402
|
# would produce '"expr"' with literal double-quote characters. |
|
1403
|
if [[ "$marker_buf" == '"'*'"' ]]; then |
|
1404
|
marker_buf="${marker_buf#\"}" |
|
1405
|
marker_buf="${marker_buf%\"}" |
|
1406
|
fi |
|
1407
|
# Flush the collected marker expression |
|
1408
|
if [[ "$marker_buf" == *" "* || "$marker_buf" == *"("* ]]; then |
|
1409
|
output+="'${marker_buf}' " |
|
1410
|
else |
|
1411
|
output+="${marker_buf} " |
|
1412
|
fi |
|
1413
|
collecting=false |
|
1414
|
marker_buf="" |
|
1415
|
# Check if this boundary word itself starts a new -m/-k |
|
1416
|
if [[ "$word" == "-m" || "$word" == "-k" ]]; then |
|
1417
|
output+="${word} " |
|
1418
|
collecting=true |
|
1419
|
# Drop stray backslash tokens silently |
|
1420
|
elif [[ "$word" == "\\" ]]; then |
|
1421
|
: |
|
1422
|
else |
|
1423
|
output+="${word} " |
|
1424
|
fi |
|
1425
|
else |
|
1426
|
# Accumulate into marker buffer |
|
1427
|
if [[ -n "$marker_buf" ]]; then |
|
1428
|
marker_buf+=" ${word}" |
|
1429
|
else |
|
1430
|
marker_buf="${word}" |
|
1431
|
fi |
|
1432
|
fi |
|
1433
|
elif [[ "$word" == "-m" || "$word" == "-k" ]]; then |
|
1434
|
output+="${word} " |
|
1435
|
collecting=true |
|
1436
|
marker_buf="" |
|
1437
|
else |
|
1438
|
output+="${word} " |
|
1439
|
fi |
|
1440
|
done |
|
1441
|
|
|
1442
|
# Flush any trailing marker expression (marker at end of command) |
|
1443
|
if $collecting && [[ -n "$marker_buf" ]]; then |
|
1444
|
# Strip surrounding double quotes (see mid-stream flush comment) |
|
1445
|
if [[ "$marker_buf" == '"'*'"' ]]; then |
|
1446
|
marker_buf="${marker_buf#\"}" |
|
1447
|
marker_buf="${marker_buf%\"}" |
|
1448
|
fi |
|
1449
|
if [[ "$marker_buf" == *" "* || "$marker_buf" == *"("* ]]; then |
|
1450
|
output+="'${marker_buf}'" |
|
1451
|
else |
|
1452
|
output+="${marker_buf}" |
|
1453
|
fi |
|
1454
|
fi |
|
1455
|
|
|
1456
|
echo "${output% }" |
|
1457
|
} |
|
1458
|
|
|
1459
|
# shellcheck disable=SC2317 # Called indirectly by the EXIT trap. |
|
1460
|
handle_amd_runner_exit() { |
|
1461
|
local exit_code=${1:-$?} |
|
1462
|
if [[ "${exit_code}" -ne 0 ]]; then |
|
1463
|
collect_rocm_failure_diagnostics "${exit_code}" |
|
1464
|
fi |
|
1465
|
return 0 |
|
1466
|
} |
|
1467
|
|
|
1468
|
# Catch both test failures and wrapper/setup failures. Runtime-specific cleanup |
|
1469
|
# traps below replace this trap and call the same handler after cleanup. |
|
1470
|
trap handle_amd_runner_exit EXIT |
|
1471
|
|
|
1472
|
############################################################################### |
|
1473
|
# Main |
|
1474
|
############################################################################### |
|
1475
|
|
|
1476
|
if is_native_runtime; then |
|
1477
|
echo "--- Native in-pod ROCm CI (AMD_CI_RUNTIME=${AMD_CI_RUNTIME:-unset}, NATIVE_CI=${NATIVE_CI:-unset})" |
|
1478
|
artifact_work_dir="" |
|
1479
|
|
|
1480
|
# shellcheck disable=SC2317 # Called indirectly by the EXIT trap. |
|
1481
|
cleanup_native_workspace() { |
|
1482
|
local exit_code=$? |
|
1483
|
if [[ -n "${artifact_work_dir}" ]]; then |
|
1484
|
rm -rf "${artifact_work_dir}" |
|
1485
|
fi |
|
1486
|
handle_amd_runner_exit "${exit_code}" |
|
1487
|
} |
|
1488
|
trap cleanup_native_workspace EXIT |
|
1489
|
|
|
1490
|
if [[ -n "${VLLM_TEST_COMMANDS:-}" ]]; then |
|
1491
|
commands="${VLLM_TEST_COMMANDS}" |
|
1492
|
commands_source="env" |
|
1493
|
else |
|
1494
|
commands="$*" |
|
1495
|
commands_source="argv" |
|
1496
|
if [[ -z "$commands" ]]; then |
|
1497
|
echo "Error: No test commands provided for native CI." >&2 |
|
1498
|
exit 1 |
|
1499
|
fi |
|
1500
|
fi |
|
1501
|
|
|
1502
|
if [[ "$commands_source" == "argv" ]]; then |
|
1503
|
commands=$(re_quote_pytest_markers "$commands") |
|
1504
|
fi |
|
1505
|
|
|
1506
|
if is_multi_node "$commands"; then |
|
1507
|
echo "Native CI does not support multi-node jobs yet." |
|
1508
|
exit 1 |
|
1509
|
fi |
|
1510
|
|
|
1511
|
if ! initialize_native_environment; then |
|
1512
|
echo "Failed to initialize the native test environment" |
|
1513
|
exit 1 |
|
1514
|
fi |
|
1515
|
if [[ "${BUILDKITE_AGENT_META_DATA_QUEUE:-}" == *dpx* \ |
|
1516
|
&& "${VLLM_CI_EXPECTED_GPU_COUNT:-1}" != "0" ]]; then |
|
1517
|
check_dpx_gpu_exclusivity || exit 1 |
|
1518
|
fi |
|
1519
|
if [[ "${commands}" == *python_only_compile.sh* ]]; then |
|
1520
|
# This no-GPU job validates the ROCm precompiled/editable install path, |
|
1521
|
# rather than CPU runtime platform selection. |
|
1522
|
VLLM_TARGET_DEVICE=rocm |
|
1523
|
export VLLM_TARGET_DEVICE |
|
1524
|
fi |
|
1525
|
if ! prepare_native_workspace "${commands}"; then |
|
1526
|
echo "Failed to prepare native test workspace" |
|
1527
|
exit 1 |
|
1528
|
fi |
|
1529
|
|
|
1530
|
export PYTHONPATH="${VLLM_CI_WORKSPACE:-/vllm-workspace}" |
|
1531
|
|
|
1532
|
echo "Native test commands: $commands" |
|
1533
|
run_native_preflight || exit 1 |
|
1534
|
echo "--- Test log" |
|
1535
|
# Keep AMD CI orchestration variables out of vLLM's runtime environment. |
|
1536
|
clear_ci_orchestration_env |
|
1537
|
/bin/bash -o pipefail -c "${commands}" |
|
1538
|
handle_pytest_exit "$?" |
|
1539
|
fi |
|
1540
|
|
|
1541
|
# --- GPU initialization for legacy Docker execution --- |
|
1542
|
echo "--- ROCm info" |
|
1543
|
rocminfo |
|
1544
|
|
|
1545
|
# --- Docker status --- |
|
1546
|
report_docker_usage |
|
1547
|
|
|
1548
|
# --- Pull test image --- |
|
1549
|
echo "--- Pulling container" |
|
1550
|
image_name="${VLLM_CI_FALLBACK_IMAGE:-rocm/vllm-ci:${BUILDKITE_COMMIT:-local}}" |
|
1551
|
artifact_work_dir="" |
|
1552
|
container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)" |
|
1553
|
|
|
1554
|
# shellcheck disable=SC2317 # Called indirectly by the EXIT trap. |
|
1555
|
remove_docker_container() { |
|
1556
|
local exit_code=$? |
|
1557
|
if docker container inspect "${container_name}" >/dev/null 2>&1; then |
|
1558
|
docker rm -f "${container_name}" || true |
|
1559
|
fi |
|
1560
|
if [[ "${VLLM_CI_REMOVE_TEST_IMAGE:-0}" == "1" ]]; then |
|
1561
|
docker image rm -f "${image_name}" || true |
|
1562
|
else |
|
1563
|
# Keep images by default so later jobs on the same AMD node can reuse layers. |
|
1564
|
echo "Keeping ROCm test image locally: ${image_name}" |
|
1565
|
fi |
|
1566
|
if [[ -n "${artifact_work_dir}" ]]; then |
|
1567
|
rm -rf "${artifact_work_dir}" |
|
1568
|
fi |
|
1569
|
handle_amd_runner_exit "${exit_code}" |
|
1570
|
} |
|
1571
|
trap remove_docker_container EXIT |
|
1572
|
|
|
1573
|
# python_only_compile.sh runs `python setup.py develop` and needs the full repo tree |
|
1574
|
# under /vllm-workspace (Dockerfile.rocm test stage: mkdir src && mv vllm). |
|
1575
|
# The ROCm wheel artifact tarball only ships a thin tree (tests, etc.), so |
|
1576
|
# artifact images cannot satisfy that test — use the full rocm/vllm-ci image. |
|
1577
|
_cmd_probe="${VLLM_TEST_COMMANDS:-}" |
|
1578
|
if [[ -z "${_cmd_probe}" ]]; then |
|
1579
|
_cmd_probe="$*" |
|
1580
|
fi |
|
1581
|
if [[ "${VLLM_CI_USE_ARTIFACTS:-0}" == "1" && "${_cmd_probe}" == *python_only_compile.sh* ]]; then |
|
1582
|
echo "INFO: disabling VLLM_CI_USE_ARTIFACTS for python_only_compile (requires full /vllm-workspace tree)" |
|
1583
|
export VLLM_CI_USE_ARTIFACTS=0 |
|
1584
|
fi |
|
1585
|
unset -v _cmd_probe |
|
1586
|
|
|
1587
|
if ! prepare_artifact_image; then |
|
1588
|
echo "Using full ROCm CI image: ${image_name}" |
|
1589
|
docker pull "${image_name}" || exit 1 |
|
1590
|
fi |
|
1591
|
|
|
1592
|
# --- Prepare commands --- |
|
1593
|
echo "--- Running container" |
|
1594
|
|
|
1595
|
HF_CACHE="$(realpath ~)/huggingface" |
|
1596
|
mkdir -p "${HF_CACHE}" |
|
1597
|
HF_MOUNT="/root/.cache/huggingface" |
|
1598
|
|
|
1599
|
# Hugging Face Hub defaults to 10s request/download timeouts, while the ROCm |
|
1600
|
# CI image currently raises downloads to 60s. AMD model-test jobs routinely |
|
1601
|
# start from a cold or partially-populated shared cache, and the 60s read cap |
|
1602
|
# has still timed out before pytest reached the vLLM behavior under test. |
|
1603
|
# Keep the CI default explicit and overridable from the Buildkite environment. |
|
1604
|
: "${HF_HUB_DOWNLOAD_TIMEOUT:=300}" |
|
1605
|
: "${HF_HUB_ETAG_TIMEOUT:=60}" |
|
1606
|
|
|
1607
|
# ---- Command source selection ---- |
|
1608
|
# Prefer VLLM_TEST_COMMANDS (preserves all inner quoting intact). |
|
1609
|
# Fall back to $* for backward compatibility, but warn that inner |
|
1610
|
# double-quotes will have been stripped by the calling shell. |
|
1611
|
if [[ -n "${VLLM_TEST_COMMANDS:-}" ]]; then |
|
1612
|
commands="${VLLM_TEST_COMMANDS}" |
|
1613
|
commands_source="env" |
|
1614
|
echo "Commands sourced from VLLM_TEST_COMMANDS (quoting preserved)" |
|
1615
|
else |
|
1616
|
commands="$*" |
|
1617
|
commands_source="argv" |
|
1618
|
if [[ -z "$commands" ]]; then |
|
1619
|
echo "Error: No test commands provided." >&2 |
|
1620
|
echo "Usage:" >&2 |
|
1621
|
echo " Preferred: VLLM_TEST_COMMANDS='...' bash $0" >&2 |
|
1622
|
echo " Legacy: bash $0 \"commands here\"" >&2 |
|
1623
|
exit 1 |
|
1624
|
fi |
|
1625
|
echo "Commands sourced from positional args (legacy mode)" |
|
1626
|
echo "WARNING: Inner double-quotes in the command string may have been" |
|
1627
|
echo " stripped by the calling shell. If you see syntax errors, switch to:" |
|
1628
|
echo " export VLLM_TEST_COMMANDS='your commands here'" |
|
1629
|
echo " bash $0" |
|
1630
|
fi |
|
1631
|
|
|
1632
|
echo "Raw commands: $commands" |
|
1633
|
|
|
1634
|
# Only try to repair stripped pytest -m/-k quoting in legacy argv mode. |
|
1635
|
# VLLM_TEST_COMMANDS preserves inner quoting already, and re-quoting that path |
|
1636
|
# can corrupt embedded echo strings or otherwise well-formed shell fragments. |
|
1637
|
if [[ "$commands_source" == "argv" ]]; then |
|
1638
|
commands=$(re_quote_pytest_markers "$commands") |
|
1639
|
echo "After re-quoting: $commands" |
|
1640
|
else |
|
1641
|
echo "Skipping re-quoting for VLLM_TEST_COMMANDS input" |
|
1642
|
fi |
|
1643
|
|
|
1644
|
echo "Final commands: $commands" |
|
1645
|
|
|
1646
|
# Match native CPU jobs even when the container can see AMD devices. |
|
1647
|
cpu_platform_env=() |
|
1648
|
if [[ "${VLLM_CI_EXPECTED_GPU_COUNT:-1}" == "0" \ |
|
1649
|
&& "$commands" != *python_only_compile.sh* ]]; then |
|
1650
|
cpu_platform_env=(-e "VLLM_TARGET_DEVICE=cpu") |
|
1651
|
fi |
|
1652
|
|
|
1653
|
standalone_merge_base_env=() |
|
1654
|
if [[ "$commands" == *python_only_compile.sh* ]]; then |
|
1655
|
# The ROCm test image often ships /vllm-workspace without .git. Resolve the |
|
1656
|
# wheels.vllm.ai commit from the agent checkout for this test only. |
|
1657
|
vllm_standalone_merge_base="" |
|
1658
|
checkout="${BUILDKITE_BUILD_CHECKOUT_PATH:-}" |
|
1659
|
if [[ -z "${checkout}" || ! -d "${checkout}" ]]; then |
|
1660
|
checkout="." |
|
1661
|
fi |
|
1662
|
# Pass safe.directory per-command because Buildkite uses mixed user IDs. |
|
1663
|
if git -c "safe.directory=${checkout}" -C "${checkout}" rev-parse --is-inside-work-tree >/dev/null 2>&1; then |
|
1664
|
vllm_standalone_merge_base="$( |
|
1665
|
git -c "safe.directory=${checkout}" -C "${checkout}" merge-base HEAD origin/main 2>/dev/null || true |
|
1666
|
)" |
|
1667
|
fi |
|
1668
|
if [[ -z "${vllm_standalone_merge_base}" ]]; then |
|
1669
|
vllm_standalone_merge_base="${BUILDKITE_COMMIT:-}" |
|
1670
|
fi |
|
1671
|
echo "INFO: passing CI_STANDALONE_MERGE_BASE into container: ${vllm_standalone_merge_base}" |
|
1672
|
standalone_merge_base_env=(-e "CI_STANDALONE_MERGE_BASE=${vllm_standalone_merge_base}") |
|
1673
|
fi |
|
1674
|
|
|
1675
|
MYPYTHONPATH="/vllm-workspace" |
|
1676
|
|
|
1677
|
container_job_id="${BUILDKITE_JOB_ID:-${BUILDKITE_PARALLEL_JOB:-0}}" |
|
1678
|
container_job_id="${container_job_id//[^A-Za-z0-9_.-]/_}" |
|
1679
|
container_job_id_short="${container_job_id:0:8}" |
|
1680
|
CONTAINER_TMPDIR="/tmp/vllm-${container_job_id_short}" |
|
1681
|
CONTAINER_CACHE_ROOT="/tmp/vllm-buildkite-${container_job_id}/cache" |
|
1682
|
CONTAINER_PREFLIGHT="mkdir -p \"\$TMPDIR\" \"\$TIKTOKEN_RS_CACHE_DIR\" \"\$TORCHINDUCTOR_CACHE_DIR\" \"\$TRITON_CACHE_DIR\" \"\$VLLM_CACHE_ROOT\" \"\$XDG_CACHE_HOME\" && python -c \"import encodings, importlib.metadata as im, importlib.util as iu; [im.version(d) for d in ('transformers', 'torch', 'ray', 'sympy', 'markupsafe', 'vllm')]; missing=[m for m in ('torch.utils.model_zoo', 'transformers.models.nomic_bert', 'ray.dag', 'sympy.physics', 'markupsafe._speedups') if iu.find_spec(m) is None]; assert not missing, missing\"" |
|
1683
|
|
|
1684
|
# Verify GPU access |
|
1685
|
render_gid=$(getent group render | cut -d: -f3) |
|
1686
|
if [[ -z "$render_gid" ]]; then |
|
1687
|
echo "Error: 'render' group not found. This is required for GPU access." >&2 |
|
1688
|
exit 1 |
|
1689
|
fi |
|
1690
|
|
|
1691
|
# --- RDMA device passthrough (conditional) --- |
|
1692
|
# If the host has RDMA devices, pass them through so tests like |
|
1693
|
# test_moriio_connector can access ibverbs. On hosts without RDMA |
|
1694
|
# hardware the tests will gracefully skip via _rdma_available(). |
|
1695
|
RDMA_FLAGS="" |
|
1696
|
if [ -d /dev/infiniband ]; then |
|
1697
|
echo "RDMA devices detected on host, enabling passthrough" |
|
1698
|
RDMA_FLAGS="--device /dev/infiniband --cap-add=IPC_LOCK" |
|
1699
|
else |
|
1700
|
echo "No RDMA devices found on host, RDMA tests will be skipped" |
|
1701
|
fi |
|
1702
|
|
|
1703
|
# --- Route: multi-node vs single-node --- |
|
1704
|
clear_ci_orchestration_env |
|
1705
|
if is_multi_node "$commands"; then |
|
1706
|
echo "--- Multi-node job detected" |
|
1707
|
DCKR_VER=$(docker --version | sed 's/Docker version \(.*\), build .*/\1/') |
|
1708
|
export DCKR_VER |
|
1709
|
|
|
1710
|
# Parse the bracket syntax: prefix ; [node0_cmds] && [node1_cmds] |
|
1711
|
# BASH_REMATCH[1] = prefix (everything before first bracket) |
|
1712
|
# BASH_REMATCH[2] = comma-separated node0 commands |
|
1713
|
# BASH_REMATCH[3] = comma-separated node1 commands |
|
1714
|
if [[ "$commands" =~ ^(.*)\[(.*)"] && ["(.*)\]$ ]]; then |
|
1715
|
prefix=${BASH_REMATCH[1]//;/} |
|
1716
|
echo "PREFIX: ${prefix}" |
|
1717
|
|
|
1718
|
export composite_command="(command rocm-smi || true)" |
|
1719
|
saved_IFS=$IFS |
|
1720
|
IFS=',' |
|
1721
|
read -ra node0 <<< "${BASH_REMATCH[2]}" |
|
1722
|
read -ra node1 <<< "${BASH_REMATCH[3]}" |
|
1723
|
IFS=$saved_IFS |
|
1724
|
|
|
1725
|
if [[ ${#node0[@]} -ne ${#node1[@]} ]]; then |
|
1726
|
echo "Warning: node0 has ${#node0[@]} commands, node1 has ${#node1[@]}. They will be paired by index." |
|
1727
|
fi |
|
1728
|
|
|
1729
|
for i in "${!node0[@]}"; do |
|
1730
|
command_node_0=${node0[i]//\"/} |
|
1731
|
command_node_1=${node1[i]//\"/} |
|
1732
|
|
|
1733
|
step_cmd="./.buildkite/scripts/run-multi-node-test.sh /vllm-workspace/tests 2 2 ${image_name} '${command_node_0}' '${command_node_1}'" |
|
1734
|
echo "COMMANDS: ${step_cmd}" |
|
1735
|
composite_command="${composite_command} && ${step_cmd}" |
|
1736
|
done |
|
1737
|
|
|
1738
|
/bin/bash -c "${composite_command}" |
|
1739
|
exit_code=$? |
|
1740
|
handle_pytest_exit "$exit_code" |
|
1741
|
else |
|
1742
|
echo "Multi-node job detected but failed to parse bracket command syntax." |
|
1743
|
echo "Expected format: prefix ; [node0_cmd1, node0_cmd2] && [node1_cmd1, node1_cmd2]" |
|
1744
|
echo "Got: $commands" |
|
1745
|
exit 111 |
|
1746
|
fi |
|
1747
|
else |
|
1748
|
echo "--- Single-node job" |
|
1749
|
echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES" |
|
1750
|
docker_run_terminal_args=(-i) |
|
1751
|
if [[ "${ROCM_DOCKER_TTY}" == "1" ]]; then |
|
1752
|
docker_run_terminal_args+=(-t) |
|
1753
|
echo "Docker interactive stdin: enabled; TTY allocation: enabled" |
|
1754
|
else |
|
1755
|
echo "Docker interactive stdin: enabled; TTY allocation: disabled" |
|
1756
|
fi |
|
1757
|
|
|
1758
|
ulimit_core_hard=$(ulimit -H -c) |
|
1759
|
if [[ "$ulimit_core_hard" == "unlimited" ]]; then |
|
1760
|
# docker run can't pass "unlimited" to --ulimit |
|
1761
|
ulimit_core_hard="-1" |
|
1762
|
fi |
|
1763
|
# Disable core dumps in the ROCm test container unless the ROCm debug agent is enabled |
|
1764
|
coredump_flags=(--ulimit "core=0:$ulimit_core_hard") |
|
1765
|
if [[ "$commands" == *"ROCm debug agent enabled"* ]]; then |
|
1766
|
# Works around https://github.com/rocm/rocm-systems/issues/6206 |
|
1767
|
coredump_flags=(-e 'HSA_COREDUMP_PATTERN="/tmp/gpucore.%p"') |
|
1768
|
else |
|
1769
|
echo "ROCm debug agent not enabled, coredumps are disabled in the test container." |
|
1770
|
fi |
|
1771
|
|
|
1772
|
exit_code=0 |
|
1773
|
# shellcheck disable=SC2086 # word splitting is intentional: both hold multiple docker flags |
|
1774
|
run_docker_with_ci_timeout docker run \ |
|
1775
|
"${docker_run_terminal_args[@]}" \ |
|
1776
|
--device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \ |
|
1777
|
$RDMA_FLAGS \ |
|
1778
|
--network=host \ |
|
1779
|
--shm-size=16gb \ |
|
1780
|
--group-add "$render_gid" \ |
|
1781
|
--rm \ |
|
1782
|
"${coredump_flags[@]}" \ |
|
1783
|
-e HF_TOKEN \ |
|
1784
|
-e "HF_HUB_DOWNLOAD_TIMEOUT=${HF_HUB_DOWNLOAD_TIMEOUT}" \ |
|
1785
|
-e "HF_HUB_ETAG_TIMEOUT=${HF_HUB_ETAG_TIMEOUT}" \ |
|
1786
|
-e AWS_ACCESS_KEY_ID \ |
|
1787
|
-e AWS_SECRET_ACCESS_KEY \ |
|
1788
|
-e BUILDKITE_PARALLEL_JOB \ |
|
1789
|
-e BUILDKITE_PARALLEL_JOB_COUNT \ |
|
1790
|
-e TERM \ |
|
1791
|
-e FORCE_COLOR \ |
|
1792
|
-e CLICOLOR_FORCE \ |
|
1793
|
-e PY_COLORS \ |
|
1794
|
-e PYTHONFAULTHANDLER \ |
|
1795
|
-e PYTEST_ADDOPTS \ |
|
1796
|
-e PYTEST_TIMEOUT \ |
|
1797
|
-v "${HF_CACHE}:${HF_MOUNT}" \ |
|
1798
|
-e "HF_HOME=${HF_MOUNT}" \ |
|
1799
|
-e "PYTHONPATH=${MYPYTHONPATH}" \ |
|
1800
|
-e "TMPDIR=${CONTAINER_TMPDIR}/tmp" \ |
|
1801
|
-e "TIKTOKEN_RS_CACHE_DIR=${HF_MOUNT}/tiktoken-rs-cache" \ |
|
1802
|
-e "TORCHINDUCTOR_CACHE_DIR=${CONTAINER_CACHE_ROOT}/torchinductor" \ |
|
1803
|
-e "TRITON_CACHE_DIR=${CONTAINER_CACHE_ROOT}/triton" \ |
|
1804
|
-e "VLLM_CACHE_ROOT=${CONTAINER_CACHE_ROOT}/vllm" \ |
|
1805
|
-e "XDG_CACHE_HOME=${CONTAINER_CACHE_ROOT}/xdg" \ |
|
1806
|
-e "PYTORCH_ROCM_ARCH=" \ |
|
1807
|
"${cpu_platform_env[@]}" \ |
|
1808
|
"${standalone_merge_base_env[@]}" \ |
|
1809
|
--name "${container_name}" \ |
|
1810
|
"${image_name}" \ |
|
1811
|
/bin/bash -c "${CONTAINER_PREFLIGHT} && ${commands}" || exit_code=$? |
|
1812
|
handle_pytest_exit "${exit_code}" |
|
1813
|
fi |