[ROCm][CI] Use the same-build wheel in Python-only CI (#49514)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Andreas Karatzas committed
Aug 16, 2026 at 22:52 UTC
71b578b9cc37c262aeeb23815e6c478112c97d8b
4 files changed
+182
-88
.buildkite/scripts/hardware_ci/run-amd-test.sh
+58
-1
@@ -261,6 +261,8 @@ validate_native_workspace() {
261
}
262
263
prepare_native_workspace() {
264
+ local test_commands="${1:-}"
265
+
266
if [[ "${VLLM_CI_USE_ARTIFACTS:-0}" != "1" ]]; then
267
echo "Native CI requires VLLM_CI_USE_ARTIFACTS=1"
268
return 1
@@ -281,6 +283,8 @@ prepare_native_workspace() {
283
local recorded_base=""
284
local recorded_commit=""
285
local recorded_wheel=""
286
+ local checkout=""
287
+ local checkout_commit=""
288
local workspace_dir="${VLLM_CI_WORKSPACE:-/vllm-workspace}"
289
local wheel_dir=""
290
local attempt=0
@@ -400,6 +404,53 @@ prepare_native_workspace() {
404
return 1
405
fi
406
407
+ # The ROCm artifact intentionally contains only the installed wheel and the
408
+ # test workspace. The Python-only compilation job also needs setup.py and the
409
+ # vllm source tree, so overlay the matching Buildkite checkout for that job.
410
+ if [[ "${test_commands}" == *python_only_compile.sh* ]]; then
411
+ checkout="${BUILDKITE_BUILD_CHECKOUT_PATH:-}"
412
+ if [[ -z "${checkout}" || ! -d "${checkout}" ]]; then
413
+ echo "Python-only native CI requires BUILDKITE_BUILD_CHECKOUT_PATH" >&2
414
+ return 1
415
+ fi
416
+ if ! git -c "safe.directory=${checkout}" -C "${checkout}" \
417
+ rev-parse --is-inside-work-tree >/dev/null 2>&1; then
418
+ echo "Buildkite checkout is not a Git worktree: ${checkout}" >&2
419
+ return 1
420
+ fi
421
+ checkout_commit=$(
422
+ git -c "safe.directory=${checkout}" -C "${checkout}" rev-parse HEAD
423
+ ) || return 1
424
+ if [[ "${checkout_commit}" != "${recorded_commit}" ]]; then
425
+ echo "Buildkite checkout ${checkout_commit} does not match ROCm artifact ${recorded_commit}" >&2
426
+ return 1
427
+ fi
428
+
429
+ # setup.py normally derives this from .git via setuptools-scm. The native
430
+ # source overlay deliberately excludes Git metadata, so preserve the exact
431
+ # version from the already installed, artifact-matched wheel.
432
+ VLLM_VERSION_OVERRIDE=$(
433
+ python3 -c 'import importlib.metadata as m; print(m.version("vllm"))'
434
+ ) || return 1
435
+ export VLLM_VERSION_OVERRIDE
436
+ VLLM_PRECOMPILED_WHEEL_LOCATION="${wheels[0]}"
437
+ export VLLM_PRECOMPILED_WHEEL_LOCATION
438
+ echo "INFO: native Python-only wheel=${VLLM_PRECOMPILED_WHEEL_LOCATION}"
439
+
440
+ echo "--- Overlaying full source checkout for Python-only compilation"
441
+ # Archive the verified commit instead of copying the worktree so dirty or
442
+ # untracked agent files cannot contaminate the artifact-matched workspace.
443
+ git -c "safe.directory=${checkout}" -C "${checkout}" \
444
+ archive --format=tar "${recorded_commit}" \
445
+ | tar --no-same-owner -C "${workspace_dir}" -xf - || return 1
446
+ for required_source in setup.py pyproject.toml vllm; do
447
+ if [[ ! -e "${workspace_dir}/${required_source}" ]]; then
448
+ echo "Full source checkout is missing ${required_source}" >&2
449
+ return 1
450
+ fi
451
+ done
452
+ fi
453
+
454
return 0
455
}
456
@@ -954,7 +1005,13 @@ if is_native_runtime; then
1005
echo "Failed to initialize the native test environment"
1006
exit 1
1007
fi
957
- if ! prepare_native_workspace; then
1008
+ if [[ "${commands}" == *python_only_compile.sh* ]]; then
1009
+ # This no-GPU job validates the ROCm precompiled/editable install path,
1010
+ # rather than CPU runtime platform selection.
1011
+ VLLM_TARGET_DEVICE=rocm
1012
+ export VLLM_TARGET_DEVICE
1013
+ fi
1014
+ if ! prepare_native_workspace "${commands}"; then
1015
echo "Failed to prepare native test workspace"
1016
exit 1
1017
fi
.buildkite/test-amd.yaml
+3
-2
@@ -411,8 +411,9 @@ steps:
411
412
- label: Python-only Installation # TBD
413
timeout_in_minutes: 180
414
- mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
415
- agent_pool: mi250_1
414
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
415
+ dind: false
416
+ agent_pool: mi300_1
417
optional: true
418
working_dir: "/vllm-workspace/tests"
419
source_file_dependencies:
.buildkite/test_areas/misc.yaml
+2
-1
@@ -275,7 +275,8 @@ steps:
275
- bash standalone_tests/python_only_compile.sh
276
mirror:
277
amd:
278
- device: mi250_1
278
+ dind: false
279
+ device: mi300_1
280
timeout_in_minutes: 55
281
soft_fail: true
282
depends_on:
tests/standalone_tests/python_only_compile.sh
+119
-84
@@ -4,37 +4,70 @@
4
5
set -e
6
7
-# ROCm CI runs this script inside `run-amd-test.sh` where /vllm-workspace often has no .git
8
-# (wheel artifact layout). The wrapper passes CI_STANDALONE_MERGE_BASE from the agent checkout.
7
merge_base_commit=""
10
-if [[ -n "${CI_STANDALONE_MERGE_BASE:-}" ]]; then
11
- merge_base_commit="${CI_STANDALONE_MERGE_BASE}"
12
-elif merge_base_commit="$(git -C /vllm-workspace merge-base HEAD origin/main 2>/dev/null)"; then
13
- :
14
-elif merge_base_commit="$(git merge-base HEAD origin/main 2>/dev/null)"; then
15
- :
16
-else
17
- echo "ERROR: need a git checkout or CI_STANDALONE_MERGE_BASE to resolve wheels.vllm.ai commit." >&2
18
- exit 1
8
+rocm_wheel=""
9
+is_rocm=0
10
+_vllm_target_lower="$(printf '%s' "${VLLM_TARGET_DEVICE:-}" | tr '[:upper:]' '[:lower:]')"
11
+if [[ "${_vllm_target_lower}" == "rocm" || -n "${ROCM_PATH:-}" || -d /opt/rocm ]] \
12
+ || command -v rocminfo >/dev/null 2>&1; then
13
+ is_rocm=1
14
fi
15
+unset -v _vllm_target_lower
16
21
-echo "INFO: current merge base commit with main: $merge_base_commit"
22
-if git show --oneline -s "$merge_base_commit" 2>/dev/null; then
23
- :
24
-else
25
- echo "INFO: git show unavailable in this environment; using SHA above for precompiled metadata."
17
+if [[ "${is_rocm}" == "1" ]]; then
18
+ # Native CI passes the verified wheel artifact explicitly. Legacy ROCm
19
+ # images carry the same-build wheel in /opt/vllm-wheels.
20
+ if [[ -n "${VLLM_PRECOMPILED_WHEEL_LOCATION:-}" ]]; then
21
+ rocm_wheel="${VLLM_PRECOMPILED_WHEEL_LOCATION}"
22
+ if [[ ! -f "${rocm_wheel}" || "$(basename "${rocm_wheel}")" != vllm-*.whl ]]; then
23
+ echo "ERROR: invalid ROCm wheel location: ${rocm_wheel}" >&2
24
+ exit 1
25
+ fi
26
+ rocm_wheel="$(realpath -- "${rocm_wheel}")"
27
+ elif [[ -d /opt/vllm-wheels ]]; then
28
+ shopt -s nullglob
29
+ rocm_wheels=(/opt/vllm-wheels/vllm-*.whl)
30
+ shopt -u nullglob
31
+ if [[ "${#rocm_wheels[@]}" -ne 1 ]]; then
32
+ echo "ERROR: expected exactly one vLLM wheel in /opt/vllm-wheels, found ${#rocm_wheels[@]}." >&2
33
+ exit 1
34
+ fi
35
+ rocm_wheel="${rocm_wheels[0]}"
36
+ fi
37
fi
38
28
-# test whether the metadata.json url is valid, retry each 3 minutes up to 5 times
29
-# this avoids cumbersome error messages & manual retries in case the precompiled wheel
30
-# for the given commit is still being built in the release pipeline
31
-_vllm_target_lower="$(printf '%s' "${VLLM_TARGET_DEVICE:-}" | tr '[:upper:]' '[:lower:]')"
32
-if [[ "${_vllm_target_lower}" == "rocm" ]] || [[ -d /opt/rocm ]] || command -v rocminfo >/dev/null 2>&1; then
33
- _rocm_env_variant="$(python3 - <<'PY'
39
+if [[ -n "${rocm_wheel}" ]]; then
40
+ echo "INFO: using same-build ROCm wheel: ${rocm_wheel}"
41
+else
42
+ # Some CI images do not include .git under /vllm-workspace. Their wrapper
43
+ # passes CI_STANDALONE_MERGE_BASE from the agent checkout.
44
+ if [[ -n "${CI_STANDALONE_MERGE_BASE:-}" ]]; then
45
+ merge_base_commit="${CI_STANDALONE_MERGE_BASE}"
46
+ elif merge_base_commit="$(git -C /vllm-workspace merge-base HEAD origin/main 2>/dev/null)"; then
47
+ :
48
+ elif merge_base_commit="$(git merge-base HEAD origin/main 2>/dev/null)"; then
49
+ :
50
+ else
51
+ echo "ERROR: need a git checkout or CI_STANDALONE_MERGE_BASE to resolve wheels.vllm.ai commit." >&2
52
+ exit 1
53
+ fi
54
+
55
+ echo "INFO: current merge base commit with main: $merge_base_commit"
56
+ if git show --oneline -s "$merge_base_commit" 2>/dev/null; then
57
+ :
58
+ else
59
+ echo "INFO: git show unavailable in this environment; using SHA above for precompiled metadata."
60
+ fi
61
+
62
+ # Test whether the metadata.json URL is valid, retry each 5 minutes up to 5 times.
63
+ # This avoids manual retries while a new main-branch wheel is still publishing.
64
+ if [[ "${is_rocm}" == "1" ]]; then
65
+ _rocm_env_variant="$(python3 - <<'PY'
66
import ctypes
67
import os
68
from pathlib import Path
69
70
+
71
def get_rocm_version() -> str | None:
72
rocm_home = os.environ.get("ROCM_HOME") or os.environ.get("ROCM_PATH") or "/opt/rocm"
73
try:
@@ -54,76 +87,77 @@ def get_rocm_version() -> str | None:
87
return None
88
return None
89
90
+
91
version = get_rocm_version()
92
if version:
93
print(f"rocm{version.replace('.', '')}", end="")
94
PY
95
)"
62
- _available_variants="$(curl -sf "https://wheels.vllm.ai/rocm/${merge_base_commit}/" \
63
- | grep -oP 'rocm\d+' | sort -u | tr '\n' ' ' || true)"
64
- if [[ -n "${VLLM_PRECOMPILED_WHEEL_VARIANT:-}" ]]; then
65
- _rocm_variant="${VLLM_PRECOMPILED_WHEEL_VARIANT}"
66
- if [[ -n "${_rocm_env_variant}" && "${_rocm_variant}" != "${_rocm_env_variant}" ]]; then
67
- echo "ERROR: VLLM_PRECOMPILED_WHEEL_VARIANT=${_rocm_variant} does not match detected environment ROCm variant ${_rocm_env_variant}" >&2
96
+ _available_variants="$(curl -sf "https://wheels.vllm.ai/rocm/${merge_base_commit}/" \
97
+ | grep -oP 'rocm\d+' | sort -u | tr '\n' ' ' || true)"
98
+ if [[ -n "${VLLM_PRECOMPILED_WHEEL_VARIANT:-}" ]]; then
99
+ _rocm_variant="${VLLM_PRECOMPILED_WHEEL_VARIANT}"
100
+ if [[ -n "${_rocm_env_variant}" && "${_rocm_variant}" != "${_rocm_env_variant}" ]]; then
101
+ echo "ERROR: VLLM_PRECOMPILED_WHEEL_VARIANT=${_rocm_variant} does not match detected environment ROCm variant ${_rocm_env_variant}" >&2
102
+ exit 1
103
+ fi
104
+ else
105
+ _rocm_variant="${_rocm_env_variant}"
106
+ fi
107
+ if [[ -z "${_rocm_variant}" ]]; then
108
+ echo "ERROR: Could not detect ROCm variant from the environment for commit ${merge_base_commit}" >&2
109
+ exit 1
110
+ fi
111
+ if [[ -z "${_available_variants}" ]] \
112
+ || [[ " ${_available_variants} " != *" ${_rocm_variant} "* ]]; then
113
+ echo "ERROR: Environment ROCm variant '${_rocm_variant}' is not published for commit ${merge_base_commit} (available:${_available_variants:-none})" >&2
114
exit 1
115
fi
116
+ meta_json_url="https://wheels.vllm.ai/rocm/${merge_base_commit}/${_rocm_variant}/vllm/metadata.json"
117
+ unset -v _rocm_env_variant _available_variants _rocm_variant
118
else
71
- _rocm_variant="${_rocm_env_variant}"
72
- fi
73
- if [[ -z "${_rocm_variant}" ]]; then
74
- echo "ERROR: Could not detect ROCm variant from the environment for commit ${merge_base_commit}" >&2
75
- exit 1
119
+ meta_json_url="https://wheels.vllm.ai/${merge_base_commit}/vllm/metadata.json"
120
fi
77
- if [[ -z "${_available_variants}" ]] \
78
- || [[ " ${_available_variants} " != *" ${_rocm_variant} "* ]]; then
79
- echo "ERROR: Environment ROCm variant '${_rocm_variant}' is not published for commit ${merge_base_commit} (available:${_available_variants:-none})" >&2
80
- exit 1
81
- fi
82
- meta_json_url="https://wheels.vllm.ai/rocm/${merge_base_commit}/${_rocm_variant}/vllm/metadata.json"
83
- unset -v _rocm_env_variant _available_variants _rocm_variant
84
-else
85
- meta_json_url="https://wheels.vllm.ai/${merge_base_commit}/vllm/metadata.json"
86
-fi
87
-unset -v _vllm_target_lower
88
-echo "INFO: will use metadata.json from ${meta_json_url}"
89
-
90
-for i in {1..5}; do
91
- echo "Checking metadata.json URL (attempt $i)..."
92
- if curl --fail "$meta_json_url" > metadata.json; then
93
- echo "INFO: metadata.json URL is valid."
94
- # check whether it is valid json by python (printed to stdout)
95
- if python3 -m json.tool metadata.json; then
96
- echo "INFO: metadata.json is valid JSON. Proceeding with the check."
97
- # check whether there is an object in the json matching:
98
- # "package_name": "vllm", and "platform_tag" matches the current architecture
99
- # see `determine_wheel_url` in setup.py for more details
100
- if python3 -c "import platform as p,json as j,sys as s; d = j.load(open('metadata.json')); \
101
- s.exit(int(not any(o.get('package_name') == 'vllm' and p.machine() in o.get('platform_tag') \
102
- for o in d)))" 2>/dev/null; then
103
- echo "INFO: metadata.json contains a pre-compiled wheel for the current architecture."
104
- break
121
+ echo "INFO: will use metadata.json from ${meta_json_url}"
122
+
123
+ for i in {1..5}; do
124
+ echo "Checking metadata.json URL (attempt $i)..."
125
+ if curl --fail "$meta_json_url" > metadata.json; then
126
+ echo "INFO: metadata.json URL is valid."
127
+ # check whether it is valid json by python (printed to stdout)
128
+ if python3 -m json.tool metadata.json; then
129
+ echo "INFO: metadata.json is valid JSON. Proceeding with the check."
130
+ # check whether there is an object in the json matching:
131
+ # "package_name": "vllm", and "platform_tag" matches the current architecture
132
+ # see `determine_wheel_url` in setup.py for more details
133
+ if python3 -c "import platform as p,json as j,sys as s; d = j.load(open('metadata.json')); \
134
+ s.exit(int(not any(o.get('package_name') == 'vllm' and p.machine() in o.get('platform_tag') \
135
+ for o in d)))" 2>/dev/null; then
136
+ echo "INFO: metadata.json contains a pre-compiled wheel for the current architecture."
137
+ break
138
+ else
139
+ echo "WARN: metadata.json does not have a pre-compiled wheel for the current architecture."
140
+ fi
141
else
106
- echo "WARN: metadata.json does not have a pre-compiled wheel for the current architecture."
142
+ echo "CRITICAL: metadata.json exists but is not valid JSON, please do report in #sig-ci channel!"
143
+ echo "INFO: metadata.json content:"
144
+ cat metadata.json
145
+ exit 1
146
fi
108
- else
109
- echo "CRITICAL: metadata.json exists but is not valid JSON, please do report in #sig-ci channel!"
110
- echo "INFO: metadata.json content:"
111
- cat metadata.json
147
+ fi
148
+ # failure handling & retry logic
149
+ if [ "$i" -eq 5 ]; then
150
+ echo "ERROR: metadata is still not available after 5 attempts."
151
+ echo "ERROR: Please check whether the precompiled wheel for commit $merge_base_commit is available."
152
+ echo " NOTE: If $merge_base_commit is a new commit on main, maybe try again after its release pipeline finishes."
153
+ echo " NOTE: If it fails, please report in #sig-ci channel."
154
exit 1
155
+ else
156
+ echo "WARNING: metadata is not available. Retrying after 5 minutes..."
157
+ sleep 300
158
fi
114
- fi
115
- # failure handling & retry logic
116
- if [ "$i" -eq 5 ]; then
117
- echo "ERROR: metadata is still not available after 5 attempts."
118
- echo "ERROR: Please check whether the precompiled wheel for commit $merge_base_commit is available."
119
- echo " NOTE: If $merge_base_commit is a new commit on main, maybe try again after its release pipeline finishes."
120
- echo " NOTE: If it fails, please report in #sig-ci channel."
121
- exit 1
122
- else
123
- echo "WARNING: metadata is not available. Retrying after 5 minutes..."
124
- sleep 300
125
- fi
126
-done
159
+ done
160
+fi
161
162
set -x
163
@@ -143,16 +177,17 @@ fi
177
apt remove --purge build-essential -y
178
apt autoremove -y
179
180
+rm -f /tmp/changed.file
181
echo 'import os; os.system("touch /tmp/changed.file")' >> vllm/__init__.py
182
183
# ROCm CI uses setuptools develop for editable installs (see Dockerfile.rocm and run-amd-test.sh).
149
-_vllm_target_lower="$(printf '%s' "${VLLM_TARGET_DEVICE:-}" | tr '[:upper:]' '[:lower:]')"
150
-if [[ "${_vllm_target_lower}" == "rocm" ]]; then
151
- VLLM_PRECOMPILED_WHEEL_COMMIT=$merge_base_commit VLLM_USE_PRECOMPILED=1 python3 setup.py develop
184
+if [[ -n "${rocm_wheel}" ]]; then
185
+ VLLM_PRECOMPILED_WHEEL_LOCATION="${rocm_wheel}" VLLM_USE_PRECOMPILED=1 python3 setup.py develop --no-deps
186
+elif [[ "${is_rocm}" == "1" ]]; then
187
+ VLLM_PRECOMPILED_WHEEL_COMMIT=$merge_base_commit VLLM_USE_PRECOMPILED=1 python3 setup.py develop --no-deps
188
else
153
- VLLM_PRECOMPILED_WHEEL_COMMIT=$merge_base_commit VLLM_USE_PRECOMPILED=1 pip3 install -vvv -e .
189
+ VLLM_PRECOMPILED_WHEEL_COMMIT=$merge_base_commit VLLM_USE_PRECOMPILED=1 pip3 install -vvv -e .
190
fi
155
-unset -v _vllm_target_lower
191
# Run the script
192
python3 -c 'import vllm'
193