[CI][ROCm] Avoid redundant image pulls during smoke validation (#54408)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
Andreas Karatzas committed
Aug 31, 2026 at 19:48 UTC
907b1a7f22509b0095f2711cd47637c6ca345325
5 files changed
+277
-31
.buildkite/scripts/ci-bake-rocm.sh
+54
-5
@@ -842,6 +842,29 @@ should_upload_wheel_artifacts() {
842
|| "${TARGET}" == *"artifact"* ]]
843
}
844
845
+should_export_rocm_smoke() {
846
+ [[ "${TARGET}" == "test-rocm-ci-with-wheel" \
847
+ || "${TARGET}" == "smoke-test-rocm-ci" ]]
848
+}
849
+
850
+verify_rocm_smoke_export() {
851
+ local marker="./build/rocm-smoke-export/vllm-smoke-ok"
852
+ local expected_smoke_id="${BUILDKITE_BUILD_ID:-local}"
853
+ local actual_smoke_id=""
854
+
855
+ should_export_rocm_smoke || return 0
856
+ if [[ ! -f "${marker}" ]]; then
857
+ echo "ROCm BuildKit smoke marker is missing: ${marker}" >&2
858
+ return 1
859
+ fi
860
+ actual_smoke_id="$(< "${marker}")"
861
+ if [[ "${actual_smoke_id}" != "${expected_smoke_id}" ]]; then
862
+ echo "ROCm BuildKit smoke marker belongs to ${actual_smoke_id}, not ${expected_smoke_id}" \
863
+ >&2
864
+ return 1
865
+ fi
866
+}
867
+
868
get_remote_image_label() {
869
local image_ref="$1"
870
local label_key="$2"
@@ -1386,8 +1409,9 @@ maybe_skip_existing_image() {
1409
echo "FORCE_BUILD=1 set; skipping existing-image check"
1410
return 0
1411
fi
1389
- if ! is_ci_base_target && should_upload_wheel_artifacts; then
1390
- echo "Artifact-producing targets always run for the current build"
1412
+ if ! is_ci_base_target \
1413
+ && { should_upload_wheel_artifacts || should_export_rocm_smoke; }; then
1414
+ echo "Local-output targets always run for the current build"
1415
return 0
1416
fi
1417
@@ -1741,7 +1765,12 @@ EOF
1765
1766
uses_rocm_csrc_cache() {
1767
case "${TARGET}" in
1744
- csrc-rocm-ci|test-rocm-ci|test-rocm-ci-with-wheel|test-rocm-ci-with-artifacts|export-wheel-rocm)
1768
+ csrc-rocm-ci \
1769
+ | test-rocm-ci \
1770
+ | test-rocm-ci-with-wheel \
1771
+ | test-rocm-ci-with-artifacts \
1772
+ | export-wheel-rocm \
1773
+ | smoke-test-rocm-ci)
1774
return 0
1775
;;
1776
*)
@@ -1752,7 +1781,12 @@ uses_rocm_csrc_cache() {
1781
1782
uses_rocm_rust_cache() {
1783
case "${TARGET}" in
1755
- rust-rocm-ci|test-rocm-ci|test-rocm-ci-with-wheel|test-rocm-ci-with-artifacts|export-wheel-rocm)
1784
+ rust-rocm-ci \
1785
+ | test-rocm-ci \
1786
+ | test-rocm-ci-with-wheel \
1787
+ | test-rocm-ci-with-artifacts \
1788
+ | export-wheel-rocm \
1789
+ | smoke-test-rocm-ci)
1790
return 0
1791
;;
1792
*)
@@ -2140,7 +2174,7 @@ write_rocm_cache_override() {
2174
# exporter unless it is requested explicitly.
2175
if ((${#rust_cache_to[@]} > 0)); then
2176
case "${TARGET}" in
2143
- test-rocm-ci|export-wheel-rocm)
2177
+ test-rocm-ci|export-wheel-rocm|smoke-test-rocm-ci)
2178
BAKE_TARGETS=("rust-rocm-ci" "${BAKE_TARGETS[@]}")
2179
;;
2180
esac
@@ -2190,6 +2224,15 @@ EOF
2224
cat <<EOF
2225
}
2226
2227
+target "smoke-test-rocm-ci" {
2228
+ cache-from = concat(
2229
+ get_cache_from_rocm(),
2230
+EOF
2231
+ write_hcl_string_list " " "${combined_content_cache_from[@]}"
2232
+ cat <<EOF
2233
+ )
2234
+}
2235
+
2236
target "export-wheel-rocm" {
2237
cache-from = concat(
2238
get_cache_from_rocm(),
@@ -2675,8 +2718,14 @@ main() {
2718
# clean prevents a failed/retried export from packaging a stale wheel.
2719
rm -rf ./wheel-export
2720
fi
2721
+ if should_export_rocm_smoke; then
2722
+ # The marker is a build output, not a cache. Never accept stale output
2723
+ # from an earlier build or retry.
2724
+ rm -rf ./build/rocm-smoke-export
2725
+ fi
2726
seed_dependency_caches_if_needed
2727
run_bake
2728
+ verify_rocm_smoke_export
2729
promote_stable_ci_base_tag
2730
publish_ci_base_handoff_ref
2731
upload_wheel_artifacts_if_present
.buildkite/scripts/rocm/smoke-test-image.sh
+66
-25
@@ -3,6 +3,53 @@
3
4
set -euo pipefail
5
6
+run_smoke_checks() {
7
+ local required_dir=""
8
+
9
+ for required_dir in \
10
+ /vllm-workspace \
11
+ /vllm-workspace/tests \
12
+ /vllm-workspace/src/vllm; do
13
+ if [[ ! -d "${required_dir}" ]]; then
14
+ echo "Missing directory: ${required_dir}" >&2
15
+ return 1
16
+ fi
17
+ done
18
+ if [[ ! -x /vllm-workspace/src/vllm/vllm-rs ]]; then
19
+ echo "Missing executable: /vllm-workspace/src/vllm/vllm-rs" >&2
20
+ return 1
21
+ fi
22
+
23
+ command -v python3
24
+ command -v uv
25
+ command -v pytest
26
+
27
+ if ! command -v amd-smi >/dev/null 2>&1 \
28
+ && ! command -v rocminfo >/dev/null 2>&1; then
29
+ echo "No ROCm CLI found in image" >&2
30
+ return 1
31
+ fi
32
+
33
+ PYTHONDONTWRITEBYTECODE=1 python3 - <<'PY'
34
+import torch
35
+import vllm
36
+
37
+print(torch.__version__)
38
+print(vllm.__version__)
39
+PY
40
+
41
+ echo "AMD image smoke OK"
42
+}
43
+
44
+if [[ "${1:-}" == "--inside" ]]; then
45
+ run_smoke_checks
46
+ exit
47
+fi
48
+if (($#)); then
49
+ echo "Usage: $0 [--inside]" >&2
50
+ exit 2
51
+fi
52
+
53
if [[ "${ROCM_CI_ARTIFACT_ONLY:-0}" == "1" ]]; then
54
base_refreshed=""
55
if command -v buildkite-agent >/dev/null 2>&1; then
@@ -14,30 +61,24 @@ if [[ "${ROCM_CI_ARTIFACT_ONLY:-0}" == "1" ]]; then
61
fi
62
fi
63
17
-image_ref="${VLLM_CI_SMOKE_IMAGE:-${IMAGE_TAG:-rocm/vllm-ci:${BUILDKITE_COMMIT:?BUILDKITE_COMMIT is required}}}"
18
-
19
-docker run --rm --network=none --entrypoint /bin/bash "${image_ref}" -ec '
20
- if [ ! -d /vllm-workspace ]; then echo Missing directory: /vllm-workspace >&2; exit 1; fi
21
- if [ ! -d /vllm-workspace/tests ]; then echo Missing directory: /vllm-workspace/tests >&2; exit 1; fi
22
- if [ ! -d /vllm-workspace/src/vllm ]; then echo Missing directory: /vllm-workspace/src/vllm >&2; exit 1; fi
23
- if [ ! -x /vllm-workspace/src/vllm/vllm-rs ]; then echo Missing executable: /vllm-workspace/src/vllm/vllm-rs >&2; exit 1; fi
24
-
25
- command -v python3
26
- command -v uv
27
- command -v pytest
28
-
29
- if ! command -v amd-smi >/dev/null 2>&1 && ! command -v rocminfo >/dev/null 2>&1; then
30
- echo No ROCm CLI found in image >&2
31
- exit 1
32
- fi
33
-
34
- python3 - <<PY
35
-import torch
36
-import vllm
64
+smoke_marker="./build/rocm-smoke-export/vllm-smoke-ok"
65
+expected_smoke_id="${BUILDKITE_BUILD_ID:-local}"
66
+if [[ -f "${smoke_marker}" \
67
+ && ( -z "${VLLM_CI_SMOKE_IMAGE:-}" \
68
+ || "${VLLM_CI_SMOKE_IMAGE}" == "${IMAGE_TAG:-}" ) ]]; then
69
+ actual_smoke_id="$(< "${smoke_marker}")"
70
+ if [[ "${actual_smoke_id}" != "${expected_smoke_id}" ]]; then
71
+ echo "ROCm smoke marker belongs to ${actual_smoke_id}, not ${expected_smoke_id}" \
72
+ >&2
73
+ exit 1
74
+ fi
75
+ echo "AMD image smoke OK (verified inside BuildKit)"
76
+ rm -f -- "${smoke_marker}"
77
+ rmdir -- "$(dirname "${smoke_marker}")" 2>/dev/null || true
78
+ exit
79
+fi
80
38
-print(torch.__version__)
39
-print(vllm.__version__)
40
-PY
81
+image_ref="${VLLM_CI_SMOKE_IMAGE:-${IMAGE_TAG:-rocm/vllm-ci:${BUILDKITE_COMMIT:?BUILDKITE_COMMIT is required}}}"
82
42
- echo AMD image smoke OK
43
-'
83
+docker run --rm -i --network=none --entrypoint /bin/bash "${image_ref}" \
84
+ -s -- --inside < "${BASH_SOURCE[0]}"
docker/Dockerfile.rocm
+11
@@ -899,6 +899,17 @@ RUN mkdir src && mv vllm src/vllm
899
# Catch GPU<->CPU syncs in execute_model/sample_tokens during tests.
900
ENV VLLM_GPU_SYNC_CHECK=error
901
902
+# Run the structural smoke test inside BuildKit, where the test image layers
903
+# are already available. Export only a build-scoped success marker.
904
+FROM test AS test_smoke
905
+ARG ROCM_SMOKE_ID
906
+RUN --network=none \
907
+ bash /vllm-workspace/.buildkite/scripts/rocm/smoke-test-image.sh --inside \
908
+ && printf '%s\n' "${ROCM_SMOKE_ID:-local}" > /vllm-smoke-ok
909
+
910
+FROM scratch AS export_test_smoke
911
+COPY --from=test_smoke /vllm-smoke-ok /
912
+
913
# -----------------------
914
# Final vLLM image
915
FROM mori_base AS final_common
docker/ci-rocm.hcl
+17
-1
@@ -281,6 +281,7 @@ target "_ci-rocm" {
281
args = {
282
ARG_PYTORCH_ROCM_ARCH = PYTORCH_ROCM_ARCH
283
CI_BASE_IMAGE = CI_BASE_IMAGE
284
+ ROCM_SMOKE_ID = BUILDKITE_BUILD_ID
285
max_jobs = CI_MAX_JOBS
286
}
287
}
@@ -297,6 +298,15 @@ target "test-rocm-ci" {
298
output = ["type=registry"]
299
}
300
301
+# Validate the test image in the shared BuildKit graph and export only the
302
+# success marker. This avoids pulling the multi-GB image into the host daemon.
303
+target "smoke-test-rocm-ci" {
304
+ inherits = ["_common-rocm", "_ci-rocm"]
305
+ target = "export_test_smoke"
306
+ cache-from = get_cache_from_rocm()
307
+ output = ["type=local,dest=./build/rocm-smoke-export"]
308
+}
309
+
310
# Cache-only target for the source-scoped ROCm native build stage.
311
# This persists the csrc-build stage in the registry cache even though the
312
# final test image only consumes it indirectly while packaging the wheel.
@@ -341,7 +351,13 @@ group "test-rocm-ci-with-artifacts" {
351
# Full test image + wheel export. Kept for fallback/debugging when a pushed,
352
# build-scoped image is useful.
353
group "test-rocm-ci-with-wheel" {
344
- targets = ["rust-rocm-ci", "csrc-rocm-ci", "test-rocm-ci", "export-wheel-rocm"]
354
+ targets = [
355
+ "rust-rocm-ci",
356
+ "csrc-rocm-ci",
357
+ "test-rocm-ci",
358
+ "smoke-test-rocm-ci",
359
+ "export-wheel-rocm",
360
+ ]
361
}
362
363
# Primary output tag for the ci_base build. In Buildkite this is a unique,
tests/tools/test_docker_build_metadata_args.py
+129
@@ -9,6 +9,7 @@ from pathlib import Path
9
REPO_ROOT = Path(__file__).resolve().parents[2]
10
HELPER = REPO_ROOT / ".buildkite" / "scripts" / "docker-build-metadata-args.sh"
11
ROCM_CI_BAKE = REPO_ROOT / ".buildkite" / "scripts" / "ci-bake-rocm.sh"
12
+ROCM_IMAGE_SMOKE = REPO_ROOT / ".buildkite" / "scripts" / "rocm" / "smoke-test-image.sh"
13
14
15
def run_helper(
@@ -198,6 +199,134 @@ def test_rocm_ci_base_metadata_inputs_cover_ci_base_files() -> None:
199
assert expected in ci_bake
200
201
202
+def test_rocm_ci_smoke_runs_in_shared_buildkit_graph() -> None:
203
+ dockerfile = (REPO_ROOT / "docker" / "Dockerfile.rocm").read_text()
204
+ ci_hcl = (REPO_ROOT / "docker" / "ci-rocm.hcl").read_text()
205
+ full_image_group = ci_hcl.split('group "test-rocm-ci-with-wheel"', maxsplit=1)[
206
+ 1
207
+ ].split("}", maxsplit=1)[0]
208
+
209
+ for expected in (
210
+ "FROM test AS test_smoke",
211
+ "smoke-test-image.sh --inside",
212
+ "FROM scratch AS export_test_smoke",
213
+ 'target "smoke-test-rocm-ci"',
214
+ 'target = "export_test_smoke"',
215
+ 'output = ["type=local,dest=./build/rocm-smoke-export"]',
216
+ ):
217
+ assert expected in dockerfile or expected in ci_hcl
218
+ assert '"smoke-test-rocm-ci"' in full_image_group
219
+ assert 'target "smoke-test-rocm-ci"' in ROCM_CI_BAKE.read_text()
220
+
221
+
222
+def prepare_rocm_smoke_test(
223
+ tmp_path: Path,
224
+ *,
225
+ marker_id: str,
226
+ build_id: str,
227
+) -> tuple[dict[str, str], Path, Path, Path]:
228
+ fake_bin = tmp_path / "bin"
229
+ fake_bin.mkdir()
230
+ docker_called = tmp_path / "docker-called"
231
+ docker = fake_bin / "docker"
232
+ docker.write_text('#!/bin/sh\ntouch "$FAKE_DOCKER_CALLED"\nexit 99\n')
233
+ docker.chmod(0o755)
234
+
235
+ marker = tmp_path / "build" / "rocm-smoke-export" / "vllm-smoke-ok"
236
+ marker.parent.mkdir(parents=True)
237
+ marker.write_text(f"{marker_id}\n")
238
+ env = os.environ.copy()
239
+ env.update(
240
+ {
241
+ "BUILDKITE_BUILD_ID": build_id,
242
+ "FAKE_DOCKER_CALLED": str(docker_called),
243
+ "PATH": f"{fake_bin}:{env['PATH']}",
244
+ }
245
+ )
246
+ env.pop("ROCM_CI_ARTIFACT_ONLY", None)
247
+ env.pop("VLLM_CI_SMOKE_IMAGE", None)
248
+ return env, marker, docker, docker_called
249
+
250
+
251
+def test_rocm_smoke_marker_avoids_host_image_pull(tmp_path: Path) -> None:
252
+ env, marker, _, docker_called = prepare_rocm_smoke_test(
253
+ tmp_path,
254
+ marker_id="build-123",
255
+ build_id="build-123",
256
+ )
257
+
258
+ result = subprocess.run(
259
+ ["bash", str(ROCM_IMAGE_SMOKE)],
260
+ check=True,
261
+ cwd=tmp_path,
262
+ env=env,
263
+ stdout=subprocess.PIPE,
264
+ text=True,
265
+ )
266
+
267
+ assert "verified inside BuildKit" in result.stdout
268
+ assert not docker_called.exists()
269
+ assert not marker.exists()
270
+
271
+
272
+def test_rocm_smoke_rejects_marker_from_another_build(tmp_path: Path) -> None:
273
+ env, marker, _, docker_called = prepare_rocm_smoke_test(
274
+ tmp_path,
275
+ marker_id="previous-build",
276
+ build_id="current-build",
277
+ )
278
+
279
+ result = subprocess.run(
280
+ ["bash", str(ROCM_IMAGE_SMOKE)],
281
+ check=False,
282
+ cwd=tmp_path,
283
+ env=env,
284
+ stderr=subprocess.PIPE,
285
+ text=True,
286
+ )
287
+
288
+ assert result.returncode == 1
289
+ assert "previous-build, not current-build" in result.stderr
290
+ assert not docker_called.exists()
291
+ assert marker.exists()
292
+
293
+
294
+def test_rocm_smoke_override_streams_current_checks_to_docker(
295
+ tmp_path: Path,
296
+) -> None:
297
+ env, marker, docker, _ = prepare_rocm_smoke_test(
298
+ tmp_path,
299
+ marker_id="build-123",
300
+ build_id="build-123",
301
+ )
302
+ docker_args = tmp_path / "docker-args"
303
+ docker_stdin = tmp_path / "docker-stdin"
304
+ docker.write_text(
305
+ '#!/bin/sh\nprintf "%s\\n" "$@" > "$FAKE_DOCKER_ARGS"\n'
306
+ 'cat > "$FAKE_DOCKER_STDIN"\n'
307
+ )
308
+ env.update(
309
+ {
310
+ "FAKE_DOCKER_ARGS": str(docker_args),
311
+ "FAKE_DOCKER_STDIN": str(docker_stdin),
312
+ "IMAGE_TAG": "rocm/vllm-ci:built",
313
+ "VLLM_CI_SMOKE_IMAGE": "rocm/vllm-ci:override",
314
+ }
315
+ )
316
+
317
+ subprocess.run(
318
+ ["bash", str(ROCM_IMAGE_SMOKE)],
319
+ check=True,
320
+ cwd=tmp_path,
321
+ env=env,
322
+ )
323
+
324
+ assert "rocm/vllm-ci:override" in docker_args.read_text().splitlines()
325
+ assert docker_args.read_text().splitlines()[-3:] == ["-s", "--", "--inside"]
326
+ assert "run_smoke_checks()" in docker_stdin.read_text()
327
+ assert marker.exists()
328
+
329
+
330
def test_rocm_git_fetch_disables_automatic_maintenance(tmp_path: Path) -> None:
331
fake_bin = tmp_path / "bin"
332
fake_bin.mkdir()