@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 7f6432cc0d

[ROCm][CI] Reuse equivalent ROCm CI images (#48646)

Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>

Andreas Karatzas committed Aug 9, 2026 at 03:25 UTC 7f6432cc0daf97c5b9a2df141641c595a766ff40
13 files changed +1530 -907
.buildkite/ci_config_rocm.yaml
+13
@@ -2,11 +2,13 @@ name: vllm_rocm_ci
2 job_dirs:
3 - ".buildkite/hardware_tests"
4 run_all_patterns:
5 + - ".dockerignore"
6 - "docker/Dockerfile.rocm"
7 - "docker/Dockerfile.rocm_base"
8 - "docker/ci-rocm.hcl"
9 - "docker/docker-bake-rocm.hcl"
10 - ".buildkite/hardware_tests/amd.yaml"
11 + - ".buildkite/ci_config_rocm.yaml"
12 - ".buildkite/scripts/ci-bake-rocm.sh"
13 - ".buildkite/scripts/rocm/"
14 - ".buildkite/scripts/hardware_ci/run-amd-test.py"
@@ -14,9 +16,20 @@ run_all_patterns:
16 - "CMakeLists.txt"
17 - "requirements/common.txt"
18 - "requirements/rocm.txt"
19 + - "requirements/build/rust.txt"
20 - "requirements/build/rocm.txt"
21 - "requirements/test/rocm.txt"
22 + - "pyproject.toml"
23 - "setup.py"
24 + - "build_rust.sh"
25 + - "rust/"
26 + - "rust-toolchain.toml"
27 + - "tools/build_rust.py"
28 + - "tools/install_protoc.sh"
29 + - "tools/install_torchcodec_rocm.sh"
30 + - "tests/vllm_test_utils/"
31 + - "vllm/envs.py"
32 + - "vllm/__init__.py"
33 - "csrc/"
34 - "cmake/"
35 run_all_exclude_patterns:
.buildkite/hardware_tests/amd.yaml
+12 -8
@@ -1,9 +1,9 @@
1 group: Hardware - AMD Build
2
3 # ROCm image flow:
4 -# 1. Refresh the long-lived ROCm base image only when Dockerfile.rocm_base changes.
5 -# 2. Build ci_base from either the stable base or the freshly refreshed base.
6 -# 3. Build the per-commit ROCm CI image and smoke-test it before GPU jobs run.
4 +# 1. Select the content-matching ROCm base, building only on a cache miss.
5 +# 2. Build ci_base from either the stable base or the newly selected base.
6 +# 3. Build the build-scoped ROCm CI image and smoke-test it before GPU jobs run.
7 steps:
8 - label: "AMD: :docker: refresh ROCm base"
9 key: refresh-rocm-base-amd
@@ -12,6 +12,8 @@ steps:
12 no_plugin: true
13 commands:
14 - bash .buildkite/scripts/rocm/refresh-base-image.sh
15 + concurrency: 1
16 + concurrency_group: "vllm/rocm/base-build/$BUILDKITE_COMMIT"
17 env:
18 DOCKER_BUILDKIT: "1"
19 BUILDKIT_PROGRESS: "tty"
@@ -38,18 +40,21 @@ steps:
40 no_plugin: true
41 commands:
42 - bash .buildkite/scripts/rocm/build-ci-base.sh
43 + concurrency: 1
44 + concurrency_group: "vllm/rocm/ci-base-build/$BUILDKITE_COMMIT"
45 env:
46 DOCKER_BUILDKIT: "1"
47 BUILDKIT_PROGRESS: "tty"
48 TERM: "xterm-256color"
49 VLLM_BAKE_FILE: "docker/docker-bake-rocm.hcl"
50 PYTORCH_ROCM_ARCH: "gfx90a;gfx942;gfx950"
47 - REMOTE_VLLM: "1"
48 - VLLM_BRANCH: "$BUILDKITE_COMMIT"
51 + REMOTE_VLLM: "0"
52 retry:
53 automatic:
54 - exit_status: 1 # Transient Docker/BuildKit failure
55 limit: 1
56 + - exit_status: 2 # Exact image found but registry handoff failed
57 + limit: 1
58 - exit_status: -1 # Agent was lost
59 limit: 1
60 - exit_status: -10 # Agent was lost
@@ -71,9 +76,8 @@ steps:
76 TERM: "xterm-256color"
77 VLLM_BAKE_FILE: "docker/docker-bake-rocm.hcl"
78 PYTORCH_ROCM_ARCH: "gfx90a;gfx942;gfx950"
74 - IMAGE_TAG: "rocm/vllm-ci:$BUILDKITE_COMMIT"
75 - REMOTE_VLLM: "1"
76 - VLLM_BRANCH: "$BUILDKITE_COMMIT"
79 + IMAGE_TAG: "rocm/vllm-ci:build-$BUILDKITE_BUILD_ID"
80 + REMOTE_VLLM: "0"
81 retry:
82 automatic:
83 - exit_status: 1 # Transient Docker/BuildKit failure
.buildkite/scripts/ci-bake-rocm.sh
+702 -482
@@ -3,7 +3,7 @@
3 #
4 # The wrapper keeps three build concerns separate:
5 # * ci_base builds are content-addressed by vllm.ci_base.content_hash.
6 -# * test image builds are commit-addressed by org.opencontainers.image.revision.
6 +# * test image contents record the commit while runtime tags stay build-scoped.
7 # * ROCm install artifacts are uploaded only for targets that export wheels.
8 #
9 # Usage:
@@ -15,11 +15,20 @@ set -euo pipefail
15
16 DEFAULT_REPO_SLUG="vllm-project/vllm"
17 DEFAULT_CI_HCL_SOURCE="docker/ci-rocm.hcl"
18 -DEFAULT_CI_BASE_CONTENT_FILES="requirements/common.txt requirements/rocm.txt requirements/test/rocm.txt docker/Dockerfile.rocm_base docker/ci-rocm.hcl docker/docker-bake-rocm.hcl tools/install_torchcodec_rocm.sh tools/install_protoc.sh rust-toolchain.toml tests/vllm_test_utils .buildkite/scripts/ci-bake-rocm.sh .buildkite/scripts/rocm/build-ci-base.sh"
18 +DEFAULT_CI_BASE_CONTENT_FILES=".dockerignore requirements/common.txt requirements/rocm.txt requirements/test/rocm.txt tools/install_torchcodec_rocm.sh tools/install_protoc.sh rust-toolchain.toml tests/vllm_test_utils"
19 DEFAULT_CI_BASE_DOCKERFILE="docker/Dockerfile.rocm"
20 -DEFAULT_CI_BASE_DOCKERFILE_STAGES="base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust-toolchain build_nixl build_rocshmem build_deepep mori_base ci_base"
21 -DEFAULT_CI_BASE_METADATA_VERSION="1"
22 -IMAGE_EXISTED_BEFORE_BUILD=0
20 +DEFAULT_CI_BASE_DOCKERFILE_STAGES="base rust_toolchain_input_0 rust-toolchain-input rust-toolchain build_nixl build_rocshmem build_deepep mori_base ci_base"
21 +DEFAULT_CI_BASE_METADATA_VERSION="3"
22 +# ROCm CI forces REMOTE_VLLM=0, so content identity covers only the selected
23 +# local-source stages rather than unreachable remote-fetch alternatives.
24 +DEFAULT_ROCM_CSRC_CONTENT_FILES=".dockerignore requirements/common.txt requirements/rocm.txt pyproject.toml setup.py CMakeLists.txt cmake csrc vllm/envs.py vllm/__init__.py tools/build_rust.py"
25 +DEFAULT_ROCM_CSRC_DOCKERFILE_STAGES="base fetch_vllm_0 fetch_vllm build_vllm_dependencies csrc-build"
26 +DEFAULT_ROCM_RUST_CONTENT_FILES=".dockerignore requirements/build/rust.txt rust/Cargo.lock rust/Cargo.toml rust/proto rust/src rust-toolchain.toml tools/build_rust.py tools/install_protoc.sh build_rust.sh"
27 +DEFAULT_ROCM_RUST_DOCKERFILE_STAGES="base rust_toolchain_input_0 rust-toolchain-input rust_input_0 rust-input rust-toolchain rust-build"
28 +# Docker's 128-character tag limit minus the longest cache prefix
29 +# ("csrc-rocm-branch-" and "rust-rocm-branch-", both 17 characters).
30 +ROCM_CACHE_BRANCH_TAG_MAX_LEN=111
31 +CI_BASE_WRITE_SCOPE=""
32
33 TARGET=""
34 CI_HCL_SOURCE="${CI_HCL_SOURCE:-}"
@@ -86,10 +95,11 @@ compose_cache_branch_tag() {
95 local suffix=""
96 local prefix=""
97 local max_prefix_len=0
98 + local max_tag_len="${ROCM_CACHE_BRANCH_TAG_MAX_LEN}"
99
100 suffix="$(cache_scope_suffix)"
101 prefix="$(clean_docker_tag "${repo_slug}")-$(clean_docker_tag "${branch}")"
92 - max_prefix_len=$((128 - ${#suffix} - 1))
102 + max_prefix_len=$((max_tag_len - ${#suffix} - 1))
103 if (( max_prefix_len < 1 )); then
104 max_prefix_len=1
105 fi
@@ -112,6 +122,57 @@ parse_repo_slug() {
122 printf '%s\n' "${repo_slug}"
123 }
124
125 +normalize_repo_slug() {
126 + local repo_slug="${1:-}"
127 +
128 + repo_slug="${repo_slug%/}"
129 + repo_slug="${repo_slug%.git}"
130 + repo_slug="${repo_slug#https://github.com/}"
131 + repo_slug="${repo_slug#http://github.com/}"
132 + repo_slug="${repo_slug#ssh://git@github.com/}"
133 + repo_slug="${repo_slug#git@github.com:}"
134 + repo_slug="${repo_slug#github.com/}"
135 + printf '%s\n' "${repo_slug}"
136 +}
137 +
138 +is_trusted_ci_cache_writer() {
139 + local actual_repo=""
140 + local trusted_repo=""
141 +
142 + [[ "${BUILDKITE:-false}" == "true" ]] || return 1
143 + [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" ]] || return 1
144 + [[ "${BUILDKITE_BRANCH:-}" == "${CI_BASE_STABLE_BRANCH:-main}" ]] || return 1
145 + actual_repo=$(normalize_repo_slug "${BUILDKITE_REPO:-}")
146 + trusted_repo=$(normalize_repo_slug \
147 + "${CI_BASE_STABLE_REPO_SLUG:-${DEFAULT_REPO_SLUG}}")
148 + [[ -n "${actual_repo}" && "${actual_repo}" == "${trusted_repo}" ]]
149 +}
150 +
151 +ci_base_write_scope() {
152 + local identity=""
153 + local source_repo="${BUILDKITE_PULL_REQUEST_REPO:-${BUILDKITE_REPO:-local}}"
154 +
155 + if is_trusted_ci_cache_writer; then
156 + return 0
157 + fi
158 + identity=$(printf '%s\n' "${source_repo}" | sha256sum | cut -c1-12)
159 + printf 'preview-%s\n' "${identity}"
160 +}
161 +
162 +configure_ci_base_write_scope() {
163 + local scope=""
164 +
165 + scope=$(ci_base_write_scope)
166 + if [[ -n "${scope}" ]]; then
167 + CI_BASE_WRITE_SCOPE=$(clean_docker_tag "${scope}")
168 + echo "Non-canonical cache writes use source scope: ${CI_BASE_WRITE_SCOPE}"
169 + else
170 + CI_BASE_WRITE_SCOPE=""
171 + echo "Trusted main build: publishing canonical ci_base refs"
172 + fi
173 + export CI_BASE_WRITE_SCOPE
174 +}
175 +
176 get_buildkite_repo_slug() {
177 parse_repo_slug "${BUILDKITE_PULL_REQUEST_REPO:-${BUILDKITE_REPO:-}}"
178 }
@@ -145,25 +206,100 @@ hash_string_short() {
206 printf '%s' "$1" | sha256sum | cut -c1-16
207 }
208
209 +list_content_files() {
210 + # Hash the checkout inputs Docker can intentionally consume, not ignored
211 + # compiler/test debris left behind on a reused worker.
212 + git ls-files -z --cached --others --exclude-standard -- "$1" \
213 + | LC_ALL=C sort -z
214 +}
215 +
216 +hash_content_file() {
217 + local file="$1"
218 +
219 + if [[ -L "${file}" ]]; then
220 + printf 'symlink:%s\ntarget:%s\n' "${file}" "$(readlink "${file}")"
221 + return
222 + fi
223 + if [[ ! -f "${file}" ]]; then
224 + printf 'missing:%s\n' "${file}"
225 + return
226 + fi
227 + printf 'file:%s\nmode:%s\n' "${file}" "$(stat -c '%a' "${file}")"
228 + sha256sum "${file}"
229 +}
230 +
231 +hash_content_directory() {
232 + if ! list_content_files "$1" | while IFS= read -r -d '' file; do
233 + hash_content_file "${file}" || exit $?
234 + done; then
235 + echo "Failed to hash content under $1" >&2
236 + return 1
237 + fi
238 +}
239 +
240 compute_content_hash() {
149 - local path
150 - local file
241 + local path=""
242
243 for path in "$@"; do
153 - if [[ -d "${path}" ]]; then
154 - while IFS= read -r -d '' file; do
155 - printf 'file:%s\n' "${file}"
156 - sha256sum "${file}"
157 - done < <(find "${path}" -type f -print0 | sort -z)
158 - elif [[ -f "${path}" ]]; then
159 - printf 'file:%s\n' "${path}"
160 - sha256sum "${path}"
244 + if [[ -L "${path}" || -f "${path}" ]]; then
245 + hash_content_file "${path}" || return $?
246 + elif [[ -d "${path}" ]]; then
247 + hash_content_directory "${path}" || return $?
248 else
249 printf 'missing:%s\n' "${path}"
250 fi
251 done | sha256sum | cut -d' ' -f1
252 }
253
254 +normalize_ci_worktree_modes() {
255 + local entry=""
256 + local metadata=""
257 + local mode=""
258 + local stage=""
259 + local path=""
260 + local index_modes_file="${SCRIPT_TMP_DIR}/git-index-modes"
261 + local -a regular_files=()
262 + local -a executable_files=()
263 +
264 + [[ "${BUILDKITE:-false}" == "true" ]] || return 0
265 + [[ "${REMOTE_VLLM:-0}" == "0" ]] || return 0
266 + if ! git ls-files --stage -z > "${index_modes_file}"; then
267 + echo "Failed to read Git modes for the CI Docker context" >&2
268 + return 1
269 + fi
270 +
271 + # Some shared workspaces expose 0644/0755 files as 0664/0775. Restore the
272 + # index modes so both the identity hash and Docker context are canonical.
273 + while IFS= read -r -d '' entry; do
274 + metadata="${entry%%$'\t'*}"
275 + mode="${metadata%% *}"
276 + stage="${metadata##* }"
277 + path="${entry#*$'\t'}"
278 + if [[ "${stage}" != "0" ]]; then
279 + echo "Unmerged Git entry cannot be used as Docker context: ${path}" >&2
280 + return 1
281 + fi
282 + case "${mode}" in
283 + 100644)
284 + regular_files+=("${path}")
285 + ;;
286 + 100755)
287 + executable_files+=("${path}")
288 + ;;
289 + 120000|160000)
290 + ;;
291 + *)
292 + echo "Unsupported Git mode ${mode} for ${path}" >&2
293 + return 1
294 + ;;
295 + esac
296 + done < "${index_modes_file}"
297 +
298 + ((${#regular_files[@]} == 0)) || chmod 0644 -- "${regular_files[@]}"
299 + ((${#executable_files[@]} == 0)) || chmod 0755 -- "${executable_files[@]}"
300 + echo "Normalized Git file modes for the CI Docker context"
301 +}
302 +
303 compose_dependency_cache_key() {
304 local prefix="$1"
305 local material="$2"
@@ -185,7 +321,7 @@ hash_dockerfile_stages() {
321 wanted[stage_list[idx]] = 1
322 }
323 }
188 - emit = 1
324 + emit = 0
325 }
326 $1 == "FROM" {
327 stage = ""
@@ -285,9 +421,10 @@ get_content_arg_names() {
421 fi | awk 'NF && !seen[$0]++'
422 }
423
288 -compute_ci_base_content_hash_once() {
424 +compute_ci_base_content_hash() {
425 local -a content_paths=()
426 local -a content_args=()
427 + local content_files_hash=""
428 local dockerfile="${CI_BASE_DOCKERFILE:-}"
429 local stages="${CI_BASE_DOCKERFILE_STAGES:-}"
430
@@ -295,9 +432,13 @@ compute_ci_base_content_hash_once() {
432 mapfile -t content_args < <(
433 get_content_arg_names "${dockerfile}" "${stages}" "${CI_BASE_CONTENT_ARGS:-}"
434 )
435 + if ! content_files_hash=$(compute_content_hash "${content_paths[@]}"); then
436 + echo "Failed to hash ci_base content files" >&2
437 + return 1
438 + fi
439
440 {
300 - printf 'content-files-hash:%s\n' "$(compute_content_hash "${content_paths[@]}")"
441 + printf 'content-files-hash:%s\n' "${content_files_hash}"
442 if [[ -n "${dockerfile}" ]]; then
443 printf 'dockerfile:%s\n' "${dockerfile}"
444 printf 'resolved-build-args:\n'
@@ -315,67 +456,52 @@ compute_ci_base_content_hash_once() {
456 } | sha256sum | cut -d' ' -f1
457 }
458
318 -compute_ci_base_content_hash() {
319 - local attempts="${CI_BASE_HASH_ATTEMPTS:-3}"
320 - local delay_secs="${CI_BASE_HASH_RETRY_DELAY:-5}"
459 +extract_dockerfile_arg_default() {
460 + local dockerfile="$1"
461 + local arg_name="$2"
462 +
463 + sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
464 + "${dockerfile}" | head -1
465 +}
466 +
467 +resolve_image_digest() {
468 + local image_ref="$1"
469 + local attempts="${ROCM_IMAGE_DIGEST_ATTEMPTS:-4}"
470 + local delay_secs="${ROCM_IMAGE_DIGEST_RETRY_DELAY:-2}"
471 local attempt=0
322 - local hash=""
323 - local failed=0
324 - local -a hashes=()
472 + local digest=""
473 + local output=""
474 + local status=0
475
326 - if [[ ! "${attempts}" =~ ^[1-9][0-9]*$ ]]; then
327 - echo "Invalid CI_BASE_HASH_ATTEMPTS: ${attempts}" >&2
328 - return 1
476 + if [[ "${image_ref}" =~ @(sha256:[0-9a-f]{64})$ ]]; then
477 + printf '%s\n' "${BASH_REMATCH[1]}"
478 + return
479 fi
330 - if [[ ! "${delay_secs}" =~ ^[0-9]+$ ]]; then
331 - echo "Invalid CI_BASE_HASH_RETRY_DELAY: ${delay_secs}" >&2
480 + if [[ ! "${attempts}" =~ ^[1-9][0-9]*$ \
481 + || ! "${delay_secs}" =~ ^[0-9]+$ ]]; then
482 + echo "Invalid image digest retry configuration" >&2
483 return 1
484 fi
485
486 for ((attempt = 1; attempt <= attempts; attempt++)); do
336 - if ! hash=$(compute_ci_base_content_hash_once); then
337 - echo "ci_base content hash calculation ${attempt}/${attempts} failed" >&2
338 - failed=1
339 - else
340 - hashes+=("${hash}")
341 - echo "ci_base content hash calculation ${attempt}/${attempts}: ${hash}" >&2
487 + status=0
488 + output=$(docker buildx imagetools inspect "${image_ref}" 2>&1) || status=$?
489 + digest=$(awk '$1 == "Digest:" { print $2; exit }' <<< "${output}")
490 + if ((status == 0)) && [[ "${digest}" =~ ^sha256:[0-9a-f]{64}$ ]]; then
491 + printf '%s\n' "${digest}"
492 + return
493 fi
343 -
494 if ((attempt < attempts)); then
495 + printf \
496 + 'Image digest lookup failed for %s (%d/%d, status %d); retrying\n' \
497 + "${image_ref}" "${attempt}" "${attempts}" "${status}" >&2
498 sleep "${delay_secs}"
499 fi
500 done
501
349 - if ((failed)) || ((${#hashes[@]} != attempts)); then
350 - echo "Could not calculate a reliable ci_base content hash" >&2
351 - return 1
352 - fi
353 -
354 - for hash in "${hashes[@]:1}"; do
355 - if [[ "${hash}" != "${hashes[0]}" ]]; then
356 - echo "ci_base content hash changed between calculations" >&2
357 - printf ' observed: %s\n' "${hashes[@]}" >&2
358 - return 1
359 - fi
360 - done
361 -
362 - printf '%s\n' "${hashes[0]}"
363 -}
364 -
365 -extract_dockerfile_arg_default() {
366 - local dockerfile="$1"
367 - local arg_name="$2"
368 -
369 - sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
370 - "${dockerfile}" | head -1
371 -}
372 -
373 -resolve_image_digest() {
374 - local image_ref="$1"
375 -
376 - docker buildx imagetools inspect "${image_ref}" 2>/dev/null \
377 - | sed -n -E 's/^Digest:[[:space:]]+//p' \
378 - | head -1
502 + printf 'Failed to resolve digest for %s (status %d)\n%s\n' \
503 + "${image_ref}" "${status}" "${output:-<no output>}" >&2
504 + return 1
505 }
506
507 resolve_dockerfile_arg_value() {
@@ -388,6 +514,9 @@ resolve_dockerfile_arg_value() {
514 ARG_PYTORCH_ROCM_ARCH)
515 env_name="PYTORCH_ROCM_ARCH"
516 ;;
517 + max_jobs)
518 + env_name="CI_MAX_JOBS"
519 + ;;
520 esac
521
522 value="${!env_name:-}"
@@ -411,18 +540,35 @@ hash_dockerfile_arg_values() {
540 for arg_name in "$@"; do
541 [[ -n "${arg_name}" ]] || continue
542 arg_value=$(resolve_dockerfile_arg_value "${dockerfile}" "${arg_name}")
414 - printf 'arg:%s=%s\n' "${arg_name}" "${arg_value:-<empty>}"
543 if [[ "${arg_name}" == "BASE_IMAGE" && -n "${arg_value}" ]]; then
416 - digest=$(resolve_image_digest "${arg_value}")
417 - if [[ -z "${digest}" ]]; then
544 + if ! digest=$(resolve_image_digest "${arg_value}"); then
545 echo "Failed to resolve digest for BASE_IMAGE=${arg_value}" >&2
546 return 1
547 fi
548 printf 'arg:%s.digest=%s\n' "${arg_name}" "${digest}"
549 + else
550 + printf 'arg:%s=%s\n' "${arg_name}" "${arg_value:-<empty>}"
551 fi
552 done
553 }
554
555 +pin_base_image() {
556 + local dockerfile="${CI_BASE_DOCKERFILE:-${DEFAULT_CI_BASE_DOCKERFILE}}"
557 + local base_image=""
558 + local digest=""
559 +
560 + base_image=$(resolve_dockerfile_arg_value "${dockerfile}" "BASE_IMAGE")
561 + [[ -n "${base_image}" ]] || return 0
562 + if ! digest=$(resolve_image_digest "${base_image}"); then
563 + echo "Error: could not resolve base image digest for ${base_image}" >&2
564 + echo "Refusing to compute content-addressed cache keys from a mutable tag." >&2
565 + return 1
566 + fi
567 + BASE_IMAGE="${base_image%@*}@${digest}"
568 + export BASE_IMAGE
569 + echo "Pinned base image for this build: ${BASE_IMAGE}"
570 +}
571 +
572 is_ci_base_target() {
573 [[ "${TARGET}" == *"ci-base-rocm"* ]]
574 }
@@ -433,11 +579,6 @@ is_commit_image_target() {
579 return 0
580 }
581
436 -image_tag_is_commit_scoped() {
437 - [[ -n "${IMAGE_TAG:-}" && -n "${BUILDKITE_COMMIT:-}" ]] || return 1
438 - [[ "${IMAGE_TAG}" == *"${BUILDKITE_COMMIT}"* ]]
439 -}
440 -
582 should_upload_wheel_artifacts() {
583 [[ "${UPLOAD_ROCM_WHEEL_ARTIFACTS:-0}" == "1" ]] && return 0
584 [[ "${TARGET}" == *"with-wheel"* \
@@ -445,168 +586,99 @@ should_upload_wheel_artifacts() {
586 || "${TARGET}" == *"artifact"* ]]
587 }
588
448 -set_buildkite_metadata() {
449 - local key="$1"
450 - local value="$2"
451 -
452 - [[ -n "${value}" ]] || return 0
453 - if command -v buildkite-agent >/dev/null 2>&1; then
454 - buildkite-agent meta-data set "${key}" "${value}" || true
455 - fi
456 -}
457 -
589 get_remote_image_label() {
590 local image_ref="$1"
591 local label_key="$2"
592 + local format="{{ index .Image.Config.Labels \"${label_key}\" }}"
593
462 - docker buildx imagetools inspect "${image_ref}" --raw 2>/dev/null \
463 - | python3 -c '
464 -import json
465 -import subprocess
466 -import sys
467 -import urllib.parse
468 -import urllib.request
469 -
470 -image_ref = sys.argv[1]
471 -label_key = sys.argv[2]
472 -
473 -
474 -def docker_hub_repo(image_name):
475 - image_name = image_name.split("@", 1)[0]
476 - last_component = image_name.rsplit("/", 1)[-1]
477 - if ":" in last_component:
478 - image_name = image_name.rsplit(":", 1)[0]
479 -
480 - parts = image_name.split("/")
481 - if len(parts) > 1 and (
482 - "." in parts[0] or ":" in parts[0] or parts[0] == "localhost"
483 - ):
484 - registry = parts[0]
485 - if registry not in {
486 - "docker.io",
487 - "index.docker.io",
488 - "registry-1.docker.io",
489 - }:
490 - return None
491 - image_name = "/".join(parts[1:])
492 - elif len(parts) == 1:
493 - image_name = f"library/{image_name}"
494 -
495 - return image_name
496 -
497 -
498 -try:
499 - data = json.load(sys.stdin)
500 - if data.get("manifests"):
501 - manifest = next(
502 - (
503 - entry
504 - for entry in data["manifests"]
505 - if entry.get("platform", {}).get("os") != "unknown"
506 - and entry.get("platform", {}).get("architecture") != "unknown"
507 - ),
508 - data["manifests"][0],
509 - )
510 - digest = manifest["digest"]
511 - result = subprocess.run(
512 - [
513 - "docker",
514 - "buildx",
515 - "imagetools",
516 - "inspect",
517 - image_ref + "@" + digest,
518 - "--raw",
519 - ],
520 - capture_output=True,
521 - text=True,
522 - check=False,
523 - )
524 - if result.returncode != 0 or not result.stdout:
525 - raise RuntimeError("digest inspect failed")
526 - data = json.loads(result.stdout)
527 -
528 - annotations = data.get("annotations", {})
529 - if label_key in annotations:
530 - print(annotations[label_key])
531 - raise SystemExit(0)
532 -
533 - config_digest = data.get("config", {}).get("digest")
534 - if not config_digest:
535 - print("")
536 - raise SystemExit(0)
537 -
538 - image_name = docker_hub_repo(image_ref)
539 - if not image_name:
540 - print("")
541 - raise SystemExit(0)
542 -
543 - token_url = (
544 - "https://auth.docker.io/token?"
545 - + urllib.parse.urlencode(
546 - {
547 - "service": "registry.docker.io",
548 - "scope": f"repository:{image_name}:pull",
549 - }
550 - )
551 - )
552 - with urllib.request.urlopen(token_url, timeout=30) as response:
553 - token = json.load(response)["token"]
554 -
555 - request = urllib.request.Request(
556 - f"https://registry-1.docker.io/v2/{image_name}/blobs/{config_digest}",
557 - headers={"Authorization": f"Bearer {token}"},
558 - )
559 - with urllib.request.urlopen(request, timeout=30) as response:
560 - config_blob = json.load(response)
561 -
562 - labels = config_blob.get("config", {}).get("Labels", {})
563 - print(labels.get(label_key, ""))
564 -except Exception:
565 - print("")
566 -' "${image_ref}" "${label_key}" 2>/dev/null || echo ""
594 + docker buildx imagetools inspect "${image_ref}" \
595 + --format "${format}" 2>/dev/null | awk 'NF { print; exit }' || true
596 }
597
569 -get_remote_image_label_with_retry() {
598 +remote_ci_base_identity_is_current_with_retry() {
599 local image_ref="$1"
571 - local label_key="$2"
572 - local attempts="${3:-6}"
573 - local delay_secs="${4:-5}"
574 - local label_value=""
575 - local attempt
600 + local attempts="${CI_BASE_LABEL_ATTEMPTS:-4}"
601 + local delay_secs="${CI_BASE_LABEL_RETRY_DELAY:-2}"
602 + local identity=""
603 + local remote_hash=""
604 + local remote_version=""
605 + local content_files_hash=""
606 + local base_digest=""
607 + local attempt=0
608 + local expected_version="${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}"
609 + local format='{{ index .Image.Config.Labels "vllm.ci_base.content_hash" }}|{{ index .Image.Config.Labels "vllm.ci_base.metadata_version" }}|{{ index .Image.Config.Labels "vllm.ci_base.content_files_hash" }}|{{ index .Image.Config.Labels "vllm.rocm.base_image_digest" }}'
610 +
611 + if [[ ! "${attempts}" =~ ^[1-9][0-9]*$ \
612 + || ! "${delay_secs}" =~ ^[0-9]+$ ]]; then
613 + echo "Invalid ci_base label retry configuration" >&2
614 + return 1
615 + fi
616
617 for ((attempt = 1; attempt <= attempts; attempt++)); do
578 - label_value=$(get_remote_image_label "${image_ref}" "${label_key}")
579 - if [[ -n "${label_value}" ]]; then
580 - printf '%s\n' "${label_value}"
618 + identity=$(docker buildx imagetools inspect "${image_ref}" \
619 + --format "${format}" 2>/dev/null || true)
620 + IFS='|' read -r \
621 + remote_hash remote_version content_files_hash base_digest <<< "${identity}"
622 + if [[ "${remote_hash}" == "${CI_BASE_CONTENT_HASH:-}" \
623 + && "${remote_version}" == "${expected_version}" \
624 + && "${content_files_hash}" =~ ^[0-9a-f]{64}$ \
625 + && "${base_digest}" =~ ^sha256:[0-9a-f]{64}$ ]]; then
626 return 0
627 fi
583 - if [[ ${attempt} -lt ${attempts} ]]; then
584 - sleep "${delay_secs}"
585 - fi
628 + ((attempt == attempts)) || sleep "${delay_secs}"
629 done
630
588 - return 0
631 + echo "ci_base identity did not match after ${attempts} attempts: ${image_ref}" >&2
632 + echo " expected hash/version: ${CI_BASE_CONTENT_HASH:-<missing>}/${expected_version}" >&2
633 + echo " observed hash/version: ${remote_hash:-<missing>}/${remote_version:-<missing>}" >&2
634 + return 1
635 }
636
591 -remote_ci_base_metadata_is_current() {
592 - local image_ref="$1"
593 - local metadata_version=""
637 +registry_ref_exists_with_retry() {
638 + local inspect_kind="$1"
639 + local image_ref="$2"
640 + local attempts="${ROCM_REGISTRY_PROBE_ATTEMPTS:-2}"
641 + local delay_secs="${ROCM_REGISTRY_PROBE_RETRY_DELAY:-1}"
642 + local output=""
643 + local status=0
644 + local attempt=0
645
595 - metadata_version=$(get_remote_image_label "${image_ref}" "vllm.ci_base.metadata_version")
596 - [[ "${metadata_version}" == "${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}" ]]
597 -}
646 + if [[ ! "${attempts}" =~ ^[1-9][0-9]*$ \
647 + || ! "${delay_secs}" =~ ^[0-9]+$ ]]; then
648 + echo "Invalid registry probe retry configuration" >&2
649 + return 1
650 + fi
651
599 -remote_ci_base_metadata_is_current_with_retry() {
600 - local image_ref="$1"
601 - local metadata_version=""
652 + for ((attempt = 1; attempt <= attempts; attempt++)); do
653 + status=0
654 + if [[ "${inspect_kind}" == "manifest" ]]; then
655 + output=$(docker manifest inspect "${image_ref}" 2>&1) || status=$?
656 + else
657 + output=$(docker buildx imagetools inspect "${image_ref}" 2>&1) \
658 + || status=$?
659 + fi
660 + if ((status == 0)); then
661 + return 0
662 + fi
663 + # A definitive registry miss is normal for a new content key. Retry
664 + # transport/rate-limit failures, but do not add latency to a real 404.
665 + if grep -Eiq \
666 + 'manifest unknown|no such manifest|name unknown|not found|does not exist' \
667 + <<< "${output}"; then
668 + return 1
669 + fi
670 + if ((attempt < attempts)); then
671 + echo "Registry probe failed for ${image_ref} (${attempt}/${attempts}); retrying" >&2
672 + sleep "${delay_secs}"
673 + fi
674 + done
675
603 - metadata_version=$(get_remote_image_label_with_retry "${image_ref}" "vllm.ci_base.metadata_version")
604 - [[ "${metadata_version}" == "${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}" ]]
676 + echo "Registry probe failed after ${attempts} attempts: ${image_ref}" >&2
677 + return 1
678 }
679
680 remote_image_exists() {
608 - local image_ref="$1"
609 - docker manifest inspect "${image_ref}" >/dev/null 2>&1
681 + registry_ref_exists_with_retry manifest "$1"
682 }
683
684 use_existing_builder() {
@@ -680,7 +752,7 @@ print_header() {
752 if is_ci_base_target; then
753 echo "Build mode: ci_base"
754 elif is_commit_image_target; then
683 - echo "Build mode: commit image"
755 + echo "Build mode: build-scoped commit image"
756 else
757 echo "Build mode: generic"
758 fi
@@ -723,13 +795,25 @@ compute_ci_base_hash_if_needed() {
795 if [[ -z "${CI_BASE_CONTENT_FILES:-}" ]]; then
796 return 0
797 fi
798 + pin_base_image
799 + if ! is_ci_base_target; then
800 + return 0
801 + fi
802 + if [[ "${REMOTE_VLLM:-0}" != "0" ]]; then
803 + echo "Error: content-addressed ci_base builds require REMOTE_VLLM=0" >&2
804 + return 1
805 + fi
806
807 CI_BASE_CONTENT_HASH=$(compute_ci_base_content_hash)
808 export CI_BASE_CONTENT_HASH
809 echo "ci_base content hash: ${CI_BASE_CONTENT_HASH:0:16}..."
810 }
811
732 -should_push_stable_ci_base_tag() {
812 +wants_stable_ci_base_tag() {
813 + if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then
814 + return 1
815 + fi
816 +
817 if [[ "${CI_BASE_PUSH_STABLE_TAG:-}" == "1" ]]; then
818 return 0
819 fi
@@ -740,6 +824,34 @@ should_push_stable_ci_base_tag() {
824 [[ "${NIGHTLY:-0}" == "1" && "${BUILDKITE_BRANCH:-}" == "${CI_BASE_STABLE_BRANCH:-main}" ]]
825 }
826
827 +trusted_ci_base_tip_matches_build() {
828 + local branch="${CI_BASE_STABLE_BRANCH:-main}"
829 + local build_commit="${BUILDKITE_COMMIT:-}"
830 + local remote_tip=""
831 +
832 + is_trusted_ci_cache_writer || return 1
833 + if [[ ! "${build_commit}" =~ ^[0-9a-fA-F]{40}$ ]]; then
834 + echo "Skipping ci_base stable tag: Buildkite commit is missing or invalid" >&2
835 + return 1
836 + fi
837 + remote_tip=$(git ls-remote --exit-code "${BUILDKITE_REPO}" \
838 + "refs/heads/${branch}" 2>/dev/null | awk 'NR == 1 { print $1 }')
839 + if [[ ! "${remote_tip}" =~ ^[0-9a-fA-F]{40}$ ]]; then
840 + echo "Skipping ci_base stable tag: could not resolve remote ${branch} tip" >&2
841 + return 1
842 + fi
843 + if [[ "${remote_tip,,}" != "${build_commit,,}" ]]; then
844 + echo "Skipping ci_base stable tag: ${branch} advanced from ${build_commit} to ${remote_tip}" >&2
845 + return 1
846 + fi
847 +}
848 +
849 +should_push_stable_ci_base_tag() {
850 + wants_stable_ci_base_tag \
851 + && is_trusted_ci_cache_writer \
852 + && trusted_ci_base_tip_matches_build
853 +}
854 +
855 ci_base_tag_with_suffix() {
856 local base_tag="$1"
857 local suffix="$2"
@@ -749,49 +861,67 @@ ci_base_tag_with_suffix() {
861
862 configure_ci_base_image_refs() {
863 local stable_tag="${CI_BASE_IMAGE_TAG:-rocm/vllm-dev:ci_base}"
864 + local metadata_version="${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}"
865 + local scope="${CI_BASE_WRITE_SCOPE:-}"
866 + local trusted_content_tag=""
867 local content_tag=""
753 - local commit_tag=""
868 + local build_tag=""
869 local primary_tag=""
870
871 + if [[ ! "${metadata_version}" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,15}$ ]]; then
872 + echo "Invalid ci_base metadata version: ${metadata_version}" >&2
873 + return 1
874 + fi
875 +
876 + if [[ "${BUILDKITE:-false}" == "true" ]] \
877 + && ! is_full_git_sha "${BUILDKITE_COMMIT:-}"; then
878 + echo "Invalid Buildkite commit for ci_base handoff: ${BUILDKITE_COMMIT:-<empty>}" >&2
879 + return 1
880 + fi
881 + if [[ "${BUILDKITE:-false}" == "true" \
882 + && -z "${BUILDKITE_BUILD_ID:-}" ]]; then
883 + echo "Buildkite build ID is required for the ci_base runtime handoff" >&2
884 + return 1
885 + fi
886 + if [[ -n "${BUILDKITE_BUILD_ID:-}" ]]; then
887 + build_tag=$(ci_base_tag_with_suffix \
888 + "${stable_tag}" "build-${BUILDKITE_BUILD_ID}")
889 + fi
890 + CI_BASE_IMAGE_TAG_BUILD_REF="${build_tag}"
891 + export CI_BASE_IMAGE_TAG_BUILD_REF
892 +
893 if [[ -z "${CI_BASE_CONTENT_HASH:-}" ]]; then
894 + if is_ci_base_target; then
895 + echo "Error: ci_base builds require a content hash" >&2
896 + return 1
897 + fi
898 CI_BASE_IMAGE="${CI_BASE_IMAGE:-${stable_tag}}"
899 export CI_BASE_IMAGE
900 return 0
901 fi
902
762 - content_tag=$(ci_base_tag_with_suffix "${stable_tag}" "${CI_BASE_CONTENT_HASH}")
763 - CI_BASE_IMAGE_TAG_CONTENT_REF="${content_tag}"
764 - if [[ -n "${BUILDKITE_COMMIT:-}" ]]; then
765 - commit_tag=$(ci_base_tag_with_suffix "${stable_tag}" "${BUILDKITE_COMMIT}")
766 - fi
767 - CI_BASE_IMAGE_TAG_COMMIT_REF="${commit_tag}"
768 -
769 - # *_REF is the logical tag recorded in metadata. *_EXTRA is only passed to
770 - # bake when that tag is not already the primary tag, avoiding duplicates.
771 - if should_push_stable_ci_base_tag; then
772 - primary_tag="${content_tag}"
773 - CI_BASE_IMAGE_TAG_STABLE="${stable_tag}"
903 + trusted_content_tag=$(ci_base_tag_with_suffix \
904 + "${stable_tag}" "v${metadata_version}-${CI_BASE_CONTENT_HASH}")
905 + if [[ -n "${scope}" ]]; then
906 + content_tag=$(ci_base_tag_with_suffix \
907 + "${stable_tag}" "v${metadata_version}-${scope}-${CI_BASE_CONTENT_HASH}")
908 else
775 - primary_tag="${commit_tag:-${content_tag}}"
776 - CI_BASE_IMAGE_TAG_STABLE=""
909 + content_tag="${trusted_content_tag}"
910 fi
911 + CI_BASE_IMAGE_TAG_CONTENT_REF="${content_tag}"
912 + CI_BASE_TRUSTED_CONTENT_REF="${trusted_content_tag}"
913 + primary_tag="${build_tag:-${content_tag}}"
914 +
915 + # Main writes canonical content refs. Other builds import those refs
916 + # read-only and publish into a source-scoped preview namespace.
917 + # A fresh CI build first publishes its unique runtime tag. Content and
918 + # stable aliases are created only after that image passes identity checks.
919 + CI_BASE_STABLE_PROMOTION_REF="${stable_tag}"
920 CI_BASE_IMAGE_TAG="${primary_tag}"
779 - if [[ "${primary_tag}" == "${content_tag}" ]]; then
780 - CI_BASE_IMAGE_TAG_CONTENT_EXTRA=""
781 - else
782 - CI_BASE_IMAGE_TAG_CONTENT_EXTRA="${content_tag}"
783 - fi
784 - if [[ -n "${commit_tag}" && "${commit_tag}" != "${primary_tag}" ]]; then
785 - CI_BASE_IMAGE_TAG_COMMIT_EXTRA="${commit_tag}"
786 - else
787 - CI_BASE_IMAGE_TAG_COMMIT_EXTRA=""
788 - fi
921 export CI_BASE_IMAGE_TAG
790 - export CI_BASE_IMAGE_TAG_COMMIT_EXTRA
791 - export CI_BASE_IMAGE_TAG_CONTENT_EXTRA
922 export CI_BASE_IMAGE_TAG_CONTENT_REF
793 - export CI_BASE_IMAGE_TAG_COMMIT_REF
794 - export CI_BASE_IMAGE_TAG_STABLE
923 + export CI_BASE_TRUSTED_CONTENT_REF
924 + export CI_BASE_STABLE_PROMOTION_REF
925
926 if is_ci_base_target; then
927 IMAGE_TAG="${primary_tag}"
@@ -800,29 +930,21 @@ configure_ci_base_image_refs() {
930 export IMAGE_TAG
931
932 echo "ci_base primary image tag: ${CI_BASE_IMAGE_TAG}"
803 - if [[ -n "${commit_tag}" ]]; then
804 - if [[ "${commit_tag}" == "${primary_tag}" ]]; then
805 - echo "ci_base commit image tag: ${commit_tag} (primary)"
806 - else
807 - echo "ci_base commit image tag: ${commit_tag}"
808 - fi
933 + if [[ -n "${build_tag}" ]]; then
934 + echo "ci_base build image tag: ${build_tag}"
935 fi
936 echo "ci_base content image tag: ${content_tag}"
811 - if [[ -n "${CI_BASE_IMAGE_TAG_STABLE}" ]]; then
812 - echo "ci_base stable alias will also be pushed: ${CI_BASE_IMAGE_TAG_STABLE}"
937 + if wants_stable_ci_base_tag && is_trusted_ci_cache_writer; then
938 + echo "ci_base stable alias is eligible for post-build promotion: ${stable_tag}"
939 else
940 echo "ci_base stable alias will not be pushed for this build"
941 echo "Set NIGHTLY=1 on ${CI_BASE_STABLE_BRANCH:-main} to refresh ${stable_tag}"
942 fi
817 - set_buildkite_metadata "rocm-ci-base-image" "${CI_BASE_IMAGE_TAG}"
818 - set_buildkite_metadata "rocm-ci-base-image-content" "${content_tag}"
819 - set_buildkite_metadata "rocm-ci-base-image-commit" "${CI_BASE_IMAGE_TAG_COMMIT_REF:-}"
820 - set_buildkite_metadata "rocm-ci-base-image-stable" "${CI_BASE_IMAGE_TAG_STABLE:-}"
943 return 0
944 fi
945
946 if [[ -z "${CI_BASE_IMAGE:-}" || "${CI_BASE_IMAGE}" == "${stable_tag}" ]]; then
825 - CI_BASE_IMAGE="${primary_tag}"
947 + CI_BASE_IMAGE="${content_tag}"
948 export CI_BASE_IMAGE
949 echo "Using ci_base image: ${CI_BASE_IMAGE}"
950 else
@@ -830,30 +952,71 @@ configure_ci_base_image_refs() {
952 fi
953 }
954
955 +publish_ci_base_handoff_ref() {
956 + local source_ref="${1:-${CI_BASE_IMAGE_TAG_BUILD_REF:-${CI_BASE_IMAGE_TAG_CONTENT_REF:-}}}"
957 + local content_ref="${CI_BASE_IMAGE_TAG_CONTENT_REF:-}"
958 + local digest=""
959 + local handoff_ref=""
960 +
961 + is_ci_base_target || return 0
962 + if [[ -z "${source_ref}" || -z "${content_ref}" ]]; then
963 + echo "Cannot publish ci_base handoff without source and content refs" >&2
964 + return 1
965 + fi
966 + if ! digest=$(resolve_image_digest "${source_ref}"); then
967 + echo "Could not resolve immutable ci_base handoff: ${source_ref}" >&2
968 + return 1
969 + fi
970 +
971 + handoff_ref="${content_ref%@*}@${digest}"
972 + if ! confirm_remote_image_push "${handoff_ref}"; then
973 + echo "Could not validate immutable ci_base handoff: ${handoff_ref}" >&2
974 + return 1
975 + fi
976 + if command -v buildkite-agent >/dev/null 2>&1; then
977 + if ! buildkite-agent meta-data set "rocm-ci-base-image" "${handoff_ref}"; then
978 + echo "Could not publish required ci_base handoff metadata" >&2
979 + return 1
980 + fi
981 + elif [[ "${BUILDKITE:-false}" == "true" ]]; then
982 + echo "buildkite-agent not found; cannot publish ci_base handoff" >&2
983 + return 1
984 + fi
985 + echo "Published immutable ci_base handoff: ${handoff_ref}"
986 +}
987 +
988 +ci_base_output_refs() {
989 + printf '%s\n' \
990 + "${CI_BASE_IMAGE_TAG_CONTENT_REF:-}" \
991 + "${CI_BASE_IMAGE_TAG_BUILD_REF:-}" \
992 + | awk 'NF && !seen[$0]++'
993 +}
994 +
995 ci_base_candidate_refs() {
996 printf '%s\n' \
835 - "${IMAGE_TAG:-}" \
836 - "${CI_BASE_IMAGE_TAG:-}" \
837 - "${CI_BASE_IMAGE_TAG_COMMIT_EXTRA:-}" \
838 - "${CI_BASE_IMAGE_TAG_CONTENT_EXTRA:-}" \
839 - "${CI_BASE_IMAGE_TAG_STABLE:-}" \
997 + "${CI_BASE_TRUSTED_CONTENT_REF:-}" \
998 + "${CI_BASE_IMAGE_TAG_CONTENT_REF:-}" \
999 + "${CI_BASE_STABLE_PROMOTION_REF:-}" \
1000 | awk 'NF && !seen[$0]++'
1001 }
1002
1003 find_matching_ci_base_ref() {
1004 local candidate=""
845 - local candidate_hash=""
1005 + local candidate_digest=""
1006 + local immutable_candidate=""
1007
1008 while IFS= read -r candidate; do
1009 [[ -n "${candidate}" ]] || continue
1010 + # Missing content refs are normal after an input change. Avoid paying
1011 + # the full digest retry budget for each definitely absent candidate.
1012 remote_image_exists "${candidate}" || continue
850 - candidate_hash=$(get_remote_image_label "${candidate}" "vllm.ci_base.content_hash")
851 - if [[ "${candidate_hash}" == "${CI_BASE_CONTENT_HASH}" ]]; then
852 - if ! remote_ci_base_metadata_is_current "${candidate}"; then
853 - echo "Found matching ci_base content hash but stale metadata: ${candidate}" >&2
854 - continue
855 - fi
856 - printf '%s\n' "${candidate}"
1013 + if ! candidate_digest=$(resolve_image_digest "${candidate}"); then
1014 + echo "Could not pin ci_base candidate: ${candidate}" >&2
1015 + continue
1016 + fi
1017 + immutable_candidate="${candidate%@*}@${candidate_digest}"
1018 + if remote_ci_base_identity_is_current_with_retry "${immutable_candidate}"; then
1019 + printf '%s\n' "${immutable_candidate}"
1020 return 0
1021 fi
1022 done < <(ci_base_candidate_refs)
@@ -863,26 +1026,100 @@ find_matching_ci_base_ref() {
1026
1027 refresh_ci_base_tags_from_ref() {
1028 local source_ref="$1"
1029 + local source_digest=""
1030 + local immutable_source=""
1031 local tag=""
867 - local tag_hash=""
1032 + local tag_digest=""
1033 +
1034 + if ! source_digest=$(resolve_image_digest "${source_ref}"); then
1035 + echo "Could not resolve selected ci_base image: ${source_ref}" >&2
1036 + return 1
1037 + fi
1038 + immutable_source="${source_ref%@*}@${source_digest}"
1039
1040 while IFS= read -r tag; do
1041 [[ -n "${tag}" ]] || continue
1042 [[ "${tag}" != "${source_ref}" ]] || continue
872 - tag_hash=$(get_remote_image_label "${tag}" "vllm.ci_base.content_hash")
873 - if [[ "${tag_hash}" == "${CI_BASE_CONTENT_HASH}" ]]; then
1043 + tag_digest=""
1044 + if remote_image_exists "${tag}"; then
1045 + tag_digest=$(resolve_image_digest "${tag}" || true)
1046 + fi
1047 + if [[ "${tag_digest}" == "${source_digest}" ]]; then
1048 echo "ci_base tag is already current: ${tag}"
1049 continue
1050 fi
877 - echo "Updating ci_base tag ${tag} -> ${source_ref}"
878 - docker buildx imagetools create -t "${tag}" "${source_ref}"
879 - done < <(ci_base_candidate_refs)
1051 + echo "Updating ci_base tag ${tag} -> ${immutable_source}"
1052 + if ! docker buildx imagetools create --prefer-index=false \
1053 + -t "${tag}" "${immutable_source}"; then
1054 + echo "Failed to update ci_base tag ${tag} from ${immutable_source}" >&2
1055 + return 1
1056 + fi
1057 + if ! tag_digest=$(resolve_image_digest "${tag}") \
1058 + || [[ "${tag_digest}" != "${source_digest}" ]]; then
1059 + echo "Updated ci_base tag does not resolve to the selected digest: ${tag}" >&2
1060 + return 1
1061 + fi
1062 + done < <(ci_base_output_refs)
1063 +}
1064 +
1065 +promote_stable_ci_base_tag() {
1066 + local source_ref="${1:-${CI_BASE_IMAGE_TAG_BUILD_REF:-${CI_BASE_IMAGE_TAG_CONTENT_REF:-}}}"
1067 + local stable_ref="${CI_BASE_STABLE_PROMOTION_REF:-}"
1068 + local digest=""
1069 + local immutable_source=""
1070 +
1071 + is_ci_base_target || return 0
1072 + wants_stable_ci_base_tag || return 0
1073 + if ! should_push_stable_ci_base_tag; then
1074 + echo "Skipping ci_base stable promotion: build is not the trusted current main tip"
1075 + return 0
1076 + fi
1077 + if [[ -z "${source_ref}" || -z "${stable_ref}" ]]; then
1078 + echo "Cannot promote ci_base stable tag without source and destination refs" >&2
1079 + return 1
1080 + fi
1081 + if ! digest=$(resolve_image_digest "${source_ref}"); then
1082 + echo "Could not pin ci_base source before stable promotion: ${source_ref}" >&2
1083 + return 1
1084 + fi
1085 + immutable_source="${source_ref%@*}@${digest}"
1086 + echo "Promoting ci_base stable tag from ${immutable_source}"
1087 + if ! docker buildx imagetools create --prefer-index=false \
1088 + -t "${stable_ref}" "${immutable_source}"; then
1089 + echo "Failed to promote ci_base stable tag" >&2
1090 + return 1
1091 + fi
1092 + if ! confirm_remote_image_push "${stable_ref}"; then
1093 + echo "Promoted ci_base stable tag did not become visible: ${stable_ref}" >&2
1094 + return 1
1095 + fi
1096 +}
1097 +
1098 +maybe_reuse_matching_ci_base_ref() {
1099 + local matching_ref=""
1100 +
1101 + matching_ref=$(find_matching_ci_base_ref) || return 1
1102 +
1103 + echo "Found existing ci_base image with matching content hash: ${matching_ref}"
1104 + if ! refresh_ci_base_tags_from_ref "${matching_ref}"; then
1105 + echo "ci_base tag refresh failed after finding an exact image; aborting" >&2
1106 + return 2
1107 + fi
1108 + if ! promote_stable_ci_base_tag "${matching_ref}"; then
1109 + echo "ci_base stable promotion failed after finding an exact image; aborting" >&2
1110 + return 2
1111 + fi
1112 + if ! publish_ci_base_handoff_ref "${matching_ref}"; then
1113 + echo "ci_base handoff failed after finding an exact image; aborting" >&2
1114 + return 2
1115 + fi
1116 + echo "Content hashes match -- ci_base is current"
1117 + return 0
1118 }
1119
1120 maybe_skip_existing_image() {
883 - local remote_hash=""
1121 local remote_revision=""
885 - local matching_ref=""
1122 + local reuse_status=0
1123
1124 if [[ -z "${IMAGE_TAG:-}" ]]; then
1125 return 0
@@ -892,60 +1129,41 @@ maybe_skip_existing_image() {
1129 echo "FORCE_BUILD=1 set; skipping existing-image check"
1130 return 0
1131 fi
1132 + if ! is_ci_base_target && should_upload_wheel_artifacts; then
1133 + echo "Artifact-producing targets always run for the current build"
1134 + return 0
1135 + fi
1136
1137 echo "--- :mag: Checking image tag"
1138 echo "Image tag: ${IMAGE_TAG}"
1139
899 - if ! remote_image_exists "${IMAGE_TAG}"; then
900 - if is_ci_base_target && [[ -n "${CI_BASE_CONTENT_HASH:-}" ]]; then
901 - matching_ref=$(find_matching_ci_base_ref || true)
902 - if [[ -n "${matching_ref}" ]]; then
903 - echo "Found existing ci_base image with matching content hash: ${matching_ref}"
904 - if ! refresh_ci_base_tags_from_ref "${matching_ref}"; then
905 - echo "ci_base tag refresh failed; rebuilding to push expected tags"
906 - return 0
907 - fi
908 - echo "Content hashes match -- ci_base is current"
1140 + if is_ci_base_target && [[ -n "${CI_BASE_CONTENT_HASH:-}" ]]; then
1141 + maybe_reuse_matching_ci_base_ref || reuse_status=$?
1142 + case "${reuse_status}" in
1143 + 0)
1144 echo "Skipping build"
1145 exit 0
911 - fi
912 - fi
1146 + ;;
1147 + 1)
1148 + echo "No current ci_base image matched the expected content hash"
1149 + echo "Proceeding with build"
1150 + return 0
1151 + ;;
1152 + *)
1153 + return "${reuse_status}"
1154 + ;;
1155 + esac
1156 + fi
1157 +
1158 + if ! remote_image_exists "${IMAGE_TAG}"; then
1159 echo "Image not found, proceeding with build"
1160 return 0
1161 fi
1162
917 - IMAGE_EXISTED_BEFORE_BUILD=1
918 -
1163 if is_ci_base_target; then
920 - if [[ -z "${CI_BASE_CONTENT_HASH:-}" ]]; then
921 - echo "ci_base image already exists and no content hash was configured"
922 - echo "Skipping build"
923 - exit 0
924 - fi
925 -
926 - remote_hash=$(get_remote_image_label "${IMAGE_TAG}" "vllm.ci_base.content_hash")
927 - if [[ -n "${remote_hash}" ]]; then
928 - echo "Remote ci_base content hash: ${remote_hash:0:16}..."
929 - if [[ "${remote_hash}" == "${CI_BASE_CONTENT_HASH}" ]]; then
930 - if ! remote_ci_base_metadata_is_current "${IMAGE_TAG}"; then
931 - echo "Content hashes match but ci_base metadata is stale; rebuilding to refresh metadata"
932 - return 0
933 - fi
934 - if ! refresh_ci_base_tags_from_ref "${IMAGE_TAG}"; then
935 - echo "ci_base tag refresh failed; rebuilding to push expected tags"
936 - return 0
937 - fi
938 - echo "Content hashes match -- ci_base is current"
939 - echo "Skipping build"
940 - exit 0
941 - fi
942 -
943 - echo "Content hashes differ -- ci_base is stale, rebuilding"
944 - return 0
945 - fi
946 -
947 - echo "Remote ci_base has no content-hash label; rebuilding to add one"
948 - return 0
1164 + echo "ci_base image already exists and no content hash was configured"
1165 + echo "Skipping build"
1166 + exit 0
1167 fi
1168
1169 if is_commit_image_target; then
@@ -957,12 +1175,6 @@ maybe_skip_existing_image() {
1175 return 0
1176 fi
1177
960 - if should_upload_wheel_artifacts; then
961 - echo "Commit image already exists: ${IMAGE_TAG}"
962 - echo "Continuing build because this target uploads per-build ROCm artifacts"
963 - return 0
964 - fi
965 -
1178 echo "Commit image already exists: ${IMAGE_TAG}"
1179 echo "Skipping build"
1180 exit 0
@@ -1031,6 +1243,18 @@ setup_builder() {
1243 docker buildx ls | grep -E '^\*|^NAME' || docker buildx ls
1244 }
1245
1246 +validate_cache_branch_tag() {
1247 + local name="$1"
1248 + local value="$2"
1249 +
1250 + if [[ -n "${value}" \
1251 + && (! "${value}" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]*$ \
1252 + || ${#value} -gt ${ROCM_CACHE_BRANCH_TAG_MAX_LEN}) ]]; then
1253 + echo "Invalid ${name}; expected a Docker tag component of at most ${ROCM_CACHE_BRANCH_TAG_MAX_LEN} characters" >&2
1254 + return 1
1255 + fi
1256 +}
1257 +
1258 prepare_git_cache_metadata() {
1259 local cache_branch_name=""
1260 local cache_base_branch="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}"
@@ -1038,6 +1262,11 @@ prepare_git_cache_metadata() {
1262 local target_repo_url=""
1263 local merge_base_ref=""
1264
1265 + if is_ci_base_target; then
1266 + echo "Skipping commit-cache ancestry lookup for content-addressed ci_base"
1267 + return 0
1268 + fi
1269 +
1270 if [[ -z "${PARENT_COMMIT:-}" || -z "${VLLM_MERGE_BASE_COMMIT:-}" ]] \
1271 && git rev-parse --is-shallow-repository 2>/dev/null | grep -q "true"; then
1272 echo "Shallow clone detected - deepening for cache key computation"
@@ -1090,6 +1319,11 @@ prepare_git_cache_metadata() {
1319 echo "Using provided ROCM_CACHE_UPSTREAM_BRANCH_TAG: ${ROCM_CACHE_UPSTREAM_BRANCH_TAG}"
1320 fi
1321
1322 + validate_cache_branch_tag \
1323 + ROCM_CACHE_BRANCH_TAG "${ROCM_CACHE_BRANCH_TAG:-}"
1324 + validate_cache_branch_tag \
1325 + ROCM_CACHE_UPSTREAM_BRANCH_TAG "${ROCM_CACHE_UPSTREAM_BRANCH_TAG:-}"
1326 +
1327 if [[ -z "${VLLM_MERGE_BASE_COMMIT:-}" ]]; then
1328 target_repo_url=$(get_buildkite_target_repo_url)
1329 merge_base_ref="refs/remotes/vllm-cache-upstream/${cache_base_branch}"
@@ -1119,13 +1353,15 @@ ci_base_metadata_pairs() {
1353 local content_files_hash=""
1354 local base_image=""
1355 local base_image_digest=""
1122 - local git_branch=""
1356 local -a content_paths=()
1357 local -a content_args=()
1358
1359 read -r -a content_paths <<< "${content_files}"
1360 if [[ ${#content_paths[@]} -gt 0 ]]; then
1128 - content_files_hash=$(compute_content_hash "${content_paths[@]}")
1361 + if ! content_files_hash=$(compute_content_hash "${content_paths[@]}"); then
1362 + echo "Failed to hash ci_base metadata content files" >&2
1363 + return 1
1364 + fi
1365 fi
1366 mapfile -t content_args < <(
1367 get_content_arg_names "${dockerfile}" "${stages}" "${CI_BASE_CONTENT_ARGS:-}"
@@ -1133,9 +1369,11 @@ ci_base_metadata_pairs() {
1369
1370 base_image=$(resolve_dockerfile_arg_value "${dockerfile}" "BASE_IMAGE")
1371 if [[ -n "${base_image}" ]]; then
1136 - base_image_digest=$(resolve_image_digest "${base_image}")
1372 + if ! base_image_digest=$(resolve_image_digest "${base_image}"); then
1373 + echo "Failed to resolve ci_base metadata digest for ${base_image}" >&2
1374 + return 1
1375 + fi
1376 fi
1138 - git_branch="${BUILDKITE_BRANCH:-${VLLM_BRANCH:-}}"
1377
1378 metadata_pair "vllm.ci_base.metadata_version" "${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}"
1379 metadata_pair "vllm.ci_base.content_hash" "${CI_BASE_CONTENT_HASH:-}"
@@ -1144,21 +1382,16 @@ ci_base_metadata_pairs() {
1382 metadata_pair "vllm.ci_base.content_args" "$(join_words "${content_args[@]}")"
1383 metadata_pair "vllm.ci_base.dockerfile" "${dockerfile}"
1384 metadata_pair "vllm.ci_base.dockerfile_stages" "${stages}"
1147 - metadata_pair "vllm.ci_base.image.primary" "${CI_BASE_IMAGE_TAG:-}"
1148 - metadata_pair "vllm.ci_base.image.content" "${CI_BASE_IMAGE_TAG_CONTENT_REF:-${CI_BASE_IMAGE_TAG_CONTENT_EXTRA:-}}"
1149 - metadata_pair "vllm.ci_base.image.commit" "${CI_BASE_IMAGE_TAG_COMMIT_REF:-${CI_BASE_IMAGE_TAG_COMMIT_EXTRA:-}}"
1150 - metadata_pair "vllm.ci_base.image.stable" "${CI_BASE_IMAGE_TAG_STABLE:-}"
1151 - metadata_pair "vllm.ci_base.git_commit" "${BUILDKITE_COMMIT:-}"
1152 - metadata_pair "vllm.ci_base.git_branch" "${git_branch}"
1153 - metadata_pair "vllm.ci_base.vllm_branch" "${VLLM_BRANCH:-}"
1154 - metadata_pair "vllm.ci_base.stable_branch" "${CI_BASE_STABLE_BRANCH:-main}"
1155 -
1156 - metadata_pair "vllm.rocm.base_image" "${base_image}"
1385 +
1386 + # The parent identity is digest-only: mutable aliases that resolve to the
1387 + # same image must produce byte-identical canonical metadata.
1388 + metadata_pair "vllm.rocm.base_image" "${base_image_digest}"
1389 metadata_pair "vllm.rocm.base_image_digest" "${base_image_digest}"
1390 metadata_pair "vllm.rocm.pytorch_rocm_arch" "${PYTORCH_ROCM_ARCH:-}"
1391 metadata_pair "vllm.rocm.nic_backend" "$(resolve_dockerfile_arg_value "${dockerfile}" "NIC_BACKEND")"
1392 metadata_pair "vllm.rocm.ainic_version" "$(resolve_dockerfile_arg_value "${dockerfile}" "AINIC_VERSION")"
1393 metadata_pair "vllm.rocm.ubuntu_codename" "$(resolve_dockerfile_arg_value "${dockerfile}" "UBUNTU_CODENAME")"
1394 + metadata_pair "vllm.rocm.max_jobs" "$(resolve_dockerfile_arg_value "${dockerfile}" "max_jobs")"
1395 metadata_pair "vllm.rocm.nixl_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "NIXL_REPO")"
1396 metadata_pair "vllm.rocm.nixl_commit" "${NIXL_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "NIXL_BRANCH")}"
1397 metadata_pair "vllm.rocm.ucx_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "UCX_REPO")"
@@ -1172,22 +1405,18 @@ ci_base_metadata_pairs() {
1405 metadata_pair "vllm.rocm.nixl_cache_key" "${NIXL_CACHE_KEY:-}"
1406 metadata_pair "vllm.rocm.rocshmem_cache_key" "${ROCSHMEM_CACHE_KEY:-}"
1407 metadata_pair "vllm.rocm.deepep_cache_key" "${DEEPEP_CACHE_KEY:-}"
1175 -
1176 - metadata_pair "vllm.buildkite.build_number" "${BUILDKITE_BUILD_NUMBER:-}"
1177 - metadata_pair "vllm.buildkite.build_id" "${BUILDKITE_BUILD_ID:-}"
1408 }
1409
1410 write_ci_base_metadata_annotations() {
1411 local metadata="$1"
1412 local key=""
1413 local value=""
1184 - local annotation=""
1414
1415 [[ -n "${metadata}" ]] || return 0
1416 while IFS=$'\t' read -r key value; do
1417 [[ -n "${key}" && -n "${value}" ]] || continue
1189 - annotation="manifest:${key}=${value}"
1190 - printf ' "%s",\n' "$(hcl_escape_string "${annotation}")"
1418 + printf ' "%s",\n' \
1419 + "$(hcl_escape_string "manifest:${key}=${value}")"
1420 done <<< "${metadata}"
1421 }
1422
@@ -1277,23 +1506,16 @@ uses_rocm_rust_cache() {
1506 compute_rocm_csrc_content_hash() {
1507 local bake_dir=""
1508 local dockerfile_rocm=""
1280 - local -a content_paths=(
1281 - "requirements/common.txt"
1282 - "requirements/rocm.txt"
1283 - "pyproject.toml"
1284 - "setup.py"
1285 - "CMakeLists.txt"
1286 - "cmake"
1287 - "csrc"
1288 - "vllm/envs.py"
1289 - "vllm/__init__.py"
1290 - )
1509 + local content_files="${ROCM_CSRC_CONTENT_FILES:-${DEFAULT_ROCM_CSRC_CONTENT_FILES}}"
1510 + local stages="${ROCM_CSRC_DOCKERFILE_STAGES:-${DEFAULT_ROCM_CSRC_DOCKERFILE_STAGES}}"
1511 + local -a content_paths=()
1512 local -a content_args=()
1513
1514 bake_dir=$(dirname "${VLLM_BAKE_FILE}")
1515 dockerfile_rocm="${bake_dir}/Dockerfile.rocm"
1516 + read -r -a content_paths <<< "${content_files}"
1517 mapfile -t content_args < <(
1296 - get_content_arg_names "${dockerfile_rocm}" "base csrc-build" "${ROCM_CSRC_CONTENT_ARGS:-}"
1518 + get_content_arg_names "${dockerfile_rocm}" "${stages}" "${ROCM_CSRC_CONTENT_ARGS:-}"
1519 )
1520
1521 {
@@ -1301,9 +1523,9 @@ compute_rocm_csrc_content_hash() {
1523 printf 'dockerfile:%s\n' "${dockerfile_rocm}"
1524 printf 'resolved-build-args:\n'
1525 hash_dockerfile_arg_values "${dockerfile_rocm}" "${content_args[@]}"
1304 - printf 'dockerfile-stages:base csrc-build\n'
1526 + printf 'dockerfile-stages:%s\n' "${stages}"
1527 if [[ -f "${dockerfile_rocm}" ]]; then
1306 - hash_dockerfile_stages "${dockerfile_rocm}" "base csrc-build"
1528 + hash_dockerfile_stages "${dockerfile_rocm}" "${stages}"
1529 else
1530 printf 'missing:%s\n' "${dockerfile_rocm}"
1531 fi
@@ -1312,14 +1534,20 @@ compute_rocm_csrc_content_hash() {
1534
1535 compute_rocm_csrc_content_hash_if_needed() {
1536 local cache_repo="${DOCKERHUB_CACHE_REPO:-rocm/vllm-ci-cache}"
1537 + local write_scope="${CI_BASE_WRITE_SCOPE:-}"
1538
1539 if [[ "${ROCM_CSRC_CONTENT_CACHE:-1}" == "0" ]] || ! uses_rocm_csrc_cache; then
1540 return 0
1541 fi
1542
1543 ROCM_CSRC_CONTENT_HASH=$(compute_rocm_csrc_content_hash)
1321 - ROCM_CSRC_CONTENT_CACHE_REF="${cache_repo}:csrc-rocm-input-${ROCM_CSRC_CONTENT_HASH}"
1544 + ROCM_CSRC_TRUSTED_CONTENT_CACHE_REF="${cache_repo}:csrc-rocm-input-${ROCM_CSRC_CONTENT_HASH}"
1545 + ROCM_CSRC_CONTENT_CACHE_REF="${ROCM_CSRC_TRUSTED_CONTENT_CACHE_REF}"
1546 + if [[ -n "${write_scope}" ]]; then
1547 + ROCM_CSRC_CONTENT_CACHE_REF="${ROCM_CSRC_CONTENT_CACHE_REF}-${write_scope}"
1548 + fi
1549 export ROCM_CSRC_CONTENT_HASH
1550 + export ROCM_CSRC_TRUSTED_CONTENT_CACHE_REF
1551 export ROCM_CSRC_CONTENT_CACHE_REF
1552 echo "ROCm csrc content cache ref: ${ROCM_CSRC_CONTENT_CACHE_REF}"
1553 }
@@ -1327,23 +1555,17 @@ compute_rocm_csrc_content_hash_if_needed() {
1555 compute_rocm_rust_content_hash() {
1556 local bake_dir=""
1557 local dockerfile_rocm=""
1330 - local -a content_paths=(
1331 - "requirements/build/rust.txt"
1332 - "rust/Cargo.lock"
1333 - "rust/Cargo.toml"
1334 - "rust/proto"
1335 - "rust/src"
1336 - "rust-toolchain.toml"
1337 - "tools/build_rust.py"
1338 - "tools/install_protoc.sh"
1339 - "build_rust.sh"
1340 - )
1558 + local content_files="${ROCM_RUST_CONTENT_FILES:-${DEFAULT_ROCM_RUST_CONTENT_FILES}}"
1559 + local stages="${ROCM_RUST_DOCKERFILE_STAGES:-${DEFAULT_ROCM_RUST_DOCKERFILE_STAGES}}"
1560 + local -a content_paths=()
1561 local -a content_args=()
1562
1563 bake_dir=$(dirname "${VLLM_BAKE_FILE}")
1564 dockerfile_rocm="${bake_dir}/Dockerfile.rocm"
1565 + read -r -a content_paths <<< "${content_files}"
1566 mapfile -t content_args < <(
1346 - get_content_arg_names "${dockerfile_rocm}" "base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust_input_0 rust_input_1 rust-input rust-toolchain rust-build" "${ROCM_RUST_CONTENT_ARGS:-}"
1567 + get_content_arg_names \
1568 + "${dockerfile_rocm}" "${stages}" "${ROCM_RUST_CONTENT_ARGS:-}"
1569 )
1570
1571 {
@@ -1351,9 +1573,9 @@ compute_rocm_rust_content_hash() {
1573 printf 'dockerfile:%s\n' "${dockerfile_rocm}"
1574 printf 'resolved-build-args:\n'
1575 hash_dockerfile_arg_values "${dockerfile_rocm}" "${content_args[@]}"
1354 - printf 'dockerfile-stages:base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust_input_0 rust_input_1 rust-input rust-toolchain rust-build\n'
1576 + printf 'dockerfile-stages:%s\n' "${stages}"
1577 if [[ -f "${dockerfile_rocm}" ]]; then
1356 - hash_dockerfile_stages "${dockerfile_rocm}" "base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust_input_0 rust_input_1 rust-input rust-toolchain rust-build"
1578 + hash_dockerfile_stages "${dockerfile_rocm}" "${stages}"
1579 else
1580 printf 'missing:%s\n' "${dockerfile_rocm}"
1581 fi
@@ -1362,14 +1584,20 @@ compute_rocm_rust_content_hash() {
1584
1585 compute_rocm_rust_content_hash_if_needed() {
1586 local cache_repo="${DOCKERHUB_CACHE_REPO:-rocm/vllm-ci-cache}"
1587 + local write_scope="${CI_BASE_WRITE_SCOPE:-}"
1588
1589 if [[ "${ROCM_RUST_CONTENT_CACHE:-1}" == "0" ]] || ! uses_rocm_rust_cache; then
1590 return 0
1591 fi
1592
1593 ROCM_RUST_CONTENT_HASH=$(compute_rocm_rust_content_hash)
1371 - ROCM_RUST_CONTENT_CACHE_REF="${cache_repo}:rust-rocm-input-${ROCM_RUST_CONTENT_HASH}"
1594 + ROCM_RUST_TRUSTED_CONTENT_CACHE_REF="${cache_repo}:rust-rocm-input-${ROCM_RUST_CONTENT_HASH}"
1595 + ROCM_RUST_CONTENT_CACHE_REF="${ROCM_RUST_TRUSTED_CONTENT_CACHE_REF}"
1596 + if [[ -n "${write_scope}" ]]; then
1597 + ROCM_RUST_CONTENT_CACHE_REF="${ROCM_RUST_CONTENT_CACHE_REF}-${write_scope}"
1598 + fi
1599 export ROCM_RUST_CONTENT_HASH
1600 + export ROCM_RUST_TRUSTED_CONTENT_CACHE_REF
1601 export ROCM_RUST_CONTENT_CACHE_REF
1602 echo "ROCm Rust content cache ref: ${ROCM_RUST_CONTENT_CACHE_REF}"
1603 }
@@ -1430,8 +1658,14 @@ write_rocm_build_arg_override() {
1658 "${dockerfile_rocm}" \
1659 "${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}" \
1660 "${CI_BASE_CONTENT_ARGS:-}"
1433 - get_content_arg_names "${dockerfile_rocm}" "base csrc-build" "${ROCM_CSRC_CONTENT_ARGS:-}"
1434 - get_content_arg_names "${dockerfile_rocm}" "base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust_input_0 rust_input_1 rust-input rust-toolchain rust-build" "${ROCM_RUST_CONTENT_ARGS:-}"
1661 + get_content_arg_names \
1662 + "${dockerfile_rocm}" \
1663 + "${ROCM_CSRC_DOCKERFILE_STAGES:-${DEFAULT_ROCM_CSRC_DOCKERFILE_STAGES}}" \
1664 + "${ROCM_CSRC_CONTENT_ARGS:-}"
1665 + get_content_arg_names \
1666 + "${dockerfile_rocm}" \
1667 + "${ROCM_RUST_DOCKERFILE_STAGES:-${DEFAULT_ROCM_RUST_DOCKERFILE_STAGES}}" \
1668 + "${ROCM_RUST_CONTENT_ARGS:-}"
1669 } | awk 'NF && !seen[$0]++'
1670 )
1671
@@ -1497,6 +1731,7 @@ validate_content_cache_export_mode() {
1731 should_export_content_cache_ref() {
1732 local cache_ref="$1"
1733 local cache_name="$2"
1734 + local trusted_ref="${3:-${cache_ref}}"
1735 local mode="${ROCM_CONTENT_CACHE_EXPORT_MODE:-missing}"
1736
1737 case "${mode}" in
@@ -1509,8 +1744,13 @@ should_export_content_cache_ref() {
1744 return 1
1745 ;;
1746 missing|"")
1512 - if docker buildx imagetools inspect "${cache_ref}" >/dev/null 2>&1; then
1513 - echo "${cache_name} content cache exists; not re-exporting ${cache_ref}"
1747 + if remote_image_exists "${trusted_ref}"; then
1748 + echo "${cache_name} trusted content cache exists; not exporting ${cache_ref}"
1749 + return 1
1750 + fi
1751 + if [[ "${cache_ref}" != "${trusted_ref}" ]] \
1752 + && remote_image_exists "${cache_ref}"; then
1753 + echo "${cache_name} scoped content cache exists; not re-exporting ${cache_ref}"
1754 return 1
1755 fi
1756 echo "${cache_name} content cache missing; will export ${cache_ref}"
@@ -1555,8 +1795,18 @@ write_rocm_cache_override() {
1795 echo "ROCm final image cache export mode: ${rocm_cache_to_mode}"
1796
1797 if [[ -n "${ROCM_CSRC_CONTENT_CACHE_REF:-}" ]]; then
1558 - csrc_content_cache_from+=("type=registry,ref=${ROCM_CSRC_CONTENT_CACHE_REF}")
1559 - if should_export_content_cache_ref "${ROCM_CSRC_CONTENT_CACHE_REF}" "ROCm csrc"; then
1798 + csrc_content_cache_from+=(
1799 + "type=registry,ref=${ROCM_CSRC_TRUSTED_CONTENT_CACHE_REF}"
1800 + )
1801 + if [[ "${ROCM_CSRC_CONTENT_CACHE_REF}" != \
1802 + "${ROCM_CSRC_TRUSTED_CONTENT_CACHE_REF}" ]]; then
1803 + csrc_content_cache_from+=(
1804 + "type=registry,ref=${ROCM_CSRC_CONTENT_CACHE_REF}"
1805 + )
1806 + fi
1807 + if should_export_content_cache_ref \
1808 + "${ROCM_CSRC_CONTENT_CACHE_REF}" "ROCm csrc" \
1809 + "${ROCM_CSRC_TRUSTED_CONTENT_CACHE_REF}"; then
1810 csrc_cache_to+=(
1811 "type=registry,ref=${ROCM_CSRC_CONTENT_CACHE_REF},mode=${csrc_cache_to_mode},ignore-error=true"
1812 )
@@ -1566,8 +1816,18 @@ write_rocm_cache_override() {
1816 fi
1817
1818 if [[ -n "${ROCM_RUST_CONTENT_CACHE_REF:-}" ]]; then
1569 - rust_content_cache_from+=("type=registry,ref=${ROCM_RUST_CONTENT_CACHE_REF}")
1570 - if should_export_content_cache_ref "${ROCM_RUST_CONTENT_CACHE_REF}" "ROCm Rust"; then
1819 + rust_content_cache_from+=(
1820 + "type=registry,ref=${ROCM_RUST_TRUSTED_CONTENT_CACHE_REF}"
1821 + )
1822 + if [[ "${ROCM_RUST_CONTENT_CACHE_REF}" != \
1823 + "${ROCM_RUST_TRUSTED_CONTENT_CACHE_REF}" ]]; then
1824 + rust_content_cache_from+=(
1825 + "type=registry,ref=${ROCM_RUST_CONTENT_CACHE_REF}"
1826 + )
1827 + fi
1828 + if should_export_content_cache_ref \
1829 + "${ROCM_RUST_CONTENT_CACHE_REF}" "ROCm Rust" \
1830 + "${ROCM_RUST_TRUSTED_CONTENT_CACHE_REF}"; then
1831 rust_cache_to+=(
1832 "type=registry,ref=${ROCM_RUST_CONTENT_CACHE_REF},mode=${rust_cache_to_mode},ignore-error=true"
1833 )
@@ -1772,7 +2032,7 @@ compose_stage_cache_material() {
2032
2033 dependency_cache_ref_exists() {
2034 local cache_ref="$1"
1775 - docker buildx imagetools inspect "${cache_ref}" >/dev/null 2>&1
2035 + registry_ref_exists_with_retry imagetools "${cache_ref}"
2036 }
2037
2038 dependency_cache_ref_for_target() {
@@ -1914,54 +2174,18 @@ print_bake_config() {
2174
2175 confirm_remote_image_push() {
2176 local image_ref="$1"
1917 - local remote_hash=""
1918 - local remote_revision=""
1919 -
1920 - if ! remote_image_exists "${image_ref}"; then
1921 - return 1
1922 - fi
1923 -
1924 - if is_ci_base_target; then
1925 - if [[ -z "${CI_BASE_CONTENT_HASH:-}" ]]; then
1926 - return 0
1927 - fi
1928 -
1929 - remote_hash=$(get_remote_image_label_with_retry "${image_ref}" "vllm.ci_base.content_hash")
1930 - if [[ -n "${remote_hash}" && "${remote_hash}" == "${CI_BASE_CONTENT_HASH}" ]]; then
1931 - if remote_ci_base_metadata_is_current_with_retry "${image_ref}"; then
1932 - return 0
1933 - fi
1934 -
1935 - echo "Remote image exists with the expected ci_base content hash but stale metadata."
1936 - echo " expected metadata version: ${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}"
1937 - return 1
1938 - fi
2177
1940 - echo "Remote image exists but does not have the expected ci_base content hash."
1941 - echo " expected: ${CI_BASE_CONTENT_HASH:0:16}..."
1942 - echo " found: ${remote_hash:0:16}..."
1943 - return 1
2178 + if [[ -z "${CI_BASE_CONTENT_HASH:-}" ]]; then
2179 + remote_image_exists "${image_ref}"
2180 + return
2181 fi
2182
1946 - if is_commit_image_target; then
1947 - remote_revision=$(get_remote_image_label_with_retry "${image_ref}" "org.opencontainers.image.revision")
1948 - if [[ -n "${remote_revision}" && "${remote_revision}" == "${BUILDKITE_COMMIT}" ]]; then
1949 - return 0
1950 - fi
1951 -
1952 - if [[ -z "${remote_revision}" \
1953 - && ${IMAGE_EXISTED_BEFORE_BUILD} -eq 0 ]] \
1954 - && image_tag_is_commit_scoped; then
1955 - echo "Remote image exists under a commit-scoped tag; accepting push despite missing revision label."
1956 - return 0
1957 - fi
1958 -
1959 - echo "Remote image exists but revision label does not match ${BUILDKITE_COMMIT}."
1960 - echo " found revision: ${remote_revision:-<missing>}"
1961 - return 1
2183 + if remote_ci_base_identity_is_current_with_retry "${image_ref}"; then
2184 + return 0
2185 fi
2186
1964 - return 0
2187 + echo "Remote image does not have the expected complete ci_base identity."
2188 + return 1
2189 }
2190
2191 verify_dependency_cache_ref() {
@@ -2017,64 +2241,27 @@ seed_dependency_caches_if_needed() {
2241 done
2242 }
2243
2020 -annotate_cache_export_warning() {
2021 - local build_rc="$1"
2244 +run_bake() {
2245 + local confirmation_ref="${IMAGE_TAG:-}"
2246
2023 - if ! command -v buildkite-agent >/dev/null 2>&1; then
2024 - return 0
2247 + if is_ci_base_target && [[ -n "${CI_BASE_IMAGE_TAG_BUILD_REF:-}" ]]; then
2248 + confirmation_ref="${CI_BASE_IMAGE_TAG_BUILD_REF}"
2249 fi
2250
2027 - buildkite-agent annotate \
2028 - --style warning \
2029 - --context "cache-export-warning" \
2030 - "### :warning: Docker cache export failed (non-fatal)
2031 -
2032 -Image was pushed successfully: \`${IMAGE_TAG}\`
2033 -
2034 -The BuildKit build returned exit code ${build_rc}, but the expected image
2035 -is present in the registry. Treating this as a registry cache export failure
2036 -so tests can continue with the pushed image." 2>/dev/null || true
2037 -}
2038 -
2039 -run_bake() {
2040 - local build_rc=0
2041 -
2251 echo "--- :docker: Building ${TARGET}"
2252 docker buildx bake \
2253 "${BAKE_FILES[@]}" \
2254 --progress "${BUILDKIT_PROGRESS:-plain}" \
2046 - "${BAKE_TARGETS[@]}" || build_rc=$?
2255 + "${BAKE_TARGETS[@]}"
2256
2048 - if [[ ${build_rc} -eq 0 ]]; then
2049 - echo "--- :white_check_mark: Build complete"
2050 - return 0
2051 - fi
2052 -
2053 - echo ""
2054 - echo "WARNING: docker buildx bake exited with code ${build_rc}"
2055 -
2056 - if [[ -n "${IMAGE_TAG:-}" ]]; then
2057 - echo "Checking if image was pushed successfully..."
2058 - if confirm_remote_image_push "${IMAGE_TAG}"; then
2059 - echo ""
2060 - echo "WARNING: Build reported failure (rc=${build_rc}) but the"
2061 - echo " image was pushed successfully: ${IMAGE_TAG}"
2062 - echo ""
2063 - echo " Treating this as a non-fatal registry cache export failure."
2064 - echo " The image is usable, but registry cache may be cold on the next build."
2065 - echo ""
2066 - annotate_cache_export_warning "${build_rc}"
2067 - echo "--- :white_check_mark: Build complete"
2068 - return 0
2257 + if is_ci_base_target; then
2258 + if ! confirm_remote_image_push "${confirmation_ref}"; then
2259 + echo "Fresh ci_base image failed identity validation: ${confirmation_ref}" >&2
2260 + return 1
2261 fi
2070 -
2071 - echo ""
2072 - echo "ERROR: Build failed and image was NOT confirmed: ${IMAGE_TAG}"
2073 - echo " This is a real build failure, not a cache export warning."
2074 - echo ""
2262 + refresh_ci_base_tags_from_ref "${confirmation_ref}"
2263 fi
2076 -
2077 - return "${build_rc}"
2264 + echo "--- :white_check_mark: Build complete"
2265 }
2266
2267 upload_wheel_artifacts_if_present() {
@@ -2082,7 +2269,10 @@ upload_wheel_artifacts_if_present() {
2269 local artifact_dir="artifacts/vllm-rocm-install"
2270 local archive_name="vllm-rocm-install.tar.gz"
2271 local metadata_dir="${wheel_dir}/.vllm-ci-artifact"
2272 + local build_base_digest=""
2273 + local expected_native_base_image=""
2274 local native_base_image=""
2275 + local native_base_digest=""
2276 local whl=""
2277 local whl_name=""
2278 local -a wheels=()
@@ -2100,11 +2290,35 @@ upload_wheel_artifacts_if_present() {
2290 fi
2291 whl="${wheels[0]}"
2292 whl_name=$(basename "${whl}")
2103 - native_base_image="${CI_BASE_IMAGE_TAG_COMMIT_REF:-${CI_BASE_IMAGE:-}}"
2293 + native_base_image="${CI_BASE_IMAGE_TAG_BUILD_REF:-${CI_BASE_IMAGE:-}}"
2294 if [[ -z "${native_base_image}" ]]; then
2295 echo "Native ROCm artifact requires a ci_base image reference" >&2
2296 return 1
2297 fi
2298 + if [[ "${BUILDKITE:-false}" == "true" ]]; then
2299 + expected_native_base_image=$(ci_base_tag_with_suffix \
2300 + "${CI_BASE_IMAGE_TAG:-rocm/vllm-dev:ci_base}" \
2301 + "build-${BUILDKITE_BUILD_ID:-}")
2302 + if [[ "${native_base_image}" != "${expected_native_base_image}" ]]; then
2303 + echo "Native ROCm artifact requires the exact ci_base build handoff: ${native_base_image}" >&2
2304 + return 1
2305 + fi
2306 + if [[ ! "${CI_BASE_IMAGE:-}" =~ @sha256:[0-9a-f]{64}$ ]]; then
2307 + echo "ROCm artifact build base must be digest-pinned: ${CI_BASE_IMAGE:-<empty>}" >&2
2308 + return 1
2309 + fi
2310 + build_base_digest="${CI_BASE_IMAGE##*@}"
2311 + if ! native_base_digest=$(resolve_image_digest "${native_base_image}"); then
2312 + echo "Could not resolve native ci_base handoff: ${native_base_image}" >&2
2313 + return 1
2314 + fi
2315 + if [[ "${native_base_digest}" != "${build_base_digest}" ]]; then
2316 + echo "Native ci_base handoff does not match the artifact build base" >&2
2317 + echo " native: ${native_base_image}@${native_base_digest}" >&2
2318 + echo " build: ${CI_BASE_IMAGE}" >&2
2319 + return 1
2320 + fi
2321 + fi
2322
2323 echo "--- :package: Uploading ROCm vLLM install artifact"
2324 rm -rf "${artifact_dir}" "${metadata_dir}"
@@ -2141,8 +2355,10 @@ upload_wheel_artifacts_if_present() {
2355
2356 main() {
2357 init_config "$@"
2358 + configure_ci_base_write_scope
2359 print_header
2360 validate_inputs
2361 + normalize_ci_worktree_modes
2362 load_ci_hcl
2363 init_bake_files
2364 compute_ci_base_hash_if_needed
@@ -2170,7 +2386,11 @@ main() {
2386 fi
2387 seed_dependency_caches_if_needed
2388 run_bake
2389 + promote_stable_ci_base_tag
2390 + publish_ci_base_handoff_ref
2391 upload_wheel_artifacts_if_present
2392 }
2393
2176 -main "$@"
2394 +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
2395 + main "$@"
2396 +fi
.buildkite/scripts/hardware_ci/run-amd-test.sh
+247
@@ -36,6 +36,19 @@ set -o pipefail
36 : "${ROCM_DOCKER_TTY:=1}"
37 : "${PYTHONFAULTHANDLER:=1}"
38 : "${PYTEST_TIMEOUT:=2400}"
39 +
40 +# Preserve runner-only values before clear_ci_orchestration_env removes them
41 +# from the environment inherited by the test process.
42 +amd_diagnostics_expected_gpu_count="${VLLM_CI_EXPECTED_GPU_COUNT:-1}"
43 +amd_diagnostics_dir="${VLLM_CI_DIAGNOSTICS_DIR:-artifacts/amd-gpu-diagnostics}"
44 +amd_diagnostics_checkout_root="${BUILDKITE_BUILD_CHECKOUT_PATH:-$(pwd -P)}"
45 +amd_diagnostics_execution_mode="${VLLM_CI_EXECUTION_MODE:-single-node}"
46 +amd_diagnostics_test_group="${VLLM_TEST_GROUP_NAME:-${BUILDKITE_STEP_KEY:-unknown}}"
47 +amd_diagnostics_k8s_node_name="${VLLM_CI_K8S_NODE_NAME:-}"
48 +amd_diagnostics_collected=0
49 +amd_diagnostics_probe_budget_seconds=25
50 +amd_diagnostics_command_timeout_seconds=5
51 +amd_diagnostics_upload_timeout_seconds=20
52 if [[ " ${PYTEST_ADDOPTS:-} " != *" --color"* ]]; then
53 PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--color=yes"
54 fi
@@ -84,9 +97,11 @@ clear_ci_orchestration_env() {
97 VLLM_CI_BASE_IMAGE \
98 VLLM_CI_FALLBACK_IMAGE \
99 VLLM_CI_DOCKER_DISABLED \
100 + VLLM_CI_DIAGNOSTICS_DIR \
101 VLLM_CI_ARTIFACT_GLOB \
102 VLLM_CI_ARTIFACT_CHECKSUM_GLOB \
103 VLLM_CI_EXPECTED_GPU_COUNT \
104 + VLLM_CI_K8S_NODE_NAME \
105 VLLM_CI_USE_ARTIFACTS \
106 VLLM_CI_RESULTS_ROOT \
107 VLLM_ALLOW_DEPRECATED_BEAM_SEARCH
@@ -350,6 +365,11 @@ prepare_native_workspace() {
365 echo "ROCm artifact commit ${recorded_commit} does not match ${BUILDKITE_COMMIT:-unset}" >&2
366 return 1
367 fi
368 + if [[ -z "${BUILDKITE_BUILD_ID:-}" \
369 + || "${recorded_base}" != *":ci_base-build-${BUILDKITE_BUILD_ID}" ]]; then
370 + echo "ROCm artifact base is not scoped to Buildkite build ${BUILDKITE_BUILD_ID:-unset}: ${recorded_base}" >&2
371 + return 1
372 + fi
373 if [[ -z "${VLLM_CI_BASE_IMAGE:-}" || "${recorded_base}" != "${VLLM_CI_BASE_IMAGE}" ]]; then
374 echo "ROCm artifact base ${recorded_base} does not match ${VLLM_CI_BASE_IMAGE:-unset}" >&2
375 return 1
@@ -495,12 +515,220 @@ is_multi_node() {
515 return 1
516 }
517
518 +run_amd_diagnostic() {
519 + local log_file=$1
520 + local probe_deadline=$2
521 + shift 2
522 + local command_status=0
523 + local command_timeout_seconds=${amd_diagnostics_command_timeout_seconds}
524 + local remaining_seconds=$((probe_deadline - SECONDS))
525 +
526 + {
527 + printf '\n$'
528 + printf ' %q' "$@"
529 + printf '\n'
530 + } | tee -a "${log_file}"
531 +
532 + if [[ "${remaining_seconds}" -le 0 ]]; then
533 + echo "[diagnostic probe budget exhausted; command skipped]" \
534 + | tee -a "${log_file}"
535 + return 0
536 + fi
537 + if [[ "${remaining_seconds}" -lt "${command_timeout_seconds}" ]]; then
538 + command_timeout_seconds=${remaining_seconds}
539 + fi
540 +
541 + timeout --kill-after=1s "${command_timeout_seconds}s" "$@" 2>&1 \
542 + | tee -a "${log_file}"
543 + command_status=${PIPESTATUS[0]}
544 + if [[ "${command_status}" -ne 0 ]]; then
545 + echo "[diagnostic command exited with status ${command_status}]" \
546 + | tee -a "${log_file}"
547 + fi
548 + return 0
549 +}
550 +
551 +collect_rocm_failure_diagnostics() {
552 + local exit_code=$1
553 + local job_id="${BUILDKITE_JOB_ID:-local}"
554 + local retry_count="${BUILDKITE_RETRY_COUNT:-0}"
555 + local parallel_job="${BUILDKITE_PARALLEL_JOB:-0}"
556 + local diagnostics_relative_path=""
557 + local diagnostics_path=""
558 + local upload_root="${amd_diagnostics_checkout_root}"
559 + local upload_path=""
560 + local exit_signal=""
561 + local probe_deadline=0
562 +
563 + if [[ "${amd_diagnostics_collected}" == "1" ]]; then
564 + return 0
565 + fi
566 + amd_diagnostics_collected=1
567 +
568 + if [[ "${amd_diagnostics_expected_gpu_count}" == "0" ]]; then
569 + echo "Skipping AMD GPU diagnostics for a CPU-only job."
570 + return 0
571 + fi
572 +
573 + if ! command -v timeout >/dev/null 2>&1; then
574 + echo "WARNING: timeout is unavailable; skipping failure diagnostics rather than risking a hung job."
575 + return 0
576 + fi
577 +
578 + job_id="${job_id//[^A-Za-z0-9_.-]/_}"
579 + retry_count="${retry_count//[^A-Za-z0-9_.-]/_}"
580 + parallel_job="${parallel_job//[^A-Za-z0-9_.-]/_}"
581 +
582 + # Buildkite keeps the supplied artifact path. Restrict the configurable
583 + # directory to a checkout-relative path so the UI shows a clean artifact key.
584 + if [[ -z "${amd_diagnostics_dir}" \
585 + || "${amd_diagnostics_dir}" == /* \
586 + || "${amd_diagnostics_dir}" == *".."* ]]; then
587 + echo "WARNING: ignoring unsafe VLLM_CI_DIAGNOSTICS_DIR"
588 + amd_diagnostics_dir="artifacts/amd-gpu-diagnostics"
589 + fi
590 + diagnostics_relative_path="${amd_diagnostics_dir}/${job_id}/amd-gpu-diagnostics.log"
591 + diagnostics_path="${amd_diagnostics_checkout_root}/${diagnostics_relative_path}"
592 + upload_path="${diagnostics_relative_path}"
593 +
594 + if ! mkdir -p "$(dirname "${diagnostics_path}")" \
595 + || ! : > "${diagnostics_path}"; then
596 + diagnostics_path=$(mktemp -t amd-gpu-diagnostics.XXXXXX.log) || {
597 + echo "WARNING: unable to create an AMD GPU diagnostics log."
598 + return 0
599 + }
600 + upload_root=$(dirname "${diagnostics_path}")
601 + upload_path=$(basename "${diagnostics_path}")
602 + fi
603 +
604 + if [[ "${exit_code}" -gt 128 && "${exit_code}" -le 192 ]]; then
605 + exit_signal=$(kill -l "$((exit_code - 128))" 2>/dev/null || true)
606 + fi
607 +
608 + {
609 + echo "AMD GPU failure diagnostics"
610 + echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
611 + echo "exit_code=${exit_code}"
612 + echo "exit_signal=${exit_signal:-none}"
613 + echo "build_url=${BUILDKITE_BUILD_URL:-unknown}"
614 + echo "job_id=${BUILDKITE_JOB_ID:-unknown}"
615 + echo "step_key=${BUILDKITE_STEP_KEY:-unknown}"
616 + echo "step_label=${BUILDKITE_LABEL:-unknown}"
617 + echo "test_group=${amd_diagnostics_test_group}"
618 + echo "retry_count=${retry_count}"
619 + echo "parallel_job=${parallel_job}"
620 + echo "execution_mode=${amd_diagnostics_execution_mode}"
621 + echo "expected_gpu_count=${amd_diagnostics_expected_gpu_count}"
622 + echo "probe_budget_seconds=${amd_diagnostics_probe_budget_seconds}"
623 + echo "command_timeout_seconds=${amd_diagnostics_command_timeout_seconds}"
624 + echo "upload_timeout_seconds=${amd_diagnostics_upload_timeout_seconds}"
625 + echo "agent_name=${BUILDKITE_AGENT_NAME:-unknown}"
626 + echo "k8s_node=${amd_diagnostics_k8s_node_name:-unknown}"
627 + echo "kernel=$(uname -srmo 2>/dev/null || echo unknown)"
628 + } > "${diagnostics_path}"
629 +
630 + echo "--- :rotating_light: AMD GPU failure diagnostics"
631 + cat "${diagnostics_path}"
632 +
633 + probe_deadline=$((SECONDS + amd_diagnostics_probe_budget_seconds))
634 +
635 + # Collect the most useful evidence first so auxiliary probes cannot consume
636 + # the shared deadline before GPU health is captured.
637 + echo "AMD GPU diagnostics" | tee -a "${diagnostics_path}"
638 + if command -v amd-smi >/dev/null 2>&1; then
639 + run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi version
640 + # Bus data identifies a card within the public node without publishing its
641 + # persistent UUID, serial number, or process list.
642 + run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi static -b -g all
643 + run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi metric -e -k -P -x -g all
644 + run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi bad-pages -p -r -u -g all
645 + run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi metric -p -t -u -m -v -g all
646 + run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" amd-smi xgmi -l -g all
647 + elif command -v rocm-smi >/dev/null 2>&1; then
648 + run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" rocm-smi \
649 + --showbus --showreplaycount --showrasinfo --showpagesinfo
650 + else
651 + echo "Neither amd-smi nor rocm-smi is available." \
652 + | tee -a "${diagnostics_path}"
653 + fi
654 +
655 + echo "Node resource diagnostics (runner-visible scope)" \
656 + | tee -a "${diagnostics_path}"
657 + if command -v df >/dev/null 2>&1; then
658 + echo "checkout_filesystem:" | tee -a "${diagnostics_path}"
659 + if [[ -d "${amd_diagnostics_checkout_root}" ]]; then
660 + (
661 + cd "${amd_diagnostics_checkout_root}" || exit 0
662 + run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" df -h \
663 + --output=fstype,size,used,avail,pcent,itotal,iused,iavail,ipcent \
664 + -- .
665 + )
666 + else
667 + echo "[checkout directory unavailable]" | tee -a "${diagnostics_path}"
668 + fi
669 + echo "tmp_filesystem:" | tee -a "${diagnostics_path}"
670 + run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" df -h \
671 + --output=fstype,size,used,avail,pcent,itotal,iused,iavail,ipcent \
672 + -- /tmp
673 + fi
674 + if command -v free >/dev/null 2>&1; then
675 + run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" free -h
676 + fi
677 + if [[ -r /sys/fs/cgroup/memory.events ]]; then
678 + run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" cat \
679 + /sys/fs/cgroup/memory.events
680 + fi
681 +
682 + echo "Unauthenticated outer-runner network reachability diagnostics" \
683 + | tee -a "${diagnostics_path}"
684 + if command -v curl >/dev/null 2>&1; then
685 + run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" curl -q \
686 + --silent --location --proto '=https' --proto-redir '=https' \
687 + --max-redirs 3 --output /dev/null --connect-timeout 2 --max-time 4 \
688 + --write-out 'target=huggingface http_code=%{http_code} dns_done_s=%{time_namelookup} connect_done_s=%{time_connect} tls_done_s=%{time_appconnect} first_byte_s=%{time_starttransfer} total_s=%{time_total}\n' \
689 + https://huggingface.co/api/models/gpt2
690 + run_amd_diagnostic "${diagnostics_path}" "${probe_deadline}" curl -q \
691 + --silent --location --proto '=https' --proto-redir '=https' \
692 + --max-redirs 3 --output /dev/null --connect-timeout 2 --max-time 4 \
693 + --write-out 'target=github_git_smart_http http_code=%{http_code} dns_done_s=%{time_namelookup} connect_done_s=%{time_connect} tls_done_s=%{time_appconnect} first_byte_s=%{time_starttransfer} total_s=%{time_total}\n' \
694 + 'https://github.com/vllm-project/ci-infra.git/info/refs?service=git-upload-pack'
695 + else
696 + echo "curl unavailable; reachability probes skipped." \
697 + | tee -a "${diagnostics_path}"
698 + fi
699 +
700 + if command -v buildkite-agent >/dev/null 2>&1 \
701 + && [[ -n "${BUILDKITE_JOB_ID:-}" ]]; then
702 + if (
703 + cd "${upload_root}" \
704 + && BUILDKITE_AGENT_DEBUG=false \
705 + BUILDKITE_AGENT_DEBUG_HTTP=false \
706 + BUILDKITE_AGENT_TRACE_HTTP=false \
707 + BUILDKITE_AGENT_LOG_LEVEL=error \
708 + timeout --kill-after=2s "${amd_diagnostics_upload_timeout_seconds}s" \
709 + buildkite-agent artifact upload "${upload_path}" \
710 + >/dev/null 2>&1
711 + ); then
712 + echo "Uploaded AMD GPU diagnostics artifact: ${upload_path}"
713 + else
714 + echo "WARNING: failed to upload AMD GPU diagnostics artifact: ${upload_path}"
715 + fi
716 + else
717 + echo "Buildkite agent unavailable; diagnostics artifact was not uploaded."
718 + fi
719 +
720 + return 0
721 +}
722 +
723 handle_pytest_exit() {
724 local exit_code=$1
725 if [ "$exit_code" -eq 5 ]; then
726 echo "Pytest exit code 5 (no tests collected) - treating as success."
727 exit 0
728 fi
729 + if [[ "${exit_code}" -ne 0 ]]; then
730 + collect_rocm_failure_diagnostics "${exit_code}"
731 + fi
732 exit "$exit_code"
733 }
734
@@ -659,6 +887,19 @@ re_quote_pytest_markers() {
887 echo "${output% }"
888 }
889
890 +# shellcheck disable=SC2317 # Called indirectly by the EXIT trap.
891 +handle_amd_runner_exit() {
892 + local exit_code=${1:-$?}
893 + if [[ "${exit_code}" -ne 0 ]]; then
894 + collect_rocm_failure_diagnostics "${exit_code}"
895 + fi
896 + return 0
897 +}
898 +
899 +# Catch both test failures and wrapper/setup failures. Runtime-specific cleanup
900 +# traps below replace this trap but call the same handler before cleaning up.
901 +trap handle_amd_runner_exit EXIT
902 +
903 ###############################################################################
904 # Main
905 ###############################################################################
@@ -667,7 +908,10 @@ if is_native_runtime; then
908 echo "--- Native in-pod ROCm CI (AMD_CI_RUNTIME=${AMD_CI_RUNTIME:-unset}, NATIVE_CI=${NATIVE_CI:-unset})"
909 artifact_work_dir=""
910
911 + # shellcheck disable=SC2317 # Called indirectly by the EXIT trap.
912 cleanup_native_workspace() {
913 + local exit_code=$?
914 + handle_amd_runner_exit "${exit_code}"
915 if [[ -n "${artifact_work_dir}" ]]; then
916 rm -rf "${artifact_work_dir}"
917 fi
@@ -727,7 +971,10 @@ image_name="${VLLM_CI_FALLBACK_IMAGE:-rocm/vllm-ci:${BUILDKITE_COMMIT:-local}}"
971 artifact_work_dir=""
972 container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)"
973
974 +# shellcheck disable=SC2317 # Called indirectly by the EXIT trap.
975 remove_docker_container() {
976 + local exit_code=$?
977 + handle_amd_runner_exit "${exit_code}"
978 if docker container inspect "${container_name}" >/dev/null 2>&1; then
979 docker rm -f "${container_name}" || true
980 fi
.buildkite/scripts/rocm/build-ci-base.sh
+19 -3
@@ -12,17 +12,33 @@ metadata_get() {
12
13 main() {
14 local base_refreshed=""
15 + local selected_base=""
16 +
17 + # The content identity below describes this checkout. Do not let legacy
18 + # external pipeline settings switch the build to a remote source tree.
19 + export REMOTE_VLLM=0
20 + unset VLLM_BRANCH
21
22 base_refreshed="$(metadata_get rocm-base-refresh)"
23 + selected_base="$(metadata_get rocm-base-image)"
24 + if [[ -n "${selected_base}" ]]; then
25 + if [[ ! "${selected_base}" =~ @sha256:[0-9a-f]{64}$ ]]; then
26 + echo "Selected ROCm base handoff is not digest-pinned: ${selected_base}" >&2
27 + return 1
28 + fi
29 + export BASE_IMAGE="${selected_base}"
30 + echo "Using selected ROCm base image for ci_base: ${BASE_IMAGE}"
31 + elif [[ "${BUILDKITE:-false}" == "true" ]]; then
32 + echo "Required ROCm base handoff metadata is missing" >&2
33 + return 1
34 + fi
35 +
36 if [[ "${base_refreshed}" == "1" ]]; then
18 - export BASE_IMAGE
37 export CI_BASE_PUSH_STABLE_TAG
38
21 - BASE_IMAGE="$(metadata_get rocm-base-image)"
39 CI_BASE_PUSH_STABLE_TAG="$(metadata_get rocm-base-push-stable-tag)"
40 CI_BASE_PUSH_STABLE_TAG="${CI_BASE_PUSH_STABLE_TAG:-0}"
41
25 - echo "Using refreshed ROCm base image for ci_base: ${BASE_IMAGE}"
42 echo "Push stable ci_base tag: ${CI_BASE_PUSH_STABLE_TAG}"
43 fi
44
.buildkite/scripts/rocm/build-test-image.sh
+39 -30
@@ -1,8 +1,8 @@
1 #!/usr/bin/env bash
2 # Build the ROCm CI test image or wheel artifact.
3 #
4 -# When Dockerfile.rocm_base changes, always build the full image so downstream
5 -# ROCm tests can validate the freshly rebuilt base -> ci_base -> ci image chain.
4 +# When base selection changes, build the full image so downstream ROCm tests
5 +# validate the selected base -> ci_base -> CI image chain.
6
7 set -euo pipefail
8
@@ -13,46 +13,55 @@ metadata_get() {
13 fi
14 }
15
16 -use_ci_base_if_present() {
17 - local ci_base_image=""
16 +load_digest_handoff() {
17 + local metadata_key="$1"
18 + local env_name="$2"
19 + local description="$3"
20 + local image_ref=""
21
19 - ci_base_image="$(metadata_get rocm-ci-base-image)"
20 - if [[ -z "${ci_base_image}" ]]; then
22 + image_ref="$(metadata_get "${metadata_key}")"
23 + if [[ -z "${image_ref}" ]]; then
24 return 1
25 fi
23 -
24 - export CI_BASE_IMAGE="${ci_base_image}"
25 - echo "Using ROCm ci_base image selected by the preceding build step: ${CI_BASE_IMAGE}"
26 -}
27 -
28 -use_refreshed_base_if_present() {
29 - local base_refreshed=""
30 -
31 - base_refreshed="$(metadata_get rocm-base-refresh)"
32 - if [[ "${base_refreshed}" != "1" ]]; then
26 + if [[ ! "${image_ref}" =~ @sha256:[0-9a-f]{64}$ ]]; then
27 + echo "${description} is not digest-pinned: ${image_ref}" >&2
28 return 1
29 fi
30
36 - export BASE_IMAGE
37 - export IMAGE_TAG_LATEST
38 -
39 - BASE_IMAGE="$(metadata_get rocm-base-image)"
40 - IMAGE_TAG_LATEST="$(metadata_get rocm-ci-image-descriptive)"
41 -
42 - echo "Using refreshed ROCm base image for test image: ${BASE_IMAGE}"
43 - if [[ -n "${IMAGE_TAG_LATEST}" ]]; then
44 - echo "Also tagging full ROCm CI image as: ${IMAGE_TAG_LATEST}"
45 - fi
46 -
47 - return 0
31 + printf -v "${env_name}" '%s' "${image_ref}"
32 + export "${env_name?}"
33 + echo "Using ${description}: ${image_ref}"
34 }
35
36 main() {
37 local base_refreshed=0
38
53 - use_ci_base_if_present || true
39 + # This job always builds the checked-out commit. Some externally generated
40 + # pipeline templates still inject remote-fetch settings; do not let those
41 + # settings make the source identity commit-specific or bypass local edits.
42 + export REMOTE_VLLM=0
43 + unset VLLM_BRANCH
44 +
45 + if ! load_digest_handoff \
46 + rocm-ci-base-image CI_BASE_IMAGE "ROCm ci_base handoff"; then
47 + if [[ "${BUILDKITE:-false}" == "true" ]]; then
48 + echo "Required ROCm ci_base handoff metadata is missing or invalid" >&2
49 + return 1
50 + fi
51 + echo "No ROCm ci_base handoff metadata found; using the local default"
52 + fi
53 +
54 + if ! load_digest_handoff \
55 + rocm-base-image BASE_IMAGE "ROCm base handoff"; then
56 + if [[ "${BUILDKITE:-false}" == "true" ]]; then
57 + echo "Required ROCm base handoff metadata is missing or invalid" >&2
58 + return 1
59 + fi
60 + echo "No ROCm base handoff metadata found; using the local default"
61 + fi
62
55 - if use_refreshed_base_if_present; then
63 + if [[ "$(metadata_get rocm-base-refresh)" == "1" ]]; then
64 + echo "The selected ROCm base differs from the current stable base"
65 base_refreshed=1
66 fi
67
.buildkite/scripts/rocm/refresh-base-image.sh
+398 -323
@@ -1,19 +1,20 @@
1 #!/usr/bin/env bash
2 -# Build and publish a fresh ROCm base image when Dockerfile.rocm_base changes.
3 -#
4 -# Normal AMD CI builds should not pay for this path. The script no-ops unless
5 -# docker/Dockerfile.rocm_base changed relative to the branch base, the previous
6 -# main commit, or ROCM_BASE_REFRESH_FORCE=1 is set.
2 +# Select the content-addressed ROCm base for this checkout, building it only
3 +# when the exact content ref is absent.
4
5 set -euo pipefail
6
7 DOCKERFILE="${ROCM_BASE_DOCKERFILE:-docker/Dockerfile.rocm_base}"
8 BASE_REPO="${ROCM_BASE_IMAGE_REPO:-rocm/vllm-dev}"
12 -CI_IMAGE_REPO="${ROCM_CI_IMAGE_REPO:-rocm/vllm-ci}"
9 +CACHE_REPO="${ROCM_BASE_CACHE_REPO:-${DOCKERHUB_CACHE_REPO:-rocm/vllm-ci-cache}}"
10 BUILDER_NAME="${ROCM_BASE_BUILDER_NAME:-vllm-rocm-base-builder}"
14 -DEFAULT_ROCM_BASE_METADATA_VERSION="1"
11 +DEFAULT_ROCM_BASE_METADATA_VERSION="2"
12 DEFAULT_ROCM_BASE_CONTENT_FILES="${DOCKERFILE}"
16 -DEFAULT_ROCM_BASE_CONTENT_ARGS="BASE_IMAGE TRITON_BRANCH TRITON_REPO PYTORCH_BRANCH PYTORCH_REPO PYTORCH_VISION_BRANCH PYTORCH_VISION_REPO PYTORCH_AUDIO_BRANCH PYTORCH_AUDIO_REPO FA_BRANCH FA_REPO AITER_BRANCH AITER_REPO MORI_BRANCH MORI_REPO PYTORCH_ROCM_ARCH PYTHON_VERSION USE_SCCACHE"
13 +
14 +ROCM_BASE_LAYER_CACHE_REF=""
15 +ROCM_BASE_TRUSTED_LAYER_CACHE_REF="${CACHE_REPO}:rocm-base-main"
16 +ROCM_BASE_STABLE_TAG_UPDATED=0
17 +declare -a ROCM_BASE_CACHE_ARGS=()
18
19 metadata_set() {
20 local key="$1"
@@ -21,7 +22,10 @@ metadata_set() {
22
23 [[ -n "${value}" ]] || return 0
24 if command -v buildkite-agent >/dev/null 2>&1; then
24 - buildkite-agent meta-data set "${key}" "${value}" || true
25 + buildkite-agent meta-data set "${key}" "${value}"
26 + elif [[ "${BUILDKITE:-false}" == "true" ]]; then
27 + echo "buildkite-agent is required to publish ROCm base metadata" >&2
28 + return 1
29 fi
30 }
31
@@ -56,249 +60,263 @@ tag_component() {
60 clean_docker_tag "${input:-unknown}" | cut -c1-"${max_chars}"
61 }
62
59 -extract_arg_default() {
60 - local arg_name="$1"
61 -
62 - sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
63 - "${DOCKERFILE}" | head -1
63 +normalize_repo_slug() {
64 + local repo_slug="${1:-}"
65 +
66 + repo_slug="${repo_slug%/}"
67 + repo_slug="${repo_slug%.git}"
68 + repo_slug="${repo_slug#https://github.com/}"
69 + repo_slug="${repo_slug#http://github.com/}"
70 + repo_slug="${repo_slug#ssh://git@github.com/}"
71 + repo_slug="${repo_slug#git@github.com:}"
72 + repo_slug="${repo_slug#github.com/}"
73 + printf '%s\n' "${repo_slug}"
74 }
75
66 -resolve_image_digest() {
67 - local image_ref="$1"
68 -
69 - docker buildx imagetools inspect "${image_ref}" 2>/dev/null \
70 - | sed -n -E 's/^Digest:[[:space:]]+//p' \
71 - | head -1 || true
72 -}
73 -
74 -resolve_rocm_base_arg_value() {
75 - local arg_name="$1"
76 - local use_sccache="$2"
77 -
78 - case "${arg_name}" in
79 - USE_SCCACHE)
80 - printf '%s\n' "${use_sccache}"
81 - ;;
82 - *)
83 - extract_arg_default "${arg_name}"
84 - ;;
85 - esac
76 +is_trusted_main_build() {
77 + local actual_repo=""
78 + local trusted_repo=""
79 +
80 + [[ "${BUILDKITE:-false}" == "true" ]] || return 1
81 + [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" ]] || return 1
82 + [[ "${BUILDKITE_BRANCH:-}" == "${ROCM_BASE_STABLE_BRANCH:-main}" ]] || return 1
83 + actual_repo=$(normalize_repo_slug "${BUILDKITE_REPO:-}")
84 + trusted_repo=$(normalize_repo_slug \
85 + "${ROCM_BASE_STABLE_REPO_SLUG:-vllm-project/vllm}")
86 + [[ -n "${actual_repo}" && "${actual_repo}" == "${trusted_repo}" ]]
87 }
88
88 -hash_rocm_base_arg_values() {
89 - local use_sccache="$1"
90 - local base_image_digest="$2"
91 - local arg_name=""
92 - local arg_value=""
93 - shift 2 || true
94 -
95 - for arg_name in "$@"; do
96 - [[ -n "${arg_name}" ]] || continue
97 - arg_value=$(resolve_rocm_base_arg_value "${arg_name}" "${use_sccache}")
98 - printf 'arg:%s=%s\n' "${arg_name}" "${arg_value:-<empty>}"
99 - if [[ "${arg_name}" == "BASE_IMAGE" && -n "${arg_value}" ]]; then
100 - printf 'arg:%s.digest=%s\n' "${arg_name}" "${base_image_digest:-unknown}"
101 - fi
102 - done
103 -}
89 +rocm_base_layer_cache_scope() {
90 + local pull_request="${BUILDKITE_PULL_REQUEST:-false}"
91 + local branch="${BUILDKITE_PULL_REQUEST_HEAD_BRANCH:-${BUILDKITE_BRANCH:-local}}"
92 + local identity=""
93 + local repo_slug=""
94
105 -rocm_version_from_base_image() {
106 - local base_image="$1"
107 - local version=""
95 + if is_trusted_main_build; then
96 + printf 'main\n'
97 + return 0
98 + fi
99 + if [[ "${pull_request}" != "false" && -n "${pull_request}" ]]; then
100 + repo_slug=$(normalize_repo_slug "${BUILDKITE_REPO:-local}")
101 + identity=$(printf '%s\n' "${repo_slug:-local}" | sha256sum | cut -c1-12)
102 + printf 'pr-%s-%s\n' \
103 + "$(tag_component "${pull_request}" 32)" "${identity}"
104 + return 0
105 + fi
106
109 - version="$(sed -n -E 's/.*:([0-9]+\.[0-9]+(\.[0-9]+)?)-.*/\1/p' <<<"${base_image}")"
110 - tag_component "${version:-${base_image}}" 16
107 + identity=$(printf '%s\n%s\n' "${BUILDKITE_REPO:-local}" "${branch}" \
108 + | sha256sum | cut -c1-12)
109 + printf 'preview-%s-%s\n' "$(tag_component "${branch}" 24)" "${identity}"
110 }
111
113 -git_diff_changed_base() {
114 - local range="$1"
115 - [[ -n "$(git diff --name-only "${range}" -- "${DOCKERFILE}" 2>/dev/null)" ]]
116 -}
112 +configure_rocm_base_layer_cache() {
113 + local scope=""
114
118 -short_git_ref() {
119 - local ref="$1"
115 + ROCM_BASE_CACHE_ARGS=()
116 + if [[ "${ROCM_BASE_NO_CACHE:-0}" == "1" ]]; then
117 + ROCM_BASE_CACHE_ARGS+=(--no-cache)
118 + ROCM_BASE_LAYER_CACHE_REF="disabled"
119 + return 0
120 + fi
121
121 - git rev-parse --short "${ref}" 2>/dev/null || printf '%s\n' "${ref}"
122 + scope=$(rocm_base_layer_cache_scope)
123 + ROCM_BASE_LAYER_CACHE_REF="${CACHE_REPO}:rocm-base-${scope}"
124 + ROCM_BASE_CACHE_ARGS+=(
125 + --cache-from "type=registry,ref=${ROCM_BASE_LAYER_CACHE_REF}"
126 + )
127 + if [[ "${ROCM_BASE_LAYER_CACHE_REF}" != \
128 + "${ROCM_BASE_TRUSTED_LAYER_CACHE_REF}" ]]; then
129 + ROCM_BASE_CACHE_ARGS+=(
130 + --cache-from "type=registry,ref=${ROCM_BASE_TRUSTED_LAYER_CACHE_REF}"
131 + )
132 + fi
133 + ROCM_BASE_CACHE_ARGS+=(
134 + --cache-to \
135 + "type=registry,ref=${ROCM_BASE_LAYER_CACHE_REF},mode=max,ignore-error=true"
136 + )
137 }
138
124 -extract_arg_default_from_ref() {
125 - local ref="$1"
126 - local arg_name="$2"
127 - local content=""
139 +extract_arg_default() {
140 + local arg_name="$1"
141
129 - content="$(git show "${ref}:${DOCKERFILE}" 2>/dev/null || true)"
142 sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
131 - <<<"${content}" | head -1
143 + "${DOCKERFILE}" | head -1
144 }
145
134 -log_arg_default_changes() {
135 - local old_ref="$1"
136 - local new_ref="$2"
137 - local content_args="${ROCM_BASE_CONTENT_ARGS:-${DEFAULT_ROCM_BASE_CONTENT_ARGS}}"
138 - local arg_name=""
139 - local old_value=""
140 - local new_value=""
141 - local changed=0
142 -
143 - echo "Changed ROCm base ARG defaults:"
144 - for arg_name in ${content_args}; do
145 - old_value="$(extract_arg_default_from_ref "${old_ref}" "${arg_name}")"
146 - new_value="$(extract_arg_default_from_ref "${new_ref}" "${arg_name}")"
147 - if [[ "${old_value}" != "${new_value}" ]]; then
148 - echo " - ${arg_name}: ${old_value:-<unset>} -> ${new_value:-<unset>}"
149 - changed=1
150 - fi
151 - done
152 -
153 - if [[ "${changed}" == "0" ]]; then
154 - echo " - none detected; Dockerfile instructions changed outside tracked ARG defaults"
146 +resolve_image_digest() {
147 + local image_ref="$1"
148 + local attempts="${ROCM_IMAGE_DIGEST_ATTEMPTS:-4}"
149 + local delay_secs="${ROCM_IMAGE_DIGEST_RETRY_DELAY:-2}"
150 + local attempt=0
151 + local output=""
152 + local digest=""
153 + local status=0
154 +
155 + if [[ "${image_ref}" =~ @(sha256:[0-9a-f]{64})$ ]]; then
156 + printf '%s\n' "${BASH_REMATCH[1]}"
157 + return 0
158 + fi
159 + if [[ ! "${attempts}" =~ ^[1-9][0-9]*$ \
160 + || ! "${delay_secs}" =~ ^[0-9]+$ ]]; then
161 + echo "Invalid image digest retry configuration" >&2
162 + return 1
163 fi
156 -}
157 -
158 -log_arg_line_diff() {
159 - local range="$1"
160 - local arg_diff=""
164
162 - arg_diff="$(
163 - git diff --unified=0 "${range}" -- "${DOCKERFILE}" 2>/dev/null \
164 - | awk '/^[+-][[:space:]]*ARG[[:space:]]/ && $0 !~ /^(---|\+\+\+)/ { print " " $0 }' \
165 - || true
166 - )"
165 + for ((attempt = 1; attempt <= attempts; attempt++)); do
166 + status=0
167 + output=$(docker buildx imagetools inspect "${image_ref}" 2>&1) || status=$?
168 + digest=$(awk '$1 == "Digest:" { print $2; exit }' <<< "${output}")
169 + if ((status == 0)) && [[ "${digest}" =~ ^sha256:[0-9a-f]{64}$ ]]; then
170 + printf '%s\n' "${digest}"
171 + return 0
172 + fi
173 + if ((attempt < attempts)); then
174 + printf \
175 + 'Image digest lookup failed for %s (%d/%d, status %d); retrying\n' \
176 + "${image_ref}" "${attempt}" "${attempts}" "${status}" >&2
177 + sleep "${delay_secs}"
178 + fi
179 + done
180
168 - if [[ -n "${arg_diff}" ]]; then
169 - echo "Changed Dockerfile ARG lines:"
170 - printf '%s\n' "${arg_diff}"
171 - fi
181 + printf 'Failed to resolve digest for %s (status %d)\n%s\n' \
182 + "${image_ref}" "${status}" "${output:-<no output>}" >&2
183 + return 1
184 }
185
174 -log_rocm_base_change_check() {
175 - local context="$1"
176 - local range="$2"
177 - local old_ref="$3"
178 - local old_short=""
179 - local head_short=""
180 -
181 - old_short="$(short_git_ref "${old_ref}")"
182 - head_short="$(short_git_ref HEAD)"
186 +trusted_base_content_ref() {
187 + local base_hash="$1"
188 + local metadata_version="$2"
189
184 - echo "--- :mag: ROCm base refresh check"
185 - echo "Context: ${context}"
186 - echo "Dockerfile: ${DOCKERFILE}"
187 - echo "Base revision: ${old_short}"
188 - echo "Head revision: ${head_short}"
189 - echo "Git diff range: ${range}"
190 + printf '%s:base-v%s-%s\n' "${BASE_REPO}" "${metadata_version}" "${base_hash}"
191 }
192
192 -log_rocm_base_rebuild_reason() {
193 - local context="$1"
194 - local range="$2"
195 - local old_ref="$3"
196 - local changed_files=""
197 -
198 - log_rocm_base_change_check "${context}" "${range}" "${old_ref}"
193 +scoped_base_content_ref() {
194 + local base_hash="$1"
195 + local metadata_version="$2"
196
200 - changed_files="$(git diff --name-only "${range}" -- "${DOCKERFILE}" 2>/dev/null || true)"
201 - echo "Changed files:"
202 - if [[ -n "${changed_files}" ]]; then
203 - sed 's/^/ - /' <<<"${changed_files}"
204 - else
205 - echo " - ${DOCKERFILE}"
206 - fi
207 - log_arg_default_changes "${old_ref}" HEAD
208 - log_arg_line_diff "${range}"
209 - echo "Decision: rebuilding ROCm base image because ${DOCKERFILE} changed."
197 + printf '%s:base-v%s-preview-%s\n' \
198 + "${BASE_REPO}" "${metadata_version}" "${base_hash}"
199 }
200
212 -rocm_base_changed_in_range() {
213 - local context="$1"
214 - local range="$2"
215 - local old_ref="$3"
216 -
217 - if git_diff_changed_base "${range}"; then
218 - log_rocm_base_rebuild_reason "${context}" "${range}" "${old_ref}"
219 - return 0
201 +find_matching_base_content_ref() {
202 + local expected_hash="$1"
203 + local expected_version="$2"
204 + local image_ref=""
205 + local digest=""
206 + local immutable_ref=""
207 + local label_values=""
208 + local remote_hash=""
209 + local remote_version=""
210 + local attempts="${ROCM_IMAGE_DIGEST_ATTEMPTS:-4}"
211 + local delay_secs="${ROCM_IMAGE_DIGEST_RETRY_DELAY:-2}"
212 + local attempt=0
213 + local inspect_status=0
214 + shift 2
215 +
216 + if [[ ! "${attempts}" =~ ^[1-9][0-9]*$ \
217 + || ! "${delay_secs}" =~ ^[0-9]+$ ]]; then
218 + echo "Invalid image identity retry configuration" >&2
219 + return 1
220 fi
221
222 - log_rocm_base_change_check "${context}" "${range}" "${old_ref}"
223 - echo "Decision: ROCm base refresh not required; ${DOCKERFILE} is unchanged."
222 + for image_ref in "$@"; do
223 + if ! digest=$(resolve_image_digest "${image_ref}"); then
224 + continue
225 + fi
226 + immutable_ref="${image_ref}@${digest}"
227 + label_values=""
228 + for ((attempt = 1; attempt <= attempts; attempt++)); do
229 + inspect_status=0
230 + label_values=$(docker buildx imagetools inspect "${immutable_ref}" \
231 + --format '{{ index .Image.Config.Labels "vllm.rocm_base.content_hash" }} {{ index .Image.Config.Labels "vllm.rocm_base.metadata_version" }}' \
232 + 2>/dev/null) || inspect_status=$?
233 + remote_hash=""
234 + remote_version=""
235 + if ((inspect_status == 0)) && [[ -n "${label_values}" ]]; then
236 + read -r remote_hash remote_version _ <<< "${label_values}"
237 + if [[ "${remote_hash}" == "${expected_hash}" \
238 + && "${remote_version}" == "${expected_version}" ]]; then
239 + printf '%s\n' "${immutable_ref}"
240 + return 0
241 + fi
242 + fi
243 + if ((attempt < attempts)); then
244 + printf \
245 + 'ROCm base identity lookup incomplete or mismatched for %s (%d/%d); retrying\n' \
246 + "${immutable_ref}" "${attempt}" "${attempts}" >&2
247 + sleep "${delay_secs}"
248 + fi
249 + done
250 + if ((inspect_status == 0)) \
251 + && [[ -n "${remote_hash}" && -n "${remote_version}" ]]; then
252 + echo "ROCm base content ref has unexpected identity: ${immutable_ref}" >&2
253 + echo " expected hash/version: ${expected_hash} / ${expected_version}" >&2
254 + echo " found hash/version: ${remote_hash} / ${remote_version}" >&2
255 + # A content-addressed ref with the wrong identity is corrupt. The
256 + # stable alias is allowed to describe older content and is only a
257 + # migration/recovery candidate.
258 + if [[ "${image_ref}" != "${BASE_REPO}:base" ]]; then
259 + return 2
260 + fi
261 + fi
262 + echo "Failed to read complete ROCm base identity labels: ${immutable_ref}" >&2
263 + done
264 return 1
265 }
266
227 -rocm_base_changed() {
228 - local base_branch="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}"
229 - local base_ref="refs/remotes/origin/${base_branch}"
230 - local merge_base=""
231 -
232 - if [[ "${ROCM_BASE_REFRESH_SKIP:-0}" == "1" ]]; then
233 - echo "ROCM_BASE_REFRESH_SKIP=1 set; skipping ROCm base refresh"
234 - return 1
235 - fi
267 +rocm_version_from_base_image() {
268 + local base_image="$1"
269 + local version=""
270
237 - if [[ "${ROCM_BASE_REFRESH_FORCE:-0}" == "1" ]]; then
238 - echo "ROCM_BASE_REFRESH_FORCE=1 set; refreshing ROCm base image"
239 - return 0
240 - fi
271 + version="$(sed -n -E 's/.*:([0-9]+\.[0-9]+(\.[0-9]+)?)-.*/\1/p' <<<"${base_image}")"
272 + tag_component "${version:-${base_image}}" 16
273 +}
274
242 - if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
243 - echo "Not in a git checkout; skipping ROCm base refresh unless forced"
275 +should_push_stable_tag() {
276 + if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then
277 return 1
278 fi
279
247 - if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then
248 - git fetch --no-tags --depth=200 origin \
249 - "+refs/heads/${base_branch}:${base_ref}" >/dev/null 2>&1 || true
250 - merge_base=$(git merge-base HEAD "${base_ref}" 2>/dev/null || true)
251 - if [[ -z "${merge_base}" ]]; then
252 - echo "Unable to determine merge base with PR base ${base_ref}; skipping ROCm base refresh unless forced"
253 - return 1
254 - fi
255 - if rocm_base_changed_in_range \
256 - "pull request build against ${base_ref}" \
257 - "${merge_base}...HEAD" \
258 - "${merge_base}"; then
259 - return 0
260 - fi
261 - elif [[ "${BUILDKITE_BRANCH:-}" == "${ROCM_BASE_STABLE_BRANCH:-main}" ]] \
262 - && git rev-parse --verify HEAD~1 >/dev/null 2>&1; then
263 - if rocm_base_changed_in_range \
264 - "stable branch build; comparing against previous ${ROCM_BASE_STABLE_BRANCH:-main} commit" \
265 - "HEAD~1..HEAD" \
266 - "HEAD~1"; then
267 - return 0
268 - fi
269 - else
270 - git fetch --no-tags --depth=200 origin \
271 - "+refs/heads/${base_branch}:${base_ref}" >/dev/null 2>&1 || true
272 - merge_base=$(git merge-base HEAD "${base_ref}" 2>/dev/null || true)
273 - if [[ -z "${merge_base}" ]]; then
274 - echo "Unable to determine merge base with branch base ${base_ref}; skipping ROCm base refresh unless forced"
275 - return 1
276 - fi
277 - if rocm_base_changed_in_range \
278 - "branch build against ${base_ref}" \
279 - "${merge_base}...HEAD" \
280 - "${merge_base}"; then
281 - return 0
282 - fi
280 + if [[ "${ROCM_BASE_PUSH_STABLE_TAG:-}" == "0" ]]; then
281 + return 1
282 fi
283
285 - return 1
284 + is_trusted_main_build
285 }
286
288 -should_push_stable_tag() {
289 - if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then
287 +trusted_main_tip_matches_build() {
288 + local branch="${ROCM_BASE_STABLE_BRANCH:-main}"
289 + local build_commit="${BUILDKITE_COMMIT:-}"
290 + local remote_tip=""
291 +
292 + is_trusted_main_build || return 1
293 + if [[ ! "${build_commit}" =~ ^[0-9a-fA-F]{40}$ ]]; then
294 + echo "Skipping ROCm stable tag: Buildkite commit is missing or invalid" >&2
295 return 1
296 fi
292 -
293 - if [[ "${ROCM_BASE_PUSH_STABLE_TAG:-}" == "1" ]]; then
294 - return 0
297 + remote_tip=$(git ls-remote --exit-code "${BUILDKITE_REPO}" \
298 + "refs/heads/${branch}" 2>/dev/null | awk 'NR == 1 { print $1 }')
299 + if [[ ! "${remote_tip}" =~ ^[0-9a-fA-F]{40}$ ]]; then
300 + echo "Skipping ROCm stable tag: could not resolve remote ${branch} tip" >&2
301 + return 1
302 fi
296 - if [[ "${ROCM_BASE_PUSH_STABLE_TAG:-}" == "0" ]]; then
303 + if [[ "${remote_tip,,}" != "${build_commit,,}" ]]; then
304 + echo "Skipping ROCm stable tag: ${branch} changed after ${build_commit} (${remote_tip})" >&2
305 return 1
306 fi
307 + return 0
308 +}
309
300 - [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" \
301 - && "${BUILDKITE_BRANCH:-}" == "${ROCM_BASE_STABLE_BRANCH:-main}" ]]
310 +promote_stable_base_tag() {
311 + local source_ref="$1"
312 + local stable_tag="$2"
313 +
314 + ROCM_BASE_STABLE_TAG_UPDATED=0
315 + should_push_stable_tag && trusted_main_tip_matches_build || return 0
316 +
317 + docker buildx imagetools create --prefer-index=false \
318 + -t "${stable_tag}" "${source_ref}"
319 + ROCM_BASE_STABLE_TAG_UPDATED=1
320 }
321
322 setup_builder() {
@@ -315,29 +333,43 @@ compute_base_content_hash() {
333 local use_sccache="$1"
334 local base_image_digest="$2"
335 local content_files="${ROCM_BASE_CONTENT_FILES:-${DEFAULT_ROCM_BASE_CONTENT_FILES}}"
318 - local content_args="${ROCM_BASE_CONTENT_ARGS:-${DEFAULT_ROCM_BASE_CONTENT_ARGS}}"
336 + local sccache_arg=""
337 + local sccache_value=""
338 local -a content_paths=()
320 - local -a content_arg_names=()
339
340 read -r -a content_paths <<< "${content_files}"
323 - read -r -a content_arg_names <<< "${content_args}"
341
342 {
343 printf 'content-files-hash:%s\n' "$(compute_content_hash "${content_paths[@]}")"
344 printf 'dockerfile:%s\n' "${DOCKERFILE}"
328 - printf 'resolved-build-args:\n'
329 - hash_rocm_base_arg_values \
330 - "${use_sccache}" "${base_image_digest}" "${content_arg_names[@]}"
345 + printf 'base-image-digest:%s\n' "${base_image_digest}"
346 + printf 'use-sccache:%s\n' "${use_sccache}"
347 + if [[ "${use_sccache}" == "1" ]]; then
348 + # These values can change the installed binary or final image
349 + # configuration. SCCACHE_ENDPOINT is transport-only and therefore
350 + # deliberately excluded from image identity.
351 + for sccache_arg in \
352 + SCCACHE_DOWNLOAD_URL \
353 + SCCACHE_BUCKET_NAME \
354 + SCCACHE_REGION_NAME \
355 + SCCACHE_S3_NO_CREDENTIALS; do
356 + sccache_value="${!sccache_arg:-}"
357 + if [[ -z "${sccache_value}" ]]; then
358 + sccache_value=$(extract_arg_default "${sccache_arg}")
359 + fi
360 + printf 'arg:%s=%s\n' \
361 + "${sccache_arg}" "${sccache_value:-<empty>}"
362 + done
363 + fi
364 } | sha256sum | cut -d' ' -f1
365 }
366
367 build_base_image() {
368 local use_sccache="${ROCM_BASE_USE_SCCACHE:-${USE_SCCACHE:-0}}"
369 local base_hash=""
337 - local build_date=""
338 - local build_suffix=""
370 local base_image_arg=""
371 local base_image_digest=""
372 + local pinned_base_image=""
373 local rocm_version=""
374 local triton_arg=""
375 local pytorch_arg=""
@@ -348,20 +380,19 @@ build_base_image() {
380 local mori_arg=""
381 local python_version_arg=""
382 local pytorch_rocm_arch_arg=""
351 - local pytorch_branch=""
352 - local aiter_branch=""
383 local dependency_summary=""
354 - local descriptor=""
355 - local ci_descriptor=""
356 - local descriptive_tag=""
384 local stable_tag="${BASE_REPO}:base"
358 - local ci_descriptive_tag=""
385 + local trusted_content_tag=""
386 + local scoped_content_tag=""
387 + local writable_content_tag=""
388 + local build_ref=""
389 + local immutable_ref=""
390 + local selected_digest=""
391 + local stable_digest=""
392 + local reuse_status=0
393 local content_files="${ROCM_BASE_CONTENT_FILES:-${DEFAULT_ROCM_BASE_CONTENT_FILES}}"
360 - local content_args="${ROCM_BASE_CONTENT_ARGS:-${DEFAULT_ROCM_BASE_CONTENT_ARGS}}"
394 local content_files_hash=""
395 local metadata_version="${ROCM_BASE_METADATA_VERSION:-${DEFAULT_ROCM_BASE_METADATA_VERSION}}"
363 - local -a tags=()
364 - local -a no_cache_args=()
396 local -a sccache_args=()
397 local -a content_paths=()
398
@@ -369,13 +400,21 @@ build_base_image() {
400 echo "Error: ROCm base Dockerfile not found: ${DOCKERFILE}" >&2
401 exit 1
402 fi
372 -
373 - build_date="${ROCM_BASE_TAG_DATE:-$(date -u +%Y%m%d)}"
374 - if [[ -n "${BUILDKITE_BUILD_NUMBER:-}" ]]; then
375 - build_suffix="_bk_${BUILDKITE_BUILD_NUMBER}"
403 + if [[ ! "${metadata_version}" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,15}$ ]]; then
404 + echo "Invalid ROCm base metadata version: ${metadata_version}" >&2
405 + return 1
406 + fi
407 + if [[ "${use_sccache}" != "0" && "${use_sccache}" != "1" ]]; then
408 + echo "ROCm base USE_SCCACHE must be 0 or 1: ${use_sccache}" >&2
409 + return 1
410 fi
411 +
412 base_image_arg="$(extract_arg_default BASE_IMAGE)"
378 - base_image_digest="$(resolve_image_digest "${base_image_arg}")"
413 + if ! base_image_digest=$(resolve_image_digest "${base_image_arg}"); then
414 + echo "Failed to resolve ROCm base input image: ${base_image_arg}" >&2
415 + return 1
416 + fi
417 + pinned_base_image="${base_image_arg%@*}@${base_image_digest}"
418 read -r -a content_paths <<< "${content_files}"
419 content_files_hash="$(compute_content_hash "${content_paths[@]}")"
420 base_hash=$(compute_base_content_hash "${use_sccache}" "${base_image_digest}")
@@ -389,125 +428,161 @@ build_base_image() {
428 mori_arg="$(extract_arg_default MORI_BRANCH)"
429 python_version_arg="$(extract_arg_default PYTHON_VERSION)"
430 pytorch_rocm_arch_arg="$(extract_arg_default PYTORCH_ROCM_ARCH)"
392 - pytorch_branch="$(tag_component "${pytorch_arg}" 16)"
393 - aiter_branch="$(tag_component "${aiter_arg}" 24)"
431 dependency_summary="base=${base_image_arg},rocm=${rocm_version},python=${python_version_arg},pytorch=${pytorch_arg},torchvision=${pytorch_vision_arg},torchaudio=${pytorch_audio_arg},triton=${triton_arg},flash-attn=${fa_arg},aiter=${aiter_arg},mori=${mori_arg},pytorch-rocm-arch=${pytorch_rocm_arch_arg}"
395 - descriptor="$(clean_docker_tag "base_custom_aiter_${aiter_branch}_torch_${pytorch_branch}_${build_date}${build_suffix}")"
396 - ci_descriptor="$(clean_docker_tag "ci_custom_aiter_${aiter_branch}_torch_${pytorch_branch}_${build_date}${build_suffix}")"
397 -
398 - descriptive_tag="${BASE_REPO}:${descriptor}"
399 - ci_descriptive_tag="${CI_IMAGE_REPO}:${ci_descriptor}"
400 -
401 - tags=(-t "${descriptive_tag}")
402 - if should_push_stable_tag; then
403 - tags+=(-t "${stable_tag}")
404 - metadata_set "rocm-base-push-stable-tag" "1"
432 + trusted_content_tag=$(trusted_base_content_ref "${base_hash}" "${metadata_version}")
433 + scoped_content_tag=$(scoped_base_content_ref "${base_hash}" "${metadata_version}")
434 + # Preview writes share an exact-content namespace so stacked PRs can reuse
435 + # identical images. They may import the trusted ref, but never overwrite it.
436 + if is_trusted_main_build; then
437 + scoped_content_tag="${trusted_content_tag}"
438 + writable_content_tag="${trusted_content_tag}"
439 else
406 - metadata_set "rocm-base-push-stable-tag" "0"
440 + writable_content_tag="${scoped_content_tag}"
441 fi
442
409 - if [[ "${ROCM_BASE_NO_CACHE:-1}" == "1" ]]; then
410 - no_cache_args=(--no-cache)
443 + configure_rocm_base_layer_cache
444 +
445 + if [[ "${use_sccache}" == "1" ]]; then
446 + for env_name in \
447 + SCCACHE_DOWNLOAD_URL \
448 + SCCACHE_ENDPOINT \
449 + SCCACHE_BUCKET_NAME \
450 + SCCACHE_REGION_NAME \
451 + SCCACHE_S3_NO_CREDENTIALS; do
452 + if [[ -n "${!env_name:-}" ]]; then
453 + sccache_args+=(--build-arg "${env_name}=${!env_name}")
454 + fi
455 + done
456 fi
457
413 - for env_name in \
414 - SCCACHE_DOWNLOAD_URL \
415 - SCCACHE_ENDPOINT \
416 - SCCACHE_BUCKET_NAME \
417 - SCCACHE_REGION_NAME \
418 - SCCACHE_S3_NO_CREDENTIALS; do
419 - if [[ -n "${!env_name:-}" ]]; then
420 - sccache_args+=(--build-arg "${env_name}=${!env_name}")
421 - fi
422 - done
423 -
424 - echo "--- :docker: Building ROCm base image"
458 + echo "--- :docker: Preparing ROCm base image"
459 echo "Dockerfile: ${DOCKERFILE}"
426 - echo "Descriptive tag: ${descriptive_tag}"
460 + echo "Trusted content tag: ${trusted_content_tag}"
461 + echo "Writable content tag: ${writable_content_tag}"
462 echo "Stable tag: ${stable_tag} ($(should_push_stable_tag && echo enabled || echo disabled))"
463 echo "Content hash: ${base_hash}"
464 echo "Dependency summary: ${dependency_summary}"
465 echo "USE_SCCACHE: ${use_sccache}"
466 + echo "BuildKit layer cache: ${ROCM_BASE_LAYER_CACHE_REF}"
467 +
468 + if [[ "${ROCM_BASE_REFRESH_FORCE:-0}" != "1" \
469 + && "${ROCM_BASE_NO_CACHE:-0}" != "1" ]]; then
470 + reuse_status=0
471 + if [[ "${trusted_content_tag}" == "${scoped_content_tag}" ]]; then
472 + immutable_ref=$(find_matching_base_content_ref \
473 + "${base_hash}" "${metadata_version}" \
474 + "${trusted_content_tag}" "${stable_tag}") || reuse_status=$?
475 + else
476 + immutable_ref=$(find_matching_base_content_ref \
477 + "${base_hash}" "${metadata_version}" \
478 + "${trusted_content_tag}" "${scoped_content_tag}" \
479 + "${stable_tag}") \
480 + || reuse_status=$?
481 + fi
482 + else
483 + reuse_status=1
484 + echo "Forced/no-cache ROCm base build; bypassing content image reuse"
485 + fi
486
432 - docker buildx build \
433 - "${no_cache_args[@]}" \
434 - --pull \
435 - --progress "${BUILDKIT_PROGRESS:-plain}" \
436 - --file "${DOCKERFILE}" \
437 - --build-arg "USE_SCCACHE=${use_sccache}" \
438 - "${sccache_args[@]}" \
439 - --label "org.opencontainers.image.source=https://github.com/vllm-project/vllm" \
440 - --label "org.opencontainers.image.vendor=vLLM" \
441 - --label "org.opencontainers.image.title=vLLM ROCm base" \
442 - --label "org.opencontainers.image.revision=${BUILDKITE_COMMIT:-}" \
443 - --label "vllm.rocm_base.metadata_version=${metadata_version}" \
444 - --label "vllm.rocm_base.content_hash=${base_hash}" \
445 - --label "vllm.rocm_base.content_files_hash=${content_files_hash}" \
446 - --label "vllm.rocm_base.dockerfile=${DOCKERFILE}" \
447 - --label "vllm.rocm_base.image.descriptive=${descriptive_tag}" \
448 - --label "vllm.rocm_base.image.stable=${stable_tag}" \
449 - --label "vllm.rocm_base.git_commit=${BUILDKITE_COMMIT:-}" \
450 - --label "vllm.rocm_base.stable_branch=${ROCM_BASE_STABLE_BRANCH:-main}" \
451 - --label "vllm.rocm_base.descriptor=${descriptor}" \
452 - --label "vllm.rocm_base.dependency_summary=${dependency_summary}" \
453 - --label "vllm.rocm_base.base_image=${base_image_arg}" \
454 - --label "vllm.rocm_base.base_image_digest=${base_image_digest}" \
455 - --label "vllm.rocm_base.dependency.rocm=${rocm_version}" \
456 - --label "vllm.rocm_base.dependency.python=${python_version_arg}" \
457 - --label "vllm.rocm_base.dependency.pytorch=${pytorch_arg}" \
458 - --label "vllm.rocm_base.dependency.torchvision=${pytorch_vision_arg}" \
459 - --label "vllm.rocm_base.dependency.torchaudio=${pytorch_audio_arg}" \
460 - --label "vllm.rocm_base.dependency.triton=${triton_arg}" \
461 - --label "vllm.rocm_base.dependency.flash_attention=${fa_arg}" \
462 - --label "vllm.rocm_base.dependency.aiter=${aiter_arg}" \
463 - --label "vllm.rocm_base.dependency.mori=${mori_arg}" \
464 - --label "vllm.rocm_base.pytorch_rocm_arch=${pytorch_rocm_arch_arg}" \
465 - "${tags[@]}" \
466 - --push \
467 - .
468 -
469 - docker buildx imagetools inspect "${descriptive_tag}" >/dev/null
487 + if ((reuse_status > 1)); then
488 + return "${reuse_status}"
489 + fi
490
491 + if [[ ${reuse_status} -eq 0 && -n "${immutable_ref}" ]]; then
492 + echo "Reusing ROCm base image with matching content: ${immutable_ref}"
493 + if [[ "${immutable_ref%@*}" == "${stable_tag}" \
494 + && "${writable_content_tag}" != "${stable_tag}" ]]; then
495 + echo "Backfilling ROCm base content tag: ${writable_content_tag}"
496 + docker buildx imagetools create --prefer-index=false \
497 + -t "${writable_content_tag}" "${immutable_ref}"
498 + if ! immutable_ref=$(find_matching_base_content_ref \
499 + "${base_hash}" "${metadata_version}" \
500 + "${writable_content_tag}"); then
501 + echo "Backfilled ROCm base content tag failed identity validation" >&2
502 + return 1
503 + fi
504 + fi
505 + else
506 + build_ref="${writable_content_tag}"
507 + echo "No reusable ROCm base content image found; building ${build_ref}"
508 + setup_builder
509 + docker buildx build \
510 + "${ROCM_BASE_CACHE_ARGS[@]}" \
511 + --pull \
512 + --provenance=false \
513 + --progress "${BUILDKIT_PROGRESS:-plain}" \
514 + --file "${DOCKERFILE}" \
515 + --build-arg "BASE_IMAGE=${pinned_base_image}" \
516 + --build-arg "USE_SCCACHE=${use_sccache}" \
517 + "${sccache_args[@]}" \
518 + --label "org.opencontainers.image.source=https://github.com/vllm-project/vllm" \
519 + --label "org.opencontainers.image.vendor=vLLM" \
520 + --label "org.opencontainers.image.title=vLLM ROCm base" \
521 + --label "vllm.rocm_base.metadata_version=${metadata_version}" \
522 + --label "vllm.rocm_base.content_hash=${base_hash}" \
523 + --label "vllm.rocm_base.content_files_hash=${content_files_hash}" \
524 + --label "vllm.rocm_base.dockerfile=${DOCKERFILE}" \
525 + --label "vllm.rocm_base.dependency_summary=${dependency_summary}" \
526 + --label "vllm.rocm_base.base_image=${pinned_base_image}" \
527 + --label "vllm.rocm_base.base_image_digest=${base_image_digest}" \
528 + --label "vllm.rocm_base.dependency.rocm=${rocm_version}" \
529 + --label "vllm.rocm_base.dependency.python=${python_version_arg}" \
530 + --label "vllm.rocm_base.dependency.pytorch=${pytorch_arg}" \
531 + --label "vllm.rocm_base.dependency.torchvision=${pytorch_vision_arg}" \
532 + --label "vllm.rocm_base.dependency.torchaudio=${pytorch_audio_arg}" \
533 + --label "vllm.rocm_base.dependency.triton=${triton_arg}" \
534 + --label "vllm.rocm_base.dependency.flash_attention=${fa_arg}" \
535 + --label "vllm.rocm_base.dependency.aiter=${aiter_arg}" \
536 + --label "vllm.rocm_base.dependency.mori=${mori_arg}" \
537 + --label "vllm.rocm_base.pytorch_rocm_arch=${pytorch_rocm_arch_arg}" \
538 + -t "${build_ref}" \
539 + --push \
540 + .
541 + if ! immutable_ref=$(find_matching_base_content_ref \
542 + "${base_hash}" "${metadata_version}" "${build_ref}"); then
543 + echo "Published ROCm base image failed identity validation: ${build_ref}" >&2
544 + return 1
545 + fi
546 + fi
547 +
548 + selected_digest="${immutable_ref##*@}"
549 + metadata_set "rocm-base-image" "${immutable_ref}"
550 + metadata_set "rocm-base-push-stable-tag" "0"
551 + stable_digest=$(resolve_image_digest "${stable_tag}" 2>/dev/null || true)
552 + if [[ -n "${stable_digest}" && "${stable_digest}" == "${selected_digest}" ]]; then
553 + echo "--- :white_check_mark: Existing ROCm base is current"
554 + echo "Stable ${stable_tag} already resolves to ${selected_digest}"
555 + return 0
556 + fi
557 +
558 + promote_stable_base_tag "${immutable_ref}" "${stable_tag}"
559 +
560 + metadata_set "rocm-base-push-stable-tag" "${ROCM_BASE_STABLE_TAG_UPDATED}"
561 metadata_set "rocm-base-refresh" "1"
472 - metadata_set "rocm-base-image" "${descriptive_tag}"
473 - metadata_set "rocm-base-image-descriptive" "${descriptive_tag}"
474 - metadata_set "rocm-base-image-stable" "${stable_tag}"
475 - metadata_set "rocm-base-image-ci-descriptive" "${ci_descriptive_tag}"
476 - metadata_set "rocm-base-metadata-version" "${metadata_version}"
477 - metadata_set "rocm-base-content-hash" "${base_hash}"
478 - metadata_set "rocm-base-content-files-hash" "${content_files_hash}"
479 - metadata_set "rocm-base-content-files" "${content_files}"
480 - metadata_set "rocm-base-content-args" "${content_args}"
481 - metadata_set "rocm-base-base-image-digest" "${base_image_digest}"
482 - metadata_set "rocm-base-dockerfile" "${DOCKERFILE}"
483 - metadata_set "rocm-base-descriptor" "${descriptor}"
484 - metadata_set "rocm-base-dependency-summary" "${dependency_summary}"
485 - metadata_set "rocm-base-dependency-rocm" "${rocm_version}"
486 - metadata_set "rocm-base-dependency-python" "${python_version_arg}"
487 - metadata_set "rocm-base-dependency-pytorch" "${pytorch_arg}"
488 - metadata_set "rocm-base-dependency-torchvision" "${pytorch_vision_arg}"
489 - metadata_set "rocm-base-dependency-torchaudio" "${pytorch_audio_arg}"
490 - metadata_set "rocm-base-dependency-triton" "${triton_arg}"
491 - metadata_set "rocm-base-dependency-flash-attention" "${fa_arg}"
492 - metadata_set "rocm-base-dependency-aiter" "${aiter_arg}"
493 - metadata_set "rocm-base-dependency-mori" "${mori_arg}"
494 - metadata_set "rocm-base-pytorch-rocm-arch" "${pytorch_rocm_arch_arg}"
495 - metadata_set "rocm-ci-image-descriptive" "${ci_descriptive_tag}"
562
563 echo "--- :white_check_mark: ROCm base image published"
498 - echo "Use BASE_IMAGE=${descriptive_tag} for downstream ROCm CI builds"
564 + echo "Use BASE_IMAGE=${immutable_ref} for downstream ROCm CI builds"
565 }
566
567 main() {
568 metadata_set "rocm-base-refresh" "0"
569
504 - if ! rocm_base_changed; then
505 - echo "ROCm base Dockerfile did not change; skipping base image refresh"
570 + if [[ "${ROCM_BASE_REFRESH_SKIP:-0}" == "1" ]]; then
571 + local stable_ref="${BASE_REPO}:base"
572 + local stable_digest=""
573 +
574 + if ! stable_digest=$(resolve_image_digest "${stable_ref}"); then
575 + echo "Could not pin explicitly skipped ROCm base: ${stable_ref}" >&2
576 + return 1
577 + fi
578 + metadata_set "rocm-base-image" "${stable_ref}@${stable_digest}"
579 + metadata_set "rocm-base-push-stable-tag" "0"
580 + echo "ROCM_BASE_REFRESH_SKIP=1; pinned current stable base at ${stable_digest}"
581 return 0
582 fi
508 -
509 - setup_builder
583 build_base_image
584 }
585
513 -main "$@"
586 +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
587 + main "$@"
588 +fi
.buildkite/scripts/rocm/smoke-test-image.sh
+12 -1
@@ -3,7 +3,18 @@
3
4 set -euo pipefail
5
6 -image_ref="${VLLM_CI_SMOKE_IMAGE:-rocm/vllm-ci:${BUILDKITE_COMMIT:?BUILDKITE_COMMIT is required}}"
6 +if [[ "${ROCM_CI_ARTIFACT_ONLY:-0}" == "1" ]]; then
7 + base_refreshed=""
8 + if command -v buildkite-agent >/dev/null 2>&1; then
9 + base_refreshed="$(buildkite-agent meta-data get rocm-base-refresh 2>/dev/null || true)"
10 + fi
11 + if [[ "${base_refreshed}" != "1" ]]; then
12 + echo "ROCM_CI_ARTIFACT_ONLY=1; no full image was built, skipping smoke test"
13 + exit 0
14 + fi
15 +fi
16 +
17 +image_ref="${VLLM_CI_SMOKE_IMAGE:-${IMAGE_TAG:-rocm/vllm-ci:${BUILDKITE_COMMIT:?BUILDKITE_COMMIT is required}}}"
18
19 docker run --rm --network=none --entrypoint /bin/bash "${image_ref}" -ec '
20 if [ ! -d /vllm-workspace ]; then echo Missing directory: /vllm-workspace >&2; exit 1; fi
.dockerignore
+3
@@ -1,5 +1,8 @@
1 /.venv
2 /build
3 +/artifacts/vllm-rocm-install/
4 +/bake-config-build-*.json
5 +/wheel-export/
6 dist
7 vllm/*.so
8 vllm/vllm-rs
docker/Dockerfile.rocm
+23 -16
@@ -254,20 +254,31 @@ RUN --mount=type=cache,id=vllm-rocm-cargo-registry,target=/root/.cargo/registry,
254 fi
255
256 # -----------------------
257 -# vLLM native build stages
257 +# Shared vLLM build dependencies
258 #
259 -# csrc-build intentionally copies only files that affect ROCm native extension
260 -# compilation. That keeps unrelated CI/test/docs edits from invalidating the
261 -# expensive HIP/C++ build layer.
262 -FROM base AS csrc-build
259 +# Install requirements once, before bringing source-specific inputs into either
260 +# consumer. Native compilation and wheel packaging then share this expensive
261 +# layer while retaining their independent source cache boundaries.
262 +FROM base AS build_vllm_dependencies
263 ARG COMMON_WORKDIR
264 +ENV VLLM_TARGET_DEVICE=rocm
265 WORKDIR ${COMMON_WORKDIR}/vllm
266
266 -COPY requirements/rocm.txt requirements/rocm.txt
267 -COPY requirements/common.txt requirements/common.txt
267 +COPY --from=fetch_vllm \
268 + ${COMMON_WORKDIR}/vllm/requirements/common.txt \
269 + ${COMMON_WORKDIR}/vllm/requirements/rocm.txt \
270 + requirements/
271 +
272 RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
273 uv pip install --system -r requirements/rocm.txt
274
275 +# -----------------------
276 +# vLLM native build stages
277 +#
278 +# csrc-build intentionally copies only files that affect ROCm native extension
279 +# compilation. That keeps unrelated CI/test/docs edits from invalidating the
280 +# expensive HIP/C++ build layer.
281 +FROM build_vllm_dependencies AS csrc-build
282 # pyproject.toml is bind-mounted so it is not persisted in the csrc-build image,
283 # but it still participates in the BuildKit cache key for this RUN step.
284 COPY setup.py CMakeLists.txt ./
@@ -277,7 +288,6 @@ COPY csrc csrc/
288 COPY vllm/envs.py vllm/envs.py
289 COPY vllm/__init__.py vllm/__init__.py
290
280 -ENV VLLM_TARGET_DEVICE=rocm
291 ENV SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+rocm.csrc.build"
292
293 RUN --mount=type=bind,source=pyproject.toml,target=${COMMON_WORKDIR}/vllm/pyproject.toml \
@@ -296,11 +306,10 @@ RUN --mount=type=bind,source=pyproject.toml,target=${COMMON_WORKDIR}/vllm/pyproj
306
307 # Build the full vLLM ROCm wheel by reusing the native extension wheel from
308 # csrc-build. This stage still rebuilds for Python/package changes, but skips
299 -# the expensive HIP/C++ compile when native inputs are unchanged.
300 -FROM fetch_vllm AS build_vllm
301 -ARG COMMON_WORKDIR
302 -ENV VLLM_TARGET_DEVICE=rocm
303 -
309 +# the expensive dependency installation and HIP/C++ compilation when their
310 +# respective inputs are unchanged.
311 +FROM build_vllm_dependencies AS build_vllm
312 +COPY --from=fetch_vllm ${COMMON_WORKDIR}/vllm ${COMMON_WORKDIR}/vllm
313 COPY --from=csrc-build ${COMMON_WORKDIR}/vllm/dist /precompiled-wheels
314
315 # Drop the pre-built Rust artifacts into the source tree. setup.py detects
@@ -308,9 +317,7 @@ COPY --from=csrc-build ${COMMON_WORKDIR}/vllm/dist /precompiled-wheels
317 COPY --from=rust-build ${COMMON_WORKDIR}/vllm/vllm/vllm-rs ${COMMON_WORKDIR}/vllm/vllm/vllm-rs
318 COPY --from=rust-build ${COMMON_WORKDIR}/vllm/vllm/_rust_*.so ${COMMON_WORKDIR}/vllm/vllm/
319
311 -RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
312 - cd vllm \
313 - && uv pip install --system -r requirements/rocm.txt \
320 +RUN cd "${COMMON_WORKDIR}/vllm" \
321 && export VLLM_USE_PRECOMPILED=1 \
322 && export VLLM_PRECOMPILED_WHEEL_LOCATION="$(ls /precompiled-wheels/*.whl)" \
323 && export VLLM_DOCKER_BUILD_CONTEXT=1 \
docker/Dockerfile.rocm_base
+47 -16
@@ -134,11 +134,7 @@ RUN mkdir -p /app/install && cp /opt/rocm/share/amd_smi/dist/*.whl /app/install
134 ###
135 FROM base AS build_pytorch
136 ARG PYTORCH_BRANCH
137 -ARG PYTORCH_VISION_BRANCH
138 -ARG PYTORCH_AUDIO_BRANCH
137 ARG PYTORCH_REPO
140 -ARG PYTORCH_VISION_REPO
141 -ARG PYTORCH_AUDIO_REPO
138 ARG USE_SCCACHE
139
140 RUN apt-get update && apt-get install -y pkg-config liblzma-dev
@@ -156,8 +152,27 @@ RUN cd pytorch && python3 tools/amd_build/build_amd.py \
152 && sccache --show-stats; \
153 fi \
154 && CMAKE_PREFIX_PATH=$(python3 -c 'import sys; print(sys.prefix)') python3 setup.py bdist_wheel --dist-dir=dist \
159 - && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi \
160 - && pip install dist/*.whl
155 + && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi
156 +RUN mkdir -p /app/install && cp /app/pytorch/dist/*.whl /app/install
157 +
158 +
159 +###
160 +### Shared PyTorch runtime for dependent builds
161 +###
162 +FROM base AS build_pytorch_runtime
163 +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 130 --slave /usr/bin/g++ g++ /usr/bin/g++-13
164 +RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install,ro \
165 + pip install /install/*.whl
166 +
167 +
168 +###
169 +### TorchVision build
170 +###
171 +FROM build_pytorch_runtime AS build_torchvision
172 +ARG PYTORCH_VISION_BRANCH
173 +ARG PYTORCH_VISION_REPO
174 +ARG USE_SCCACHE
175 +
176 RUN git clone ${PYTORCH_VISION_REPO} vision
177 RUN cd vision && git checkout ${PYTORCH_VISION_BRANCH} \
178 && if [ "$USE_SCCACHE" = "1" ]; then \
@@ -166,23 +181,31 @@ RUN cd vision && git checkout ${PYTORCH_VISION_BRANCH} \
181 && export CMAKE_CXX_COMPILER_LAUNCHER=sccache; \
182 fi \
183 && python3 setup.py bdist_wheel --dist-dir=dist \
169 - && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi \
170 - && pip install dist/*.whl
184 + && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi
185 +RUN mkdir -p /app/install && cp /app/vision/dist/*.whl /app/install
186 +
187 +
188 +###
189 +### TorchAudio build
190 +###
191 +FROM build_pytorch_runtime AS build_torchaudio
192 +ARG PYTORCH_AUDIO_BRANCH
193 +ARG PYTORCH_AUDIO_REPO
194 +ARG USE_SCCACHE
195 +
196 RUN git clone ${PYTORCH_AUDIO_REPO} audio
197 RUN cd audio && git checkout ${PYTORCH_AUDIO_BRANCH} \
173 - && git submodule update --init --recursive \
174 - && pip install -r requirements.txt \
198 + && git submodule update --init --recursive
199 +RUN cd audio && pip install -r requirements.txt
200 +RUN cd audio \
201 && if [ "$USE_SCCACHE" = "1" ]; then \
202 export HIP_CLANG_PATH=/opt/sccache-wrappers \
203 && export CMAKE_C_COMPILER_LAUNCHER=sccache \
204 && export CMAKE_CXX_COMPILER_LAUNCHER=sccache; \
205 fi \
206 && python3 setup.py bdist_wheel --dist-dir=dist \
181 - && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi \
182 - && pip install dist/*.whl
183 -RUN mkdir -p /app/install && cp /app/pytorch/dist/*.whl /app/install \
184 - && cp /app/vision/dist/*.whl /app/install \
185 - && cp /app/audio/dist/*.whl /app/install
207 + && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi
208 +RUN mkdir -p /app/install && cp /app/audio/dist/*.whl /app/install
209
210
211 ###
@@ -263,6 +286,10 @@ RUN --mount=type=bind,from=build_amdsmi,src=/app/install/,target=/install \
286 cp /install/*.whl /app/debs
287 RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \
288 cp /install/*.whl /app/debs
289 +RUN --mount=type=bind,from=build_torchvision,src=/app/install/,target=/install \
290 + cp /install/*.whl /app/debs
291 +RUN --mount=type=bind,from=build_torchaudio,src=/app/install/,target=/install \
292 + cp /install/*.whl /app/debs
293 RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \
294 cp /install/*.whl /app/debs
295
@@ -277,6 +304,10 @@ RUN --mount=type=bind,from=build_amdsmi,src=/app/install/,target=/install \
304 cp /install/*.whl /app/debs
305 RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \
306 cp /install/*.whl /app/debs
307 +RUN --mount=type=bind,from=build_torchvision,src=/app/install/,target=/install \
308 + cp /install/*.whl /app/debs
309 +RUN --mount=type=bind,from=build_torchaudio,src=/app/install/,target=/install \
310 + cp /install/*.whl /app/debs
311 RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \
312 cp /install/*.whl /app/debs
313 RUN --mount=type=bind,from=build_mori,src=/app/install/,target=/install \
@@ -315,4 +346,4 @@ RUN echo "BASE_IMAGE: ${BASE_IMAGE}" > /app/versions.txt \
346 && echo "AITER_BRANCH: ${AITER_BRANCH}" >> /app/versions.txt \
347 && echo "AITER_REPO: ${AITER_REPO}" >> /app/versions.txt \
348 && echo "MORI_BRANCH: ${MORI_BRANCH}" >> /app/versions.txt \
318 - && echo "MORI_REPO: ${MORI_REPO}" >> /app/versions.txt
\ No newline at end of file
349 + && echo "MORI_REPO: ${MORI_REPO}" >> /app/versions.txt
docker/ci-rocm.hcl
+15 -24
@@ -338,33 +338,22 @@ group "test-rocm-ci-with-artifacts" {
338 targets = ["rust-rocm-ci", "csrc-rocm-ci", "export-wheel-rocm"]
339 }
340
341 -# Full test image + wheel export. Kept for fallback/debugging when a pushed
342 -# per-commit image is useful.
341 +# Full test image + wheel export. Kept for fallback/debugging when a pushed,
342 +# build-scoped image is useful.
343 group "test-rocm-ci-with-wheel" {
344 targets = ["rust-rocm-ci", "csrc-rocm-ci", "test-rocm-ci", "export-wheel-rocm"]
345 }
346
347 -# Image tags for the ci_base build. ci-bake-rocm.sh rewrites CI_BASE_IMAGE_TAG
348 -# to the primary tag for this build. Builds always publish a content-scoped tag
349 -# when the ci_base content hash is available. Builds with BUILDKITE_COMMIT also
350 -# publish a commit-scoped tag, either as the primary tag or an additional alias.
351 -# NIGHTLY=1 builds on the stable branch can additionally set
352 -# CI_BASE_IMAGE_TAG_STABLE to refresh rocm/vllm-dev:ci_base.
347 +# Primary output tag for the ci_base build. In Buildkite this is a unique,
348 +# build-scoped tag; ci-bake-rocm.sh validates it before creating content and
349 +# stable aliases.
350 variable "CI_BASE_IMAGE_TAG" {
351 default = "rocm/vllm-dev:ci_base"
352 }
353
357 -# Supplemental tags only. ci-bake-rocm.sh leaves these empty when the same ref
358 -# is already the primary CI_BASE_IMAGE_TAG.
359 -variable "CI_BASE_IMAGE_TAG_COMMIT_EXTRA" {
360 - default = ""
361 -}
362 -
363 -variable "CI_BASE_IMAGE_TAG_CONTENT_EXTRA" {
364 - default = ""
365 -}
366 -
367 -variable "CI_BASE_IMAGE_TAG_STABLE" {
354 +# Versioned, content-addressed trusted ref. Preview builds import it read-only;
355 +# the wrapper creates their source-scoped content alias after validation.
356 +variable "CI_BASE_TRUSTED_CONTENT_REF" {
357 default = ""
358 }
359
@@ -403,21 +392,23 @@ target "deepep-rocm-ci" {
392 # Uses inline cache metadata on the ci_base image itself instead of exporting a
393 # separate registry cache artifact.
394 target "ci-base-rocm-ci" {
406 - inherits = ["_common-rocm", "_ci-rocm", "_labels"]
395 + inherits = ["_common-rocm"]
396 target = "ci_base"
397 + args = {
398 + max_jobs = CI_MAX_JOBS
399 + }
400 cache-from = concat(
401 compact([
402 CI_BASE_IMAGE_TAG != "" ? "type=registry,ref=${CI_BASE_IMAGE_TAG}" : "",
411 - CI_BASE_IMAGE_TAG_COMMIT_EXTRA != "" ? "type=registry,ref=${CI_BASE_IMAGE_TAG_COMMIT_EXTRA}" : "",
412 - CI_BASE_IMAGE_TAG_CONTENT_EXTRA != "" ? "type=registry,ref=${CI_BASE_IMAGE_TAG_CONTENT_EXTRA}" : "",
413 - CI_BASE_IMAGE_TAG_STABLE != "" ? "type=registry,ref=${CI_BASE_IMAGE_TAG_STABLE}" : "",
403 + CI_BASE_TRUSTED_CONTENT_REF != "" ? "type=registry,ref=${CI_BASE_TRUSTED_CONTENT_REF}" : "",
404 ]),
405 # Import upstream dependency caches so NIXL/ROCShmem/DeepEP stages
406 # are cache hits even when ci_base itself needs rebuilding.
407 get_cache_from_rocm_deps(),
408 )
409 cache-to = ["type=inline"]
420 - tags = compact([CI_BASE_IMAGE_TAG, CI_BASE_IMAGE_TAG_COMMIT_EXTRA, CI_BASE_IMAGE_TAG_CONTENT_EXTRA, CI_BASE_IMAGE_TAG_STABLE])
410 + tags = compact([CI_BASE_IMAGE_TAG])
411 + attest = ["type=provenance,disabled=true"]
412 output = ["type=registry"]
413 }
414
tests/tools/test_docker_build_metadata_args.py
-4
@@ -171,10 +171,6 @@ def test_rocm_ci_base_metadata_inputs_cover_ci_base_files() -> None:
171 "requirements/common.txt",
172 "requirements/rocm.txt",
173 "requirements/test/rocm.txt",
174 - "docker/Dockerfile.rocm_base",
174 "docker/Dockerfile.rocm",
176 - "docker/ci-rocm.hcl",
177 - "docker/docker-bake-rocm.hcl",
178 - ".buildkite/scripts/ci-bake-rocm.sh",
175 ):
176 assert expected in ci_bake