@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
sh 2,784 lines 97 KB
Raw
1 #!/bin/bash
2 # ci-bake-rocm.sh - Docker buildx bake wrapper for ROCm CI builds.
3 #
4 # The wrapper keeps three build concerns separate:
5 # * ci_base builds are content-addressed by vllm.ci_base.content_hash.
6 # * test image contents record the commit while runtime tags stay build-scoped.
7 # * ROCm install artifacts are uploaded only for targets that export wheels.
8 #
9 # Usage:
10 # ci-bake-rocm.sh [TARGET]
11 #
12 # Set BAKE_PRINT_ONLY=1 to stop after docker buildx bake --print.
13
14 set -euo pipefail
15
16 DEFAULT_REPO_SLUG="vllm-project/vllm"
17 DEFAULT_CI_HCL_SOURCE="docker/ci-rocm.hcl"
18 DEFAULT_CI_BASE_CONTENT_FILES=".dockerignore requirements/common.txt requirements/rocm.txt requirements/test/rocm.txt tools/install_torchcodec_rocm.sh rust-toolchain.toml tests/vllm_test_utils"
19 DEFAULT_CI_BASE_DOCKERFILE="docker/Dockerfile.rocm"
20 DEFAULT_CI_BASE_DOCKERFILE_STAGES="base rust_toolchain_input_0 rust-toolchain-input rust-toolchain build_nixl lmcache_source build_lmcache build_rocshmem build_deepep mori_base ci_base"
21 DEFAULT_CI_BASE_METADATA_VERSION="3"
22 # ROCm CI forces REMOTE_VLLM=0, so content identity covers only the selected
23 # local-source stages rather than unreachable remote-fetch alternatives.
24 DEFAULT_ROCM_CSRC_CONTENT_FILES=".dockerignore requirements/common.txt requirements/rocm.txt pyproject.toml setup.py CMakeLists.txt cmake csrc vllm/envs.py vllm/__init__.py tools/build_rust.py"
25 DEFAULT_ROCM_CSRC_DOCKERFILE_STAGES="base fetch_vllm_0 fetch_vllm build_vllm_dependencies rocm-triton-kernels csrc-build"
26 DEFAULT_ROCM_RUST_CONTENT_FILES=".dockerignore .git_archival.txt pyproject.toml requirements/build/rust.txt rust/Cargo.lock rust/Cargo.toml rust/proto rust/src rust-toolchain.toml tools/build_rust.py tools/build_rust.sh"
27 DEFAULT_ROCM_RUST_DOCKERFILE_STAGES="base fetch_vllm_0 fetch_vllm vllm-version rust_toolchain_input_0 rust-toolchain-input rust_input_0 rust-input rust-toolchain rust-build"
28 # Docker's 128-character tag limit minus the longest cache prefix
29 # ("csrc-rocm-branch-" and "rust-rocm-branch-", both 17 characters).
30 ROCM_CACHE_BRANCH_TAG_MAX_LEN=111
31 CI_BASE_WRITE_SCOPE=""
32
33 TARGET=""
34 CI_HCL_SOURCE="${CI_HCL_SOURCE:-}"
35 CI_HCL_PATH=""
36 CI_BASE_LABEL_OVERRIDE_PATH=""
37 CSRC_CACHE_OVERRIDE_PATH=""
38 ROCM_ARG_OVERRIDE_PATH=""
39 BUILD_CONTEXT_OVERRIDE_PATH=""
40 SCRIPT_TMP_DIR=""
41 BAKE_CONFIG_FILE=""
42 ROCM_BUILD_CONTEXT_ROOT=""
43 ROCM_BUILD_CONTEXT_INDEX=""
44 ROCM_BUILD_CONTEXT_COMMIT=""
45 BAKE_FILES=()
46 BAKE_ALLOW_ARGS=()
47 BAKE_TARGETS=()
48 DEPENDENCY_CACHE_TARGETS=()
49
50 cleanup() {
51 if [[ -n "${SCRIPT_TMP_DIR}" && -d "${SCRIPT_TMP_DIR}" ]]; then
52 rm -rf "${SCRIPT_TMP_DIR}"
53 fi
54 }
55 trap cleanup EXIT
56
57 clean_docker_tag() {
58 local input="$1"
59 echo "${input}" | sed 's/[^a-zA-Z0-9._-]/_/g' | cut -c1-128
60 }
61
62 is_url_like() {
63 local value="${1:-}"
64 [[ "${value}" =~ ^[a-zA-Z][a-zA-Z0-9+.-]*:// || "${value}" == git@*:* ]]
65 }
66
67 is_full_git_sha() {
68 local value="${1:-}"
69 [[ "${value}" =~ ^[0-9a-fA-F]{40}$ ]]
70 }
71
72 select_cache_branch_name() {
73 local candidate=""
74 local var=""
75
76 for var in \
77 ROCM_CACHE_BRANCH_NAME \
78 BUILDKITE_PULL_REQUEST_HEAD_BRANCH \
79 BUILDKITE_HEAD_BRANCH \
80 BUILDKITE_BRANCH \
81 VLLM_BRANCH; do
82 candidate="${!var:-}"
83 [[ -n "${candidate}" ]] || continue
84 is_url_like "${candidate}" && continue
85 is_full_git_sha "${candidate}" && continue
86 printf '%s\n' "${candidate}"
87 return 0
88 done
89 }
90
91 cache_scope_suffix() {
92 local arch_hash=""
93 arch_hash=$(printf '%s' "${PYTORCH_ROCM_ARCH:-default}" | sha256sum | cut -c1-12)
94 printf 'arch-%s\n' "${arch_hash}"
95 }
96
97 compose_cache_branch_tag() {
98 local repo_slug="$1"
99 local branch="$2"
100 local suffix=""
101 local prefix=""
102 local max_prefix_len=0
103 local max_tag_len="${ROCM_CACHE_BRANCH_TAG_MAX_LEN}"
104
105 suffix="$(cache_scope_suffix)"
106 prefix="$(clean_docker_tag "${repo_slug}")-$(clean_docker_tag "${branch}")"
107 max_prefix_len=$((max_tag_len - ${#suffix} - 1))
108 if (( max_prefix_len < 1 )); then
109 max_prefix_len=1
110 fi
111 printf '%s-%s\n' "${prefix:0:${max_prefix_len}}" "${suffix}"
112 }
113
114 parse_repo_slug() {
115 local repo_url="${1:-}"
116 local repo_slug=""
117
118 if [[ -z "${repo_url}" ]]; then
119 printf '%s\n' "${DEFAULT_REPO_SLUG}"
120 return 0
121 fi
122
123 repo_slug=$(echo "${repo_url}" | sed -E 's#(git@|https?://)([^/:]+)[:/]([^/]+/[^/.]+)(\.git)?$#\3#')
124 if [[ "${repo_slug}" != */* ]]; then
125 repo_slug="${DEFAULT_REPO_SLUG}"
126 fi
127 printf '%s\n' "${repo_slug}"
128 }
129
130 normalize_repo_slug() {
131 local repo_slug="${1:-}"
132
133 repo_slug="${repo_slug%/}"
134 repo_slug="${repo_slug%.git}"
135 repo_slug="${repo_slug#https://github.com/}"
136 repo_slug="${repo_slug#http://github.com/}"
137 repo_slug="${repo_slug#ssh://git@github.com/}"
138 repo_slug="${repo_slug#git@github.com:}"
139 repo_slug="${repo_slug#github.com/}"
140 printf '%s\n' "${repo_slug}"
141 }
142
143 is_trusted_ci_cache_writer() {
144 local actual_repo=""
145 local trusted_repo=""
146
147 [[ "${BUILDKITE:-false}" == "true" ]] || return 1
148 [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" ]] || return 1
149 [[ "${BUILDKITE_BRANCH:-}" == "${CI_BASE_STABLE_BRANCH:-main}" ]] || return 1
150 actual_repo=$(normalize_repo_slug "${BUILDKITE_REPO:-}")
151 trusted_repo=$(normalize_repo_slug \
152 "${CI_BASE_STABLE_REPO_SLUG:-${DEFAULT_REPO_SLUG}}")
153 [[ -n "${actual_repo}" && "${actual_repo}" == "${trusted_repo}" ]]
154 }
155
156 ci_base_write_scope() {
157 local identity=""
158 local source_repo="${BUILDKITE_PULL_REQUEST_REPO:-${BUILDKITE_REPO:-local}}"
159
160 if is_trusted_ci_cache_writer; then
161 return 0
162 fi
163 identity=$(printf '%s\n' "${source_repo}" | sha256sum | cut -c1-12)
164 printf 'preview-%s\n' "${identity}"
165 }
166
167 configure_ci_base_write_scope() {
168 local scope=""
169
170 scope=$(ci_base_write_scope)
171 if [[ -n "${scope}" ]]; then
172 CI_BASE_WRITE_SCOPE=$(clean_docker_tag "${scope}")
173 echo "Non-canonical cache writes use source scope: ${CI_BASE_WRITE_SCOPE}"
174 else
175 CI_BASE_WRITE_SCOPE=""
176 echo "Trusted main build: publishing canonical ci_base refs"
177 fi
178 export CI_BASE_WRITE_SCOPE
179 }
180
181 get_buildkite_repo_slug() {
182 parse_repo_slug "${BUILDKITE_PULL_REQUEST_REPO:-${BUILDKITE_REPO:-}}"
183 }
184
185 get_buildkite_target_repo_slug() {
186 parse_repo_slug "${BUILDKITE_REPO:-}"
187 }
188
189 get_buildkite_target_repo_url() {
190 local repo_url="${BUILDKITE_REPO:-}"
191
192 if [[ -n "${repo_url}" ]] && is_url_like "${repo_url}"; then
193 printf '%s\n' "${repo_url}"
194 return 0
195 fi
196
197 printf 'https://github.com/%s.git\n' "${DEFAULT_REPO_SLUG}"
198 }
199
200 git_fetch_with_timeout() {
201 local timeout_secs="${ROCM_CACHE_GIT_FETCH_TIMEOUT:-60}"
202 local -a fetch_command=(git fetch --no-auto-maintenance)
203
204 # Detached maintenance can race a later shallow fetch on .git/shallow.
205 if command -v timeout >/dev/null 2>&1; then
206 timeout "${timeout_secs}s" "${fetch_command[@]}" "$@"
207 else
208 "${fetch_command[@]}" "$@"
209 fi
210 }
211
212 git_fetch_for_cache() {
213 git_fetch_with_timeout "$@" 2>/dev/null
214 }
215
216 hash_string_short() {
217 printf '%s' "$1" | sha256sum | cut -c1-16
218 }
219
220 list_content_files() {
221 # Hash the checkout inputs Docker can intentionally consume, not ignored
222 # compiler/test debris left behind on a reused worker.
223 if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" ]]; then
224 GIT_INDEX_FILE="${ROCM_BUILD_CONTEXT_INDEX}" \
225 git ls-files -z --cached -- "$1" | LC_ALL=C sort -z
226 else
227 git ls-files -z --cached --others --exclude-standard -- "$1" \
228 | LC_ALL=C sort -z
229 fi
230 }
231
232 content_regular_file() {
233 local file="$1"
234 local physical_file=""
235
236 physical_file="${file}"
237 if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${file}" != /* ]]; then
238 physical_file="${ROCM_BUILD_CONTEXT_ROOT}/${file}"
239 fi
240 [[ -f "${physical_file}" ]]
241 }
242
243 hash_content_file() {
244 local file="$1"
245 local physical_file=""
246 local checksum=""
247 local file_mode=""
248
249 physical_file="${file}"
250 if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${file}" != /* ]]; then
251 physical_file="${ROCM_BUILD_CONTEXT_ROOT}/${file}"
252 fi
253 if [[ -L "${physical_file}" ]]; then
254 printf 'symlink:%s\ntarget:' "${file}"
255 readlink -n -- "${physical_file}" || return $?
256 printf '\n'
257 return
258 fi
259 if [[ ! -f "${physical_file}" ]]; then
260 printf 'missing:%s\n' "${file}"
261 return
262 fi
263 file_mode=$(stat -c '%a' "${physical_file}") || return $?
264 printf 'file:%s\nmode:%s\n' "${file}" "${file_mode}"
265 checksum=$(sha256sum < "${physical_file}") || return $?
266 checksum="${checksum%% *}"
267 printf '%s %s\n' "${checksum}" "${file}"
268 }
269
270 hash_content_directory() {
271 if ! list_content_files "$1" | while IFS= read -r -d '' file; do
272 hash_content_file "${file}" || exit $?
273 done; then
274 echo "Failed to hash content under $1" >&2
275 return 1
276 fi
277 }
278
279 compute_content_hash() {
280 local path=""
281 local physical_path=""
282
283 for path in "$@"; do
284 physical_path="${path}"
285 if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${path}" != /* ]]; then
286 physical_path="${ROCM_BUILD_CONTEXT_ROOT}/${path}"
287 fi
288 if [[ -L "${physical_path}" || -f "${physical_path}" ]]; then
289 hash_content_file "${path}" || return $?
290 elif [[ -d "${physical_path}" ]]; then
291 hash_content_directory "${path}" || return $?
292 else
293 printf 'missing:%s\n' "${path}"
294 fi
295 done | sha256sum | cut -d' ' -f1
296 }
297
298 validate_ci_build_context_source() {
299 local source_root="$1"
300 local context_commit=""
301 local source_head=""
302 local entry=""
303 local metadata=""
304 local mode=""
305 local stage=""
306 local path=""
307 local worktree_diff_status=0
308 local staged_diff_status=0
309 local index_modes_file="${SCRIPT_TMP_DIR}/git-index-modes"
310 local info_attributes=""
311
312 if ! context_commit=$(git -C "${source_root}" rev-parse \
313 --verify "${BUILDKITE_COMMIT:-HEAD}^{commit}"); then
314 echo "Failed to resolve the CI Docker context revision" >&2
315 return 1
316 fi
317 if ! source_head=$(git -C "${source_root}" rev-parse --verify HEAD); then
318 echo "Failed to resolve the checked-out CI revision" >&2
319 return 1
320 fi
321 if [[ "${context_commit}" != "${source_head}" ]]; then
322 echo "BUILDKITE_COMMIT does not match the checked-out CI revision" >&2
323 return 1
324 fi
325 ROCM_BUILD_CONTEXT_COMMIT="${context_commit}"
326 # Shared AMD workspaces may present tracked files with inflated executable
327 # bits. Context modes come from the pinned Git tree, so ignore only that
328 # filesystem drift while continuing to reject content changes.
329 git -C "${source_root}" -c core.fileMode=false diff \
330 --quiet --no-ext-diff --ignore-submodules=none -- \
331 || worktree_diff_status=$?
332 if (( worktree_diff_status != 0 )); then
333 if (( worktree_diff_status > 1 )); then
334 printf 'Failed to inspect tracked CI worktree changes (git diff exited %s)\n' \
335 "${worktree_diff_status}" >&2
336 return 1
337 fi
338 echo "Tracked worktree changes cannot be omitted from the CI Docker context" \
339 >&2
340 echo "Tracked worktree diff (first 50 entries):" >&2
341 git -C "${source_root}" -c core.fileMode=false diff \
342 --no-ext-diff --ignore-submodules=none --name-status -- \
343 | sed -n '1,50p' >&2 || true
344 git -C "${source_root}" -c core.fileMode=false diff \
345 --no-ext-diff --ignore-submodules=none --summary -- \
346 | sed -n '1,50p' >&2 || true
347 return 1
348 fi
349 git -C "${source_root}" diff --cached \
350 --quiet --no-ext-diff --ignore-submodules=none HEAD -- \
351 || staged_diff_status=$?
352 if (( staged_diff_status != 0 )); then
353 if (( staged_diff_status > 1 )); then
354 printf 'Failed to inspect staged CI changes (git diff exited %s)\n' \
355 "${staged_diff_status}" >&2
356 return 1
357 fi
358 echo "Staged changes cannot be omitted from the CI Docker context" >&2
359 echo "Staged diff (first 50 entries):" >&2
360 git -C "${source_root}" diff --cached \
361 --no-ext-diff --ignore-submodules=none --name-status HEAD -- \
362 | sed -n '1,50p' >&2 || true
363 git -C "${source_root}" diff --cached \
364 --no-ext-diff --ignore-submodules=none --summary HEAD -- \
365 | sed -n '1,50p' >&2 || true
366 return 1
367 fi
368 # Untracked and ignored worker outputs are intentionally absent: the
369 # pinned Git tree, rather than mutable checkout contents, is the contract.
370 if ! info_attributes=$(git -C "${source_root}" \
371 rev-parse --path-format=absolute --git-path info/attributes); then
372 echo "Failed to locate repository-local Git attributes" >&2
373 return 1
374 fi
375 if [[ -s "${info_attributes}" ]]; then
376 echo "Repository-local Git attributes cannot define the CI Docker context" \
377 >&2
378 return 1
379 fi
380
381 ROCM_BUILD_CONTEXT_INDEX="${SCRIPT_TMP_DIR}/docker-context.index"
382 if ! GIT_INDEX_FILE="${ROCM_BUILD_CONTEXT_INDEX}" \
383 git -C "${source_root}" -c core.splitIndex=false \
384 read-tree "${context_commit}^{tree}"; then
385 echo "Failed to create the CI Docker context index" >&2
386 return 1
387 fi
388 if ! GIT_INDEX_FILE="${ROCM_BUILD_CONTEXT_INDEX}" \
389 git -C "${source_root}" ls-files --stage -z \
390 > "${index_modes_file}"; then
391 echo "Failed to read the Git index for the CI Docker context" >&2
392 return 1
393 fi
394 while IFS= read -r -d '' entry; do
395 metadata="${entry%%$'\t'*}"
396 mode="${metadata%% *}"
397 stage="${metadata##* }"
398 path="${entry#*$'\t'}"
399 if [[ "${stage}" != "0" ]]; then
400 echo "Unmerged Git entry cannot be used as Docker context: ${path}" >&2
401 return 1
402 fi
403 case "${mode}" in
404 100644|100755|120000)
405 ;;
406 160000)
407 echo "Git submodule cannot be materialized in the CI Docker context: ${path}" \
408 >&2
409 return 1
410 ;;
411 *)
412 echo "Unsupported Git mode ${mode} for ${path}" >&2
413 return 1
414 ;;
415 esac
416 done < "${index_modes_file}"
417 }
418
419 describe_ci_revision() {
420 git -C "$1" describe --tags --long --abbrev=10 \
421 --match 'v[0-9]*' "$2" 2>/dev/null
422 }
423
424 write_ci_git_archival_metadata() {
425 local source_root="$1"
426 local context_root="$2"
427 local commit="${ROCM_BUILD_CONTEXT_COMMIT}"
428 local commit_date=""
429 local describe=""
430 local is_shallow="false"
431
432 commit_date=$(git -C "${source_root}" show -s --format=%cI "${commit}") \
433 || return $?
434 is_shallow=$(git -C "${source_root}" rev-parse --is-shallow-repository) \
435 || return $?
436 if git -C "${source_root}" remote get-url origin >/dev/null 2>&1; then
437 # AMD agents use shallow, no-tag clones. Reach a release tag so one
438 # commit cannot acquire different versions from different depths.
439 # Versioning only needs tags and commits, not historical source trees.
440 echo "Synchronizing version tags for the canonical CI Docker context"
441 (cd "${source_root}" && git_fetch_with_timeout --quiet \
442 --filter=tree:0 --prune origin '+refs/tags/*:refs/tags/*') \
443 || return $?
444 if [[ "${is_shallow}" == "true" ]] \
445 && ! describe_ci_revision "${source_root}" "${commit}" >/dev/null; then
446 echo "Deepening history to reach a version tag"
447 (cd "${source_root}" && git_fetch_with_timeout --quiet \
448 --filter=tree:0 --no-tags --deepen=1000 origin "${commit}") \
449 || return $?
450 is_shallow=$(git -C "${source_root}" \
451 rev-parse --is-shallow-repository) || return $?
452 if [[ "${is_shallow}" == "true" ]] \
453 && ! describe_ci_revision "${source_root}" "${commit}" >/dev/null; then
454 echo "No version tag within 1,000 commits; fetching full history"
455 (cd "${source_root}" && git_fetch_with_timeout --quiet \
456 --filter=tree:0 --no-tags --unshallow origin "${commit}") \
457 || return $?
458 fi
459 fi
460 fi
461 if ! describe=$(describe_ci_revision "${source_root}" "${commit}"); then
462 echo "No numeric version tag is reachable from the CI revision" >&2
463 return 1
464 fi
465 if [[ -e "${context_root}/.git_archival.txt" \
466 || -L "${context_root}/.git_archival.txt" ]]; then
467 echo "The source tree already contains .git_archival.txt" >&2
468 return 1
469 fi
470 printf 'node: %s\nnode-date: %s\ndescribe-name: %s\n' \
471 "${commit}" "${commit_date}" "${describe}" \
472 > "${context_root}/.git_archival.txt" \
473 && chmod 0644 -- "${context_root}/.git_archival.txt"
474 }
475
476 write_build_context_override() {
477 local escaped_context=""
478
479 [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" ]] || return 0
480 escaped_context=$(hcl_escape_string "${ROCM_BUILD_CONTEXT_ROOT}") || return $?
481 if ! {
482 printf 'target "_common-rocm" {\n'
483 printf ' context = "%s"\n' "${escaped_context}"
484 printf '}\n'
485 } > "${BUILD_CONTEXT_OVERRIDE_PATH}"; then
486 echo "Failed to write the CI Docker context override" >&2
487 return 1
488 fi
489 BAKE_FILES+=(-f "${BUILD_CONTEXT_OVERRIDE_PATH}")
490 }
491
492 prepare_ci_build_context() {
493 local source_root=""
494 local context_root=""
495
496 [[ "${BUILDKITE:-false}" == "true" ]] || return 0
497 [[ "${REMOTE_VLLM:-0}" == "0" ]] || return 0
498
499 # BuildKit includes file modes in cache keys. Export the pinned revision to
500 # an owned context instead of changing modes in the shared checkout.
501 if ! source_root=$(git rev-parse --show-toplevel); then
502 echo "Failed to locate the CI source checkout" >&2
503 return 1
504 fi
505 validate_ci_build_context_source "${source_root}" || return $?
506 context_root="${SCRIPT_TMP_DIR}/docker-context"
507 if ! mkdir -m 0700 -- "${context_root}"; then
508 echo "Failed to create the owned CI Docker context" >&2
509 return 1
510 fi
511 if ! (
512 umask 0022
513 unset GIT_LFS_SKIP_SMUDGE
514 export GIT_ATTR_NOSYSTEM=1
515 export GIT_INDEX_FILE="${ROCM_BUILD_CONTEXT_INDEX}"
516 git -C "${source_root}" \
517 -c core.attributesFile=/dev/null \
518 -c core.autocrlf=false \
519 -c core.eol=lf \
520 -c core.symlinks=true \
521 --work-tree="${context_root}" \
522 checkout-index --all
523 ); then
524 echo "Failed to materialize the CI Docker context from the Git index" >&2
525 return 1
526 fi
527
528 # setuptools-scm understands Git's stable archive format, so wheels and
529 # Rust artifacts retain their exact version without copying Git history.
530 if ! is_ci_base_target; then
531 write_ci_git_archival_metadata "${source_root}" "${context_root}" \
532 || return $?
533 fi
534 if [[ -e "${context_root}/.git" ]]; then
535 echo "Canonical CI Docker context unexpectedly contains .git" >&2
536 return 1
537 fi
538 ROCM_BUILD_CONTEXT_ROOT="${context_root}"
539 BAKE_ALLOW_ARGS+=(--allow "fs.read=${ROCM_BUILD_CONTEXT_ROOT}")
540 echo "Using canonical CI Docker context: ${ROCM_BUILD_CONTEXT_ROOT}"
541 }
542
543 configure_custom_rocm_stages() {
544 using_custom_rocm_dockerfiles || return 0
545 local context_root="${ROCM_BUILD_CONTEXT_ROOT:-.}"
546 local dockerfile="${context_root}/${CI_BASE_DOCKERFILE}"
547 local stages=""
548
549 (cd "${context_root}" && \
550 validate_rocm_dockerfile "${CI_BASE_DOCKERFILE}" \
551 ci_base test export_vllm export_test_smoke csrc-build rust-build) \
552 || return $?
553 # Hash all stages for custom layouts, including additional helper stages.
554 stages=$(rocm_dockerfile_stages "${dockerfile}" | paste -sd ' ')
555 CI_BASE_DOCKERFILE_STAGES="${CI_BASE_DOCKERFILE_STAGES:-${stages}}"
556 ROCM_CSRC_DOCKERFILE_STAGES="${ROCM_CSRC_DOCKERFILE_STAGES:-${stages}}"
557 ROCM_RUST_DOCKERFILE_STAGES="${ROCM_RUST_DOCKERFILE_STAGES:-${stages}}"
558 if [[ " ${stages} " != *" build_rocshmem "* ]]; then
559 unset ROCSHMEM_BRANCH ROCSHMEM_CACHE_KEY DEEPEP_CACHE_KEY
560 fi
561 }
562
563 compose_dependency_cache_key() {
564 local prefix="$1"
565 local material="$2"
566 local cleaned_prefix=""
567
568 cleaned_prefix=$(clean_docker_tag "${prefix}" | cut -c1-96)
569 printf '%s-%s\n' "${cleaned_prefix}" "$(hash_string_short "${material}")"
570 }
571
572 hash_dockerfile_stages() {
573 local dockerfile="$1"
574 local stages="$2"
575 local physical_dockerfile=""
576
577 physical_dockerfile="${dockerfile}"
578 if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${dockerfile}" != /* ]]; then
579 physical_dockerfile="${ROCM_BUILD_CONTEXT_ROOT}/${dockerfile}"
580 fi
581
582 awk -v wanted_stages="${stages}" '
583 BEGIN {
584 split(wanted_stages, stage_list, /[[:space:]]+/)
585 for (idx in stage_list) {
586 if (stage_list[idx] != "") {
587 wanted[stage_list[idx]] = 1
588 }
589 }
590 emit = 0
591 }
592 toupper($1) == "FROM" {
593 stage = ""
594 for (idx = 1; idx <= NF; idx++) {
595 if (tolower($idx) == "as" && idx < NF) {
596 stage = tolower($(idx + 1))
597 }
598 }
599 emit = (stage in wanted)
600 }
601 emit {
602 print
603 }
604 ' "${physical_dockerfile}"
605 }
606
607 discover_dockerfile_stage_args() {
608 local dockerfile="$1"
609 local stages="$2"
610 local physical_dockerfile=""
611
612 physical_dockerfile="${dockerfile}"
613 if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${dockerfile}" != /* ]]; then
614 physical_dockerfile="${ROCM_BUILD_CONTEXT_ROOT}/${dockerfile}"
615 fi
616 [[ -f "${physical_dockerfile}" ]] || return 0
617
618 awk -v wanted_stages="${stages}" '
619 function add_arg(name) {
620 if (name != "" && !(name in seen)) {
621 seen[name] = 1
622 args[++arg_count] = name
623 }
624 }
625 BEGIN {
626 split(wanted_stages, stage_list, /[[:space:]]+/)
627 for (idx in stage_list) {
628 if (stage_list[idx] != "") {
629 wanted[stage_list[idx]] = 1
630 }
631 }
632 emit = 1
633 }
634 {
635 line = $0
636 if (toupper($1) == "FROM") {
637 stage = ""
638 for (idx = 1; idx <= NF; idx++) {
639 if (tolower($idx) == "as" && idx < NF) {
640 stage = tolower($(idx + 1))
641 }
642 }
643 emit = (stage in wanted)
644 }
645 if (emit) {
646 lines[++line_count] = line
647 }
648 }
649 END {
650 for (idx = 1; idx <= line_count; idx++) {
651 line = lines[idx]
652 arg_name = line
653 sub(/^[[:space:]]*[Aa][Rr][Gg][[:space:]]+/, "", arg_name)
654 if (arg_name != line) {
655 sub(/[=[:space:]].*/, "", arg_name)
656 if (arg_name ~ /^[A-Za-z_][A-Za-z0-9_]*$/) {
657 add_arg(arg_name)
658 }
659 }
660 }
661
662 for (idx = 1; idx <= line_count; idx++) {
663 line = lines[idx]
664 for (arg_idx = 1; arg_idx <= arg_count; arg_idx++) {
665 name = args[arg_idx]
666 if (line ~ "\\$\\{" name "([}:][^}]*)?\\}" \
667 || line ~ "\\$" name "([^A-Za-z0-9_]|$)") {
668 used[name] = 1
669 }
670 }
671 }
672
673 for (arg_idx = 1; arg_idx <= arg_count; arg_idx++) {
674 name = args[arg_idx]
675 if (used[name]) {
676 print name
677 }
678 }
679 }
680 ' "${physical_dockerfile}"
681 }
682
683 get_content_arg_names() {
684 local dockerfile="$1"
685 local stages="$2"
686 local explicit_args="${3:-}"
687
688 if [[ -n "${explicit_args}" ]]; then
689 tr ' ' '\n' <<< "${explicit_args}"
690 else
691 discover_dockerfile_stage_args "${dockerfile}" "${stages}"
692 fi | awk 'NF && !seen[$0]++'
693 }
694
695 compute_ci_base_content_hash() {
696 local -a content_paths=()
697 local -a content_args=()
698 local content_files_hash=""
699 local dockerfile="${CI_BASE_DOCKERFILE:-}"
700 local stages="${CI_BASE_DOCKERFILE_STAGES:-}"
701
702 read -r -a content_paths <<< "${CI_BASE_CONTENT_FILES}"
703 mapfile -t content_args < <(
704 get_content_arg_names "${dockerfile}" "${stages}" "${CI_BASE_CONTENT_ARGS:-}"
705 )
706 if ! content_files_hash=$(compute_content_hash "${content_paths[@]}"); then
707 echo "Failed to hash ci_base content files" >&2
708 return 1
709 fi
710
711 {
712 printf 'content-files-hash:%s\n' "${content_files_hash}"
713 if [[ -n "${dockerfile}" ]]; then
714 printf 'dockerfile:%s\n' "${dockerfile}"
715 printf 'resolved-build-args:\n'
716 hash_dockerfile_arg_values "${dockerfile}" "${content_args[@]}" \
717 || return 1
718 if [[ -n "${stages}" ]]; then
719 printf 'dockerfile-stages:%s\n' "${stages}"
720 if content_regular_file "${dockerfile}"; then
721 hash_dockerfile_stages "${dockerfile}" "${stages}"
722 else
723 printf 'missing:%s\n' "${dockerfile}"
724 fi
725 fi
726 fi
727 } | sha256sum | cut -d' ' -f1
728 }
729
730 extract_dockerfile_arg_default() {
731 local dockerfile="$1"
732 local arg_name="$2"
733 local physical_dockerfile=""
734
735 physical_dockerfile="${dockerfile}"
736 if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${dockerfile}" != /* ]]; then
737 physical_dockerfile="${ROCM_BUILD_CONTEXT_ROOT}/${dockerfile}"
738 fi
739 sed -n -E "s/^[[:space:]]*[Aa][Rr][Gg][[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
740 "${physical_dockerfile}" | head -1
741 }
742
743 resolve_image_digest() {
744 local image_ref="$1"
745 local attempts="${ROCM_IMAGE_DIGEST_ATTEMPTS:-4}"
746 local delay_secs="${ROCM_IMAGE_DIGEST_RETRY_DELAY:-2}"
747 local attempt=0
748 local digest=""
749 local output=""
750 local status=0
751
752 if [[ "${image_ref}" =~ @(sha256:[0-9a-f]{64})$ ]]; then
753 printf '%s\n' "${BASH_REMATCH[1]}"
754 return
755 fi
756 if [[ ! "${attempts}" =~ ^[1-9][0-9]*$ \
757 || ! "${delay_secs}" =~ ^[0-9]+$ ]]; then
758 echo "Invalid image digest retry configuration" >&2
759 return 1
760 fi
761
762 for ((attempt = 1; attempt <= attempts; attempt++)); do
763 status=0
764 output=$(docker buildx imagetools inspect "${image_ref}" 2>&1) || status=$?
765 digest=$(awk '$1 == "Digest:" { print $2; exit }' <<< "${output}")
766 if ((status == 0)) && [[ "${digest}" =~ ^sha256:[0-9a-f]{64}$ ]]; then
767 printf '%s\n' "${digest}"
768 return
769 fi
770 if ((attempt < attempts)); then
771 printf \
772 'Image digest lookup failed for %s (%d/%d, status %d); retrying\n' \
773 "${image_ref}" "${attempt}" "${attempts}" "${status}" >&2
774 sleep "${delay_secs}"
775 fi
776 done
777
778 printf 'Failed to resolve digest for %s (status %d)\n%s\n' \
779 "${image_ref}" "${status}" "${output:-<no output>}" >&2
780 return 1
781 }
782
783 resolve_dockerfile_arg_value() {
784 local dockerfile="$1"
785 local arg_name="$2"
786 local env_name="${arg_name}"
787 local value=""
788
789 case "${arg_name}" in
790 ARG_PYTORCH_ROCM_ARCH)
791 env_name="PYTORCH_ROCM_ARCH"
792 ;;
793 max_jobs)
794 env_name="CI_MAX_JOBS"
795 ;;
796 esac
797
798 value="${!env_name:-}"
799 if [[ -z "${value}" && "${env_name}" != "${arg_name}" ]]; then
800 value="${!arg_name:-}"
801 fi
802 if [[ -z "${value}" ]] && content_regular_file "${dockerfile}"; then
803 value=$(extract_dockerfile_arg_default "${dockerfile}" "${arg_name}")
804 fi
805
806 printf '%s\n' "${value}"
807 }
808
809 hash_dockerfile_arg_values() {
810 local dockerfile="$1"
811 local arg_name=""
812 local arg_value=""
813 local digest=""
814 shift || true
815
816 for arg_name in "$@"; do
817 [[ -n "${arg_name}" ]] || continue
818 arg_value=$(resolve_dockerfile_arg_value "${dockerfile}" "${arg_name}")
819 if [[ "${arg_name}" == "BASE_IMAGE" && -n "${arg_value}" ]]; then
820 if ! digest=$(resolve_image_digest "${arg_value}"); then
821 echo "Failed to resolve digest for BASE_IMAGE=${arg_value}" >&2
822 return 1
823 fi
824 printf 'arg:%s.digest=%s\n' "${arg_name}" "${digest}"
825 else
826 printf 'arg:%s=%s\n' "${arg_name}" "${arg_value:-<empty>}"
827 fi
828 done
829 }
830
831 pin_base_image() {
832 local dockerfile="${CI_BASE_DOCKERFILE:-${DEFAULT_CI_BASE_DOCKERFILE}}"
833 local base_image=""
834 local digest=""
835
836 base_image=$(resolve_dockerfile_arg_value "${dockerfile}" "BASE_IMAGE")
837 [[ -n "${base_image}" ]] || return 0
838 if ! digest=$(resolve_image_digest "${base_image}"); then
839 echo "Error: could not resolve base image digest for ${base_image}" >&2
840 echo "Refusing to compute content-addressed cache keys from a mutable tag." >&2
841 return 1
842 fi
843 BASE_IMAGE="${base_image%@*}@${digest}"
844 export BASE_IMAGE
845 echo "Pinned base image for this build: ${BASE_IMAGE}"
846 }
847
848 is_ci_base_target() {
849 [[ "${TARGET}" == *"ci-base-rocm"* ]]
850 }
851
852 is_commit_image_target() {
853 [[ -n "${IMAGE_TAG:-}" && -n "${BUILDKITE_COMMIT:-}" ]] || return 1
854 is_ci_base_target && return 1
855 return 0
856 }
857
858 should_upload_wheel_artifacts() {
859 [[ "${UPLOAD_ROCM_WHEEL_ARTIFACTS:-0}" == "1" ]] && return 0
860 [[ "${TARGET}" == *"with-wheel"* \
861 || "${TARGET}" == *"export-wheel"* \
862 || "${TARGET}" == *"artifact"* ]]
863 }
864
865 should_export_rocm_smoke() {
866 [[ "${TARGET}" == "test-rocm-ci-with-wheel" \
867 || "${TARGET}" == "smoke-test-rocm-ci" ]]
868 }
869
870 verify_rocm_smoke_export() {
871 local marker="./build/rocm-smoke-export/vllm-smoke-ok"
872 local expected_smoke_id="${BUILDKITE_BUILD_ID:-local}"
873 local actual_smoke_id=""
874
875 should_export_rocm_smoke || return 0
876 if [[ ! -f "${marker}" ]]; then
877 echo "ROCm BuildKit smoke marker is missing: ${marker}" >&2
878 return 1
879 fi
880 actual_smoke_id="$(< "${marker}")"
881 if [[ "${actual_smoke_id}" != "${expected_smoke_id}" ]]; then
882 echo "ROCm BuildKit smoke marker belongs to ${actual_smoke_id}, not ${expected_smoke_id}" \
883 >&2
884 return 1
885 fi
886 }
887
888 get_remote_image_label() {
889 local image_ref="$1"
890 local label_key="$2"
891 local format="{{ index .Image.Config.Labels \"${label_key}\" }}"
892
893 docker buildx imagetools inspect "${image_ref}" \
894 --format "${format}" 2>/dev/null | awk 'NF { print; exit }' || true
895 }
896
897 remote_ci_base_identity_is_current_with_retry() {
898 local image_ref="$1"
899 local attempts="${CI_BASE_LABEL_ATTEMPTS:-4}"
900 local delay_secs="${CI_BASE_LABEL_RETRY_DELAY:-2}"
901 local identity=""
902 local remote_hash=""
903 local remote_version=""
904 local content_files_hash=""
905 local base_digest=""
906 local attempt=0
907 local expected_version="${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}"
908 local format='{{ index .Image.Config.Labels "vllm.ci_base.content_hash" }}|{{ index .Image.Config.Labels "vllm.ci_base.metadata_version" }}|{{ index .Image.Config.Labels "vllm.ci_base.content_files_hash" }}|{{ index .Image.Config.Labels "vllm.rocm.base_image_digest" }}'
909
910 if [[ ! "${attempts}" =~ ^[1-9][0-9]*$ \
911 || ! "${delay_secs}" =~ ^[0-9]+$ ]]; then
912 echo "Invalid ci_base label retry configuration" >&2
913 return 1
914 fi
915
916 for ((attempt = 1; attempt <= attempts; attempt++)); do
917 identity=$(docker buildx imagetools inspect "${image_ref}" \
918 --format "${format}" 2>/dev/null || true)
919 IFS='|' read -r \
920 remote_hash remote_version content_files_hash base_digest <<< "${identity}"
921 if [[ "${remote_hash}" == "${CI_BASE_CONTENT_HASH:-}" \
922 && "${remote_version}" == "${expected_version}" \
923 && "${content_files_hash}" =~ ^[0-9a-f]{64}$ \
924 && "${base_digest}" =~ ^sha256:[0-9a-f]{64}$ ]]; then
925 return 0
926 fi
927 ((attempt == attempts)) || sleep "${delay_secs}"
928 done
929
930 echo "ci_base identity did not match after ${attempts} attempts: ${image_ref}" >&2
931 echo " expected hash/version: ${CI_BASE_CONTENT_HASH:-<missing>}/${expected_version}" >&2
932 echo " observed hash/version: ${remote_hash:-<missing>}/${remote_version:-<missing>}" >&2
933 return 1
934 }
935
936 registry_ref_exists_with_retry() {
937 local inspect_kind="$1"
938 local image_ref="$2"
939 local attempts="${ROCM_REGISTRY_PROBE_ATTEMPTS:-2}"
940 local delay_secs="${ROCM_REGISTRY_PROBE_RETRY_DELAY:-1}"
941 local output=""
942 local status=0
943 local attempt=0
944
945 if [[ ! "${attempts}" =~ ^[1-9][0-9]*$ \
946 || ! "${delay_secs}" =~ ^[0-9]+$ ]]; then
947 echo "Invalid registry probe retry configuration" >&2
948 return 1
949 fi
950
951 for ((attempt = 1; attempt <= attempts; attempt++)); do
952 status=0
953 if [[ "${inspect_kind}" == "manifest" ]]; then
954 output=$(docker manifest inspect "${image_ref}" 2>&1) || status=$?
955 else
956 output=$(docker buildx imagetools inspect "${image_ref}" 2>&1) \
957 || status=$?
958 fi
959 if ((status == 0)); then
960 return 0
961 fi
962 # A definitive registry miss is normal for a new content key. Retry
963 # transport/rate-limit failures, but do not add latency to a real 404.
964 if grep -Eiq \
965 'manifest unknown|no such manifest|name unknown|not found|does not exist' \
966 <<< "${output}"; then
967 return 1
968 fi
969 if ((attempt < attempts)); then
970 echo "Registry probe failed for ${image_ref} (${attempt}/${attempts}); retrying" >&2
971 sleep "${delay_secs}"
972 fi
973 done
974
975 echo "Registry probe failed after ${attempts} attempts: ${image_ref}" >&2
976 return 1
977 }
978
979 remote_image_exists() {
980 registry_ref_exists_with_retry manifest "$1"
981 }
982
983 use_existing_builder() {
984 echo "Using existing builder: ${BUILDER_NAME}"
985 docker buildx use "${BUILDER_NAME}"
986 docker buildx inspect --bootstrap
987 }
988
989 buildx_driver() {
990 local builder="${1:-}"
991
992 if [[ -n "${builder}" ]]; then
993 docker buildx inspect "${builder}" 2>/dev/null
994 else
995 docker buildx inspect 2>/dev/null
996 fi | awk -F': *' '$1 == "Driver" { print $2; exit }'
997 }
998
999 builder_supports_registry_cache() {
1000 local driver="$1"
1001
1002 [[ -n "${driver}" && "${driver}" != "docker" ]]
1003 }
1004
1005 create_and_bootstrap_builder() {
1006 local driver="$1"
1007 local endpoint="${2:-}"
1008
1009 echo "Creating builder '${BUILDER_NAME}' with ${driver} driver"
1010 if [[ -n "${endpoint}" ]]; then
1011 docker buildx create \
1012 --name "${BUILDER_NAME}" \
1013 --driver "${driver}" \
1014 --use \
1015 "${endpoint}"
1016 else
1017 docker buildx create --name "${BUILDER_NAME}" --driver "${driver}" --use
1018 fi
1019 docker buildx inspect --bootstrap
1020 }
1021
1022 init_config() {
1023 # shellcheck source=.buildkite/scripts/rocm/build-config.sh
1024 source "$(dirname "${BASH_SOURCE[0]}")/rocm/build-config.sh"
1025 configure_rocm_build
1026
1027 TARGET="${1:-test-ci}"
1028 if using_custom_rocm_dockerfiles; then
1029 if [[ -z "${BASE_IMAGE:-}" ]]; then
1030 echo "Custom ROCm builds require BASE_IMAGE from the selected base handoff" >&2
1031 return 1
1032 fi
1033 if ! is_ci_base_target && [[ -z "${CI_BASE_IMAGE:-}" ]]; then
1034 echo "Custom ROCm builds require CI_BASE_IMAGE from the selected ci_base handoff" >&2
1035 return 1
1036 fi
1037 fi
1038 BAKE_TARGETS=("${TARGET}")
1039 DEPENDENCY_CACHE_TARGETS=()
1040 CI_HCL_SOURCE="${CI_HCL_SOURCE:-${CI_HCL_FILE:-${DEFAULT_CI_HCL_SOURCE}}}"
1041 VLLM_BAKE_FILE="${VLLM_BAKE_FILE:-docker/docker-bake-rocm.hcl}"
1042 BUILDER_NAME="${BUILDER_NAME:-vllm-builder}"
1043 BUILDKIT_SOCKET="${BUILDKIT_SOCKET:-/run/buildkit/buildkitd.sock}"
1044 PYTORCH_ROCM_ARCH="${PYTORCH_ROCM_ARCH:-gfx90a;gfx942;gfx950}"
1045 CI_BASE_CONTENT_FILES="${CI_BASE_CONTENT_FILES:-${DEFAULT_CI_BASE_CONTENT_FILES}}"
1046 CI_BASE_DOCKERFILE="${CI_BASE_DOCKERFILE:-${DEFAULT_CI_BASE_DOCKERFILE}}"
1047 if ! using_custom_rocm_dockerfiles; then
1048 CI_BASE_DOCKERFILE_STAGES="${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}"
1049 fi
1050 CI_BASE_METADATA_VERSION="${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}"
1051 CI_BASE_IMAGE_TAG="${CI_BASE_IMAGE_TAG:-rocm/vllm-dev:ci_base}"
1052 export PYTORCH_ROCM_ARCH CI_BASE_DOCKERFILE
1053
1054 SCRIPT_TMP_DIR=$(mktemp -d -t ci-bake-rocm.XXXXXX)
1055 CI_HCL_PATH="${SCRIPT_TMP_DIR}/ci.hcl"
1056 CI_BASE_LABEL_OVERRIDE_PATH="${SCRIPT_TMP_DIR}/ci-base-label-override.hcl"
1057 CSRC_CACHE_OVERRIDE_PATH="${SCRIPT_TMP_DIR}/rocm-csrc-cache-override.hcl"
1058 ROCM_ARG_OVERRIDE_PATH="${SCRIPT_TMP_DIR}/rocm-arg-override.hcl"
1059 BUILD_CONTEXT_OVERRIDE_PATH="${SCRIPT_TMP_DIR}/build-context-override.hcl"
1060 BAKE_CONFIG_FILE="bake-config-build-${BUILDKITE_BUILD_NUMBER:-local}.json"
1061 }
1062
1063 print_header() {
1064 echo "--- :docker: Setting up Docker buildx bake"
1065 echo "Target: ${TARGET}"
1066 echo "CI HCL source: ${CI_HCL_SOURCE}"
1067 echo "vLLM bake file: ${VLLM_BAKE_FILE}"
1068 if is_ci_base_target; then
1069 echo "Build mode: ci_base"
1070 elif is_commit_image_target; then
1071 echo "Build mode: build-scoped commit image"
1072 else
1073 echo "Build mode: generic"
1074 fi
1075 if [[ "${USE_SCCACHE:-0}" == "1" ]]; then
1076 echo "Compiler cache: sccache enabled"
1077 fi
1078 }
1079
1080 validate_inputs() {
1081 if using_custom_rocm_dockerfiles; then
1082 validate_rocm_dockerfile "${ROCM_BASE_DOCKERFILE}" || return $?
1083 validate_rocm_dockerfile "${CI_BASE_DOCKERFILE}" \
1084 ci_base test export_vllm export_test_smoke csrc-build rust-build || return $?
1085 fi
1086 if [[ ! -f "${VLLM_BAKE_FILE}" ]]; then
1087 echo "Error: vLLM bake file not found at ${VLLM_BAKE_FILE}"
1088 echo "Make sure you're running from the vLLM repository root"
1089 exit 1
1090 fi
1091
1092 if [[ -n "${CI_HCL_SOURCE:-}" ]] && is_url_like "${CI_HCL_SOURCE}"; then
1093 echo "Error: remote CI HCL sources are not supported: ${CI_HCL_SOURCE}"
1094 echo "Use the vLLM-owned docker/ci-rocm.hcl or set CI_HCL_SOURCE to a local file."
1095 exit 1
1096 fi
1097
1098 if [[ -n "${CI_HCL_SOURCE:-}" && ! -f "${CI_HCL_SOURCE}" ]]; then
1099 echo "Error: CI HCL file not found at ${CI_HCL_SOURCE}"
1100 echo "Set CI_HCL_SOURCE to a local file if you need an override."
1101 exit 1
1102 fi
1103 }
1104
1105 load_ci_hcl() {
1106 echo "--- :page_facing_up: Loading ci.hcl"
1107 cp "${CI_HCL_SOURCE}" "${CI_HCL_PATH}"
1108 echo "Copied ${CI_HCL_SOURCE} to ${CI_HCL_PATH}"
1109 }
1110
1111 init_bake_files() {
1112 BAKE_FILES=(-f "${VLLM_BAKE_FILE}" -f "${CI_HCL_PATH}")
1113 if using_custom_rocm_dockerfiles; then
1114 # Custom stacks reuse local cache without replacing the
1115 # standard ROCm registry caches for the same commit or branch.
1116 BAKE_FILES+=(--set '*.cache-to=')
1117 fi
1118 }
1119
1120 compute_ci_base_hash_if_needed() {
1121 if [[ -z "${CI_BASE_CONTENT_FILES:-}" ]]; then
1122 return 0
1123 fi
1124 pin_base_image
1125 if ! is_ci_base_target; then
1126 return 0
1127 fi
1128 if [[ "${REMOTE_VLLM:-0}" != "0" ]]; then
1129 echo "Error: content-addressed ci_base builds require REMOTE_VLLM=0" >&2
1130 return 1
1131 fi
1132
1133 CI_BASE_CONTENT_HASH=$(compute_ci_base_content_hash)
1134 export CI_BASE_CONTENT_HASH
1135 echo "ci_base content hash: ${CI_BASE_CONTENT_HASH:0:16}..."
1136 }
1137
1138 wants_stable_ci_base_tag() {
1139 if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then
1140 return 1
1141 fi
1142
1143 if [[ "${CI_BASE_PUSH_STABLE_TAG:-}" == "1" ]]; then
1144 return 0
1145 fi
1146 if [[ "${CI_BASE_PUSH_STABLE_TAG:-}" == "0" ]]; then
1147 return 1
1148 fi
1149
1150 [[ "${NIGHTLY:-0}" == "1" && "${BUILDKITE_BRANCH:-}" == "${CI_BASE_STABLE_BRANCH:-main}" ]]
1151 }
1152
1153 trusted_ci_base_tip_matches_build() {
1154 local branch="${CI_BASE_STABLE_BRANCH:-main}"
1155 local build_commit="${BUILDKITE_COMMIT:-}"
1156 local remote_tip=""
1157
1158 is_trusted_ci_cache_writer || return 1
1159 if [[ ! "${build_commit}" =~ ^[0-9a-fA-F]{40}$ ]]; then
1160 echo "Skipping ci_base stable tag: Buildkite commit is missing or invalid" >&2
1161 return 1
1162 fi
1163 remote_tip=$(git ls-remote --exit-code "${BUILDKITE_REPO}" \
1164 "refs/heads/${branch}" 2>/dev/null | awk 'NR == 1 { print $1 }')
1165 if [[ ! "${remote_tip}" =~ ^[0-9a-fA-F]{40}$ ]]; then
1166 echo "Skipping ci_base stable tag: could not resolve remote ${branch} tip" >&2
1167 return 1
1168 fi
1169 if [[ "${remote_tip,,}" != "${build_commit,,}" ]]; then
1170 echo "Skipping ci_base stable tag: ${branch} advanced from ${build_commit} to ${remote_tip}" >&2
1171 return 1
1172 fi
1173 }
1174
1175 should_push_stable_ci_base_tag() {
1176 wants_stable_ci_base_tag \
1177 && is_trusted_ci_cache_writer \
1178 && trusted_ci_base_tip_matches_build
1179 }
1180
1181 ci_base_tag_with_suffix() {
1182 local base_tag="$1"
1183 local suffix="$2"
1184
1185 printf '%s-%s\n' "${base_tag}" "$(clean_docker_tag "${suffix}")"
1186 }
1187
1188 configure_ci_base_image_refs() {
1189 local stable_tag="${CI_BASE_IMAGE_TAG:-rocm/vllm-dev:ci_base}"
1190 local metadata_version="${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}"
1191 local scope="${CI_BASE_WRITE_SCOPE:-}"
1192 local trusted_content_tag=""
1193 local content_tag=""
1194 local build_tag=""
1195 local primary_tag=""
1196
1197 if [[ ! "${metadata_version}" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,15}$ ]]; then
1198 echo "Invalid ci_base metadata version: ${metadata_version}" >&2
1199 return 1
1200 fi
1201
1202 if [[ "${BUILDKITE:-false}" == "true" ]] \
1203 && ! is_full_git_sha "${BUILDKITE_COMMIT:-}"; then
1204 echo "Invalid Buildkite commit for ci_base handoff: ${BUILDKITE_COMMIT:-<empty>}" >&2
1205 return 1
1206 fi
1207 if [[ "${BUILDKITE:-false}" == "true" \
1208 && -z "${BUILDKITE_BUILD_ID:-}" ]]; then
1209 echo "Buildkite build ID is required for the ci_base runtime handoff" >&2
1210 return 1
1211 fi
1212 if [[ -n "${BUILDKITE_BUILD_ID:-}" ]]; then
1213 build_tag=$(ci_base_tag_with_suffix \
1214 "${stable_tag}" "build-${BUILDKITE_BUILD_ID}")
1215 fi
1216 CI_BASE_IMAGE_TAG_BUILD_REF="${build_tag}"
1217 export CI_BASE_IMAGE_TAG_BUILD_REF
1218
1219 if [[ -z "${CI_BASE_CONTENT_HASH:-}" ]]; then
1220 if is_ci_base_target; then
1221 echo "Error: ci_base builds require a content hash" >&2
1222 return 1
1223 fi
1224 CI_BASE_IMAGE="${CI_BASE_IMAGE:-${stable_tag}}"
1225 export CI_BASE_IMAGE
1226 return 0
1227 fi
1228
1229 trusted_content_tag=$(ci_base_tag_with_suffix \
1230 "${stable_tag}" "v${metadata_version}-${CI_BASE_CONTENT_HASH}")
1231 if [[ -n "${scope}" ]]; then
1232 content_tag=$(ci_base_tag_with_suffix \
1233 "${stable_tag}" "v${metadata_version}-${scope}-${CI_BASE_CONTENT_HASH}")
1234 else
1235 content_tag="${trusted_content_tag}"
1236 fi
1237 CI_BASE_IMAGE_TAG_CONTENT_REF="${content_tag}"
1238 CI_BASE_TRUSTED_CONTENT_REF="${trusted_content_tag}"
1239 primary_tag="${build_tag:-${content_tag}}"
1240
1241 # Main writes canonical content refs. Other builds import those refs
1242 # read-only and publish into a source-scoped preview namespace.
1243 # A fresh CI build first publishes its unique runtime tag. Content and
1244 # stable aliases are created only after that image passes identity checks.
1245 CI_BASE_STABLE_PROMOTION_REF="${stable_tag}"
1246 CI_BASE_IMAGE_TAG="${primary_tag}"
1247 export CI_BASE_IMAGE_TAG
1248 export CI_BASE_IMAGE_TAG_CONTENT_REF
1249 export CI_BASE_TRUSTED_CONTENT_REF
1250 export CI_BASE_STABLE_PROMOTION_REF
1251
1252 if is_ci_base_target; then
1253 IMAGE_TAG="${primary_tag}"
1254 CI_BASE_IMAGE="${primary_tag}"
1255 export CI_BASE_IMAGE
1256 export IMAGE_TAG
1257
1258 echo "ci_base primary image tag: ${CI_BASE_IMAGE_TAG}"
1259 if [[ -n "${build_tag}" ]]; then
1260 echo "ci_base build image tag: ${build_tag}"
1261 fi
1262 echo "ci_base content image tag: ${content_tag}"
1263 if wants_stable_ci_base_tag && is_trusted_ci_cache_writer; then
1264 echo "ci_base stable alias is eligible for post-build promotion: ${stable_tag}"
1265 else
1266 echo "ci_base stable alias will not be pushed for this build"
1267 echo "Set NIGHTLY=1 on ${CI_BASE_STABLE_BRANCH:-main} to refresh ${stable_tag}"
1268 fi
1269 return 0
1270 fi
1271
1272 if [[ -z "${CI_BASE_IMAGE:-}" || "${CI_BASE_IMAGE}" == "${stable_tag}" ]]; then
1273 CI_BASE_IMAGE="${content_tag}"
1274 export CI_BASE_IMAGE
1275 echo "Using ci_base image: ${CI_BASE_IMAGE}"
1276 else
1277 echo "Using provided CI_BASE_IMAGE override: ${CI_BASE_IMAGE}"
1278 fi
1279 }
1280
1281 publish_ci_base_handoff_ref() {
1282 local source_ref="${1:-${CI_BASE_IMAGE_TAG_BUILD_REF:-${CI_BASE_IMAGE_TAG_CONTENT_REF:-}}}"
1283 local content_ref="${CI_BASE_IMAGE_TAG_CONTENT_REF:-}"
1284 local digest=""
1285 local handoff_ref=""
1286
1287 is_ci_base_target || return 0
1288 if [[ -z "${source_ref}" || -z "${content_ref}" ]]; then
1289 echo "Cannot publish ci_base handoff without source and content refs" >&2
1290 return 1
1291 fi
1292 if ! digest=$(resolve_image_digest "${source_ref}"); then
1293 echo "Could not resolve immutable ci_base handoff: ${source_ref}" >&2
1294 return 1
1295 fi
1296
1297 handoff_ref="${content_ref%@*}@${digest}"
1298 if ! confirm_remote_image_push "${handoff_ref}"; then
1299 echo "Could not validate immutable ci_base handoff: ${handoff_ref}" >&2
1300 return 1
1301 fi
1302 if command -v buildkite-agent >/dev/null 2>&1; then
1303 if ! buildkite-agent meta-data set "rocm-ci-base-image" "${handoff_ref}"; then
1304 echo "Could not publish required ci_base handoff metadata" >&2
1305 return 1
1306 fi
1307 elif [[ "${BUILDKITE:-false}" == "true" ]]; then
1308 echo "buildkite-agent not found; cannot publish ci_base handoff" >&2
1309 return 1
1310 fi
1311 echo "Published immutable ci_base handoff: ${handoff_ref}"
1312 }
1313
1314 ci_base_output_refs() {
1315 printf '%s\n' \
1316 "${CI_BASE_IMAGE_TAG_CONTENT_REF:-}" \
1317 "${CI_BASE_IMAGE_TAG_BUILD_REF:-}" \
1318 | awk 'NF && !seen[$0]++'
1319 }
1320
1321 ci_base_candidate_refs() {
1322 printf '%s\n' \
1323 "${CI_BASE_TRUSTED_CONTENT_REF:-}" \
1324 "${CI_BASE_IMAGE_TAG_CONTENT_REF:-}" \
1325 "${CI_BASE_STABLE_PROMOTION_REF:-}" \
1326 | awk 'NF && !seen[$0]++'
1327 }
1328
1329 find_matching_ci_base_ref() {
1330 local candidate=""
1331 local candidate_digest=""
1332 local immutable_candidate=""
1333
1334 while IFS= read -r candidate; do
1335 [[ -n "${candidate}" ]] || continue
1336 # Missing content refs are normal after an input change. Avoid paying
1337 # the full digest retry budget for each definitely absent candidate.
1338 remote_image_exists "${candidate}" || continue
1339 if ! candidate_digest=$(resolve_image_digest "${candidate}"); then
1340 echo "Could not pin ci_base candidate: ${candidate}" >&2
1341 continue
1342 fi
1343 immutable_candidate="${candidate%@*}@${candidate_digest}"
1344 if remote_ci_base_identity_is_current_with_retry "${immutable_candidate}"; then
1345 printf '%s\n' "${immutable_candidate}"
1346 return 0
1347 fi
1348 done < <(ci_base_candidate_refs)
1349
1350 return 1
1351 }
1352
1353 refresh_ci_base_tags_from_ref() {
1354 local source_ref="$1"
1355 local source_digest=""
1356 local immutable_source=""
1357 local tag=""
1358 local tag_digest=""
1359
1360 if ! source_digest=$(resolve_image_digest "${source_ref}"); then
1361 echo "Could not resolve selected ci_base image: ${source_ref}" >&2
1362 return 1
1363 fi
1364 immutable_source="${source_ref%@*}@${source_digest}"
1365
1366 while IFS= read -r tag; do
1367 [[ -n "${tag}" ]] || continue
1368 [[ "${tag}" != "${source_ref}" ]] || continue
1369 tag_digest=""
1370 if remote_image_exists "${tag}"; then
1371 tag_digest=$(resolve_image_digest "${tag}" || true)
1372 fi
1373 if [[ "${tag_digest}" == "${source_digest}" ]]; then
1374 echo "ci_base tag is already current: ${tag}"
1375 continue
1376 fi
1377 echo "Updating ci_base tag ${tag} -> ${immutable_source}"
1378 if ! docker buildx imagetools create --prefer-index=false \
1379 -t "${tag}" "${immutable_source}"; then
1380 echo "Failed to update ci_base tag ${tag} from ${immutable_source}" >&2
1381 return 1
1382 fi
1383 if ! tag_digest=$(resolve_image_digest "${tag}") \
1384 || [[ "${tag_digest}" != "${source_digest}" ]]; then
1385 echo "Updated ci_base tag does not resolve to the selected digest: ${tag}" >&2
1386 return 1
1387 fi
1388 done < <(ci_base_output_refs)
1389 }
1390
1391 promote_stable_ci_base_tag() {
1392 local source_ref="${1:-${CI_BASE_IMAGE_TAG_BUILD_REF:-${CI_BASE_IMAGE_TAG_CONTENT_REF:-}}}"
1393 local stable_ref="${CI_BASE_STABLE_PROMOTION_REF:-}"
1394 local digest=""
1395 local immutable_source=""
1396
1397 is_ci_base_target || return 0
1398 wants_stable_ci_base_tag || return 0
1399 if ! should_push_stable_ci_base_tag; then
1400 echo "Skipping ci_base stable promotion: build is not the trusted current main tip"
1401 return 0
1402 fi
1403 if [[ -z "${source_ref}" || -z "${stable_ref}" ]]; then
1404 echo "Cannot promote ci_base stable tag without source and destination refs" >&2
1405 return 1
1406 fi
1407 if ! digest=$(resolve_image_digest "${source_ref}"); then
1408 echo "Could not pin ci_base source before stable promotion: ${source_ref}" >&2
1409 return 1
1410 fi
1411 immutable_source="${source_ref%@*}@${digest}"
1412 echo "Promoting ci_base stable tag from ${immutable_source}"
1413 if ! docker buildx imagetools create --prefer-index=false \
1414 -t "${stable_ref}" "${immutable_source}"; then
1415 echo "Failed to promote ci_base stable tag" >&2
1416 return 1
1417 fi
1418 if ! confirm_remote_image_push "${stable_ref}"; then
1419 echo "Promoted ci_base stable tag did not become visible: ${stable_ref}" >&2
1420 return 1
1421 fi
1422 }
1423
1424 maybe_reuse_matching_ci_base_ref() {
1425 local matching_ref=""
1426
1427 matching_ref=$(find_matching_ci_base_ref) || return 1
1428
1429 echo "Found existing ci_base image with matching content hash: ${matching_ref}"
1430 if ! refresh_ci_base_tags_from_ref "${matching_ref}"; then
1431 echo "ci_base tag refresh failed after finding an exact image; aborting" >&2
1432 return 2
1433 fi
1434 if ! promote_stable_ci_base_tag "${matching_ref}"; then
1435 echo "ci_base stable promotion failed after finding an exact image; aborting" >&2
1436 return 2
1437 fi
1438 if ! publish_ci_base_handoff_ref "${matching_ref}"; then
1439 echo "ci_base handoff failed after finding an exact image; aborting" >&2
1440 return 2
1441 fi
1442 echo "Content hashes match -- ci_base is current"
1443 return 0
1444 }
1445
1446 maybe_skip_existing_image() {
1447 local remote_revision=""
1448 local reuse_status=0
1449
1450 if [[ -z "${IMAGE_TAG:-}" ]]; then
1451 return 0
1452 fi
1453
1454 if [[ "${FORCE_BUILD:-0}" == "1" ]]; then
1455 echo "FORCE_BUILD=1 set; skipping existing-image check"
1456 return 0
1457 fi
1458 if ! is_ci_base_target \
1459 && { should_upload_wheel_artifacts || should_export_rocm_smoke; }; then
1460 echo "Local-output targets always run for the current build"
1461 return 0
1462 fi
1463
1464 echo "--- :mag: Checking image tag"
1465 echo "Image tag: ${IMAGE_TAG}"
1466
1467 if is_ci_base_target && [[ -n "${CI_BASE_CONTENT_HASH:-}" ]]; then
1468 maybe_reuse_matching_ci_base_ref || reuse_status=$?
1469 case "${reuse_status}" in
1470 0)
1471 echo "Skipping build"
1472 exit 0
1473 ;;
1474 1)
1475 echo "No current ci_base image matched the expected content hash"
1476 echo "Proceeding with build"
1477 return 0
1478 ;;
1479 *)
1480 return "${reuse_status}"
1481 ;;
1482 esac
1483 fi
1484
1485 if ! remote_image_exists "${IMAGE_TAG}"; then
1486 echo "Image not found, proceeding with build"
1487 return 0
1488 fi
1489
1490 if is_ci_base_target; then
1491 echo "ci_base image already exists and no content hash was configured"
1492 echo "Skipping build"
1493 exit 0
1494 fi
1495
1496 if is_commit_image_target; then
1497 remote_revision=$(get_remote_image_label "${IMAGE_TAG}" "org.opencontainers.image.revision")
1498 if [[ -n "${remote_revision}" && "${remote_revision}" != "${BUILDKITE_COMMIT}" ]]; then
1499 echo "Existing image revision does not match ${BUILDKITE_COMMIT}"
1500 echo " found revision: ${remote_revision}"
1501 echo "Rebuilding image"
1502 return 0
1503 fi
1504
1505 echo "Commit image already exists: ${IMAGE_TAG}"
1506 echo "Skipping build"
1507 exit 0
1508 fi
1509
1510 echo "Image already exists: ${IMAGE_TAG}"
1511 echo "Skipping build"
1512 exit 0
1513 }
1514
1515 setup_builder() {
1516 echo "--- :buildkite: Setting up buildx builder"
1517
1518 local setup_mode="${ROCM_SETUP_BUILDX_BUILDER:-auto}"
1519 local current_driver=""
1520 local named_driver=""
1521
1522 if [[ "${setup_mode}" == "0" || "${setup_mode}" == "false" ]]; then
1523 echo "Using current Docker buildx builder"
1524 echo "ROCM_SETUP_BUILDX_BUILDER=${setup_mode}; cache exporters may fail if the driver is docker"
1525 docker buildx inspect --bootstrap
1526 echo "Active builder:"
1527 docker buildx ls | grep -E '^\*|^NAME' || docker buildx ls
1528 return 0
1529 fi
1530
1531 current_driver=$(buildx_driver || true)
1532 if [[ "${setup_mode}" != "1" ]] && builder_supports_registry_cache "${current_driver}"; then
1533 echo "Using current Docker buildx builder with ${current_driver} driver"
1534 docker buildx inspect --bootstrap
1535 echo "Active builder:"
1536 docker buildx ls | grep -E '^\*|^NAME' || docker buildx ls
1537 return 0
1538 fi
1539
1540 if [[ "${setup_mode}" != "1" ]]; then
1541 echo "Current buildx driver '${current_driver:-unknown}' cannot export registry caches"
1542 echo "Creating or using a cache-capable builder: ${BUILDER_NAME}"
1543 fi
1544
1545 if docker buildx inspect "${BUILDER_NAME}" >/dev/null 2>&1; then
1546 named_driver=$(buildx_driver "${BUILDER_NAME}" || true)
1547 if ! builder_supports_registry_cache "${named_driver}"; then
1548 echo "Builder '${BUILDER_NAME}' uses ${named_driver:-unknown} driver; using ${BUILDER_NAME}-cache instead"
1549 BUILDER_NAME="${BUILDER_NAME}-cache"
1550 fi
1551 fi
1552
1553 if [[ -S "${BUILDKIT_SOCKET}" ]]; then
1554 echo "Found local buildkitd socket at ${BUILDKIT_SOCKET}"
1555 echo "Using remote driver to connect to buildkitd"
1556
1557 if docker buildx inspect "${BUILDER_NAME}" >/dev/null 2>&1; then
1558 use_existing_builder
1559 else
1560 create_and_bootstrap_builder remote "unix://${BUILDKIT_SOCKET}"
1561 fi
1562 elif docker buildx inspect "${BUILDER_NAME}" >/dev/null 2>&1; then
1563 use_existing_builder
1564 else
1565 echo "No local buildkitd found, using docker-container driver"
1566 create_and_bootstrap_builder docker-container
1567 fi
1568
1569 echo "Active builder:"
1570 docker buildx ls | grep -E '^\*|^NAME' || docker buildx ls
1571 }
1572
1573 validate_cache_branch_tag() {
1574 local name="$1"
1575 local value="$2"
1576
1577 if [[ -n "${value}" \
1578 && (! "${value}" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]*$ \
1579 || ${#value} -gt ${ROCM_CACHE_BRANCH_TAG_MAX_LEN}) ]]; then
1580 echo "Invalid ${name}; expected a Docker tag component of at most ${ROCM_CACHE_BRANCH_TAG_MAX_LEN} characters" >&2
1581 return 1
1582 fi
1583 }
1584
1585 prepare_git_cache_metadata() {
1586 local cache_branch_name=""
1587 local cache_base_branch="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}"
1588 local target_repo_slug=""
1589 local target_repo_url=""
1590 local merge_base_ref=""
1591
1592 if is_ci_base_target; then
1593 echo "Skipping commit-cache ancestry lookup for content-addressed ci_base"
1594 return 0
1595 fi
1596
1597 if [[ -z "${PARENT_COMMIT:-}" || -z "${VLLM_MERGE_BASE_COMMIT:-}" ]] \
1598 && git rev-parse --is-shallow-repository 2>/dev/null | grep -q "true"; then
1599 echo "Shallow clone detected - deepening for cache key computation"
1600 git_fetch_for_cache --filter=tree:0 --no-tags --deepen=1 \
1601 origin "$(git rev-parse HEAD)" || true
1602 fi
1603
1604 if [[ -z "${PARENT_COMMIT:-}" ]]; then
1605 PARENT_COMMIT=$(git rev-parse HEAD~1 2>/dev/null || echo "")
1606 if [[ -n "${PARENT_COMMIT}" ]]; then
1607 export PARENT_COMMIT
1608 echo "Computed parent commit for cache fallback: ${PARENT_COMMIT}"
1609 else
1610 echo "Could not determine parent commit"
1611 fi
1612 else
1613 echo "Using provided PARENT_COMMIT: ${PARENT_COMMIT}"
1614 fi
1615
1616 if [[ -z "${ROCM_CACHE_BRANCH_TAG:-}" ]]; then
1617 cache_branch_name=$(select_cache_branch_name)
1618 if [[ -z "${cache_branch_name}" && "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then
1619 cache_branch_name="pr-${BUILDKITE_PULL_REQUEST}"
1620 echo "Using pull request number for ROCm branch cache tag: ${cache_branch_name}"
1621 fi
1622 fi
1623
1624 if [[ -z "${ROCM_CACHE_BRANCH_TAG:-}" && -n "${cache_branch_name}" ]]; then
1625 ROCM_CACHE_BRANCH_TAG=$(
1626 compose_cache_branch_tag "$(get_buildkite_repo_slug)" "${cache_branch_name}"
1627 )
1628 export ROCM_CACHE_BRANCH_TAG
1629 echo "Computed ROCm branch cache tag: ${ROCM_CACHE_BRANCH_TAG} (from ${cache_branch_name})"
1630 elif [[ -n "${ROCM_CACHE_BRANCH_TAG:-}" ]]; then
1631 echo "Using provided ROCM_CACHE_BRANCH_TAG: ${ROCM_CACHE_BRANCH_TAG}"
1632 elif [[ -n "${BUILDKITE_BRANCH:-}" ]]; then
1633 echo "Skipping ROCm branch cache tag: no usable branch name found"
1634 echo " BUILDKITE_BRANCH=${BUILDKITE_BRANCH}"
1635 fi
1636
1637 if [[ -z "${ROCM_CACHE_UPSTREAM_BRANCH_TAG:-}" \
1638 && -n "${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-}" \
1639 && "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then
1640 target_repo_slug=$(get_buildkite_target_repo_slug)
1641 ROCM_CACHE_UPSTREAM_BRANCH_TAG=$(
1642 compose_cache_branch_tag "${target_repo_slug}" "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}"
1643 )
1644 export ROCM_CACHE_UPSTREAM_BRANCH_TAG
1645 echo "Computed ROCm upstream branch cache tag: ${ROCM_CACHE_UPSTREAM_BRANCH_TAG}"
1646 elif [[ -n "${ROCM_CACHE_UPSTREAM_BRANCH_TAG:-}" ]]; then
1647 echo "Using provided ROCM_CACHE_UPSTREAM_BRANCH_TAG: ${ROCM_CACHE_UPSTREAM_BRANCH_TAG}"
1648 fi
1649
1650 validate_cache_branch_tag \
1651 ROCM_CACHE_BRANCH_TAG "${ROCM_CACHE_BRANCH_TAG:-}"
1652 validate_cache_branch_tag \
1653 ROCM_CACHE_UPSTREAM_BRANCH_TAG "${ROCM_CACHE_UPSTREAM_BRANCH_TAG:-}"
1654
1655 if [[ -z "${VLLM_MERGE_BASE_COMMIT:-}" ]]; then
1656 target_repo_url=$(get_buildkite_target_repo_url)
1657 merge_base_ref="refs/remotes/vllm-cache-upstream/${cache_base_branch}"
1658 git_fetch_for_cache --no-tags --depth=200 "${target_repo_url}" \
1659 "+refs/heads/${cache_base_branch}:${merge_base_ref}" 2>/dev/null || true
1660 VLLM_MERGE_BASE_COMMIT=$(git merge-base HEAD "${merge_base_ref}" 2>/dev/null || echo "")
1661 if [[ -z "${VLLM_MERGE_BASE_COMMIT}" ]]; then
1662 git_fetch_for_cache --no-tags --deepen=1000 "${target_repo_url}" \
1663 "+refs/heads/${cache_base_branch}:${merge_base_ref}" 2>/dev/null || true
1664 VLLM_MERGE_BASE_COMMIT=$(git merge-base HEAD "${merge_base_ref}" 2>/dev/null || echo "")
1665 fi
1666 if [[ -n "${VLLM_MERGE_BASE_COMMIT}" ]]; then
1667 export VLLM_MERGE_BASE_COMMIT
1668 echo "Computed merge base commit for cache fallback: ${VLLM_MERGE_BASE_COMMIT}"
1669 else
1670 echo "Could not determine merge base with ${cache_base_branch}"
1671 fi
1672 else
1673 echo "Using provided VLLM_MERGE_BASE_COMMIT: ${VLLM_MERGE_BASE_COMMIT}"
1674 fi
1675 }
1676
1677 ci_base_metadata_pairs() {
1678 local dockerfile="${CI_BASE_DOCKERFILE:-${DEFAULT_CI_BASE_DOCKERFILE}}"
1679 local stages="${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}"
1680 local content_files="${CI_BASE_CONTENT_FILES:-${DEFAULT_CI_BASE_CONTENT_FILES}}"
1681 local content_files_hash=""
1682 local base_image=""
1683 local base_image_digest=""
1684 local -a content_paths=()
1685 local -a content_args=()
1686
1687 read -r -a content_paths <<< "${content_files}"
1688 if [[ ${#content_paths[@]} -gt 0 ]]; then
1689 if ! content_files_hash=$(compute_content_hash "${content_paths[@]}"); then
1690 echo "Failed to hash ci_base metadata content files" >&2
1691 return 1
1692 fi
1693 fi
1694 mapfile -t content_args < <(
1695 get_content_arg_names "${dockerfile}" "${stages}" "${CI_BASE_CONTENT_ARGS:-}"
1696 )
1697
1698 base_image=$(resolve_dockerfile_arg_value "${dockerfile}" "BASE_IMAGE")
1699 if [[ -n "${base_image}" ]]; then
1700 if ! base_image_digest=$(resolve_image_digest "${base_image}"); then
1701 echo "Failed to resolve ci_base metadata digest for ${base_image}" >&2
1702 return 1
1703 fi
1704 fi
1705
1706 metadata_pair "vllm.ci_base.metadata_version" "${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}"
1707 metadata_pair "vllm.ci_base.content_hash" "${CI_BASE_CONTENT_HASH:-}"
1708 metadata_pair "vllm.ci_base.content_files_hash" "${content_files_hash}"
1709 metadata_pair "vllm.ci_base.content_files" "${content_files}"
1710 metadata_pair "vllm.ci_base.content_args" "$(join_words "${content_args[@]}")"
1711 metadata_pair "vllm.ci_base.dockerfile" "${dockerfile}"
1712 metadata_pair "vllm.ci_base.dockerfile_stages" "${stages}"
1713
1714 # The parent identity is digest-only: mutable aliases that resolve to the
1715 # same image must produce byte-identical canonical metadata.
1716 metadata_pair "vllm.rocm.base_image" "${base_image_digest}"
1717 metadata_pair "vllm.rocm.base_image_digest" "${base_image_digest}"
1718 metadata_pair "vllm.rocm.pytorch_rocm_arch" "${PYTORCH_ROCM_ARCH:-}"
1719 metadata_pair "vllm.rocm.nic_backend" "$(resolve_dockerfile_arg_value "${dockerfile}" "NIC_BACKEND")"
1720 metadata_pair "vllm.rocm.ainic_version" "$(resolve_dockerfile_arg_value "${dockerfile}" "AINIC_VERSION")"
1721 metadata_pair "vllm.rocm.ubuntu_codename" "$(resolve_dockerfile_arg_value "${dockerfile}" "UBUNTU_CODENAME")"
1722 metadata_pair "vllm.rocm.max_jobs" "$(resolve_dockerfile_arg_value "${dockerfile}" "max_jobs")"
1723 metadata_pair "vllm.rocm.nixl_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "NIXL_REPO")"
1724 metadata_pair "vllm.rocm.nixl_commit" "${NIXL_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "NIXL_BRANCH")}"
1725 metadata_pair "vllm.rocm.ucx_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "UCX_REPO")"
1726 metadata_pair "vllm.rocm.ucx_commit" "${UCX_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "UCX_BRANCH")}"
1727 metadata_pair "vllm.rocm.rocshmem_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "ROCSHMEM_REPO")"
1728 metadata_pair "vllm.rocm.rocshmem_commit" "${ROCSHMEM_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "ROCSHMEM_BRANCH")}"
1729 metadata_pair "vllm.rocm.deepep_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_REPO")"
1730 metadata_pair "vllm.rocm.deepep_commit" "${DEEPEP_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_BRANCH")}"
1731 metadata_pair "vllm.rocm.deepep_nic" "$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_NIC")"
1732 metadata_pair "vllm.rocm.deepep_rocm_arch" "$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_ROCM_ARCH")"
1733 metadata_pair "vllm.rocm.nixl_cache_key" "${NIXL_CACHE_KEY:-}"
1734 metadata_pair "vllm.rocm.rocshmem_cache_key" "${ROCSHMEM_CACHE_KEY:-}"
1735 metadata_pair "vllm.rocm.deepep_cache_key" "${DEEPEP_CACHE_KEY:-}"
1736 }
1737
1738 write_ci_base_metadata_annotations() {
1739 local metadata="$1"
1740 local key=""
1741 local value=""
1742
1743 [[ -n "${metadata}" ]] || return 0
1744 while IFS=$'\t' read -r key value; do
1745 [[ -n "${key}" && -n "${value}" ]] || continue
1746 printf ' "%s",\n' \
1747 "$(hcl_escape_string "manifest:${key}=${value}")"
1748 done <<< "${metadata}"
1749 }
1750
1751 write_ci_base_metadata_labels() {
1752 local metadata="$1"
1753 local key=""
1754 local value=""
1755
1756 [[ -n "${metadata}" ]] || return 0
1757 while IFS=$'\t' read -r key value; do
1758 [[ -n "${key}" && -n "${value}" ]] || continue
1759 printf ' "%s" = "%s"\n' \
1760 "$(hcl_escape_string "${key}")" \
1761 "$(hcl_escape_string "${value}")"
1762 done <<< "${metadata}"
1763 }
1764
1765 write_ci_base_label_override() {
1766 local target_name=""
1767 local metadata=""
1768 local -a ci_base_targets=()
1769
1770 if [[ -z "${CI_BASE_CONTENT_HASH:-}" ]]; then
1771 return 0
1772 fi
1773
1774 mapfile -t ci_base_targets < <(
1775 {
1776 printf '%s\n' "ci-base-rocm"
1777 sed -n -E 's/^target "(ci-base-rocm[^"]+)".*/\1/p' "${CI_HCL_PATH}" 2>/dev/null || true
1778 } | awk '!seen[$0]++'
1779 )
1780
1781 if [[ ${#ci_base_targets[@]} -eq 0 ]]; then
1782 return 0
1783 fi
1784
1785 metadata=$(ci_base_metadata_pairs)
1786
1787 : > "${CI_BASE_LABEL_OVERRIDE_PATH}"
1788 for target_name in "${ci_base_targets[@]}"; do
1789 cat >> "${CI_BASE_LABEL_OVERRIDE_PATH}" <<EOF
1790 target "${target_name}" {
1791 annotations = [
1792 "manifest:org.opencontainers.image.revision=",
1793 EOF
1794 write_ci_base_metadata_annotations "${metadata}" >> "${CI_BASE_LABEL_OVERRIDE_PATH}"
1795 cat >> "${CI_BASE_LABEL_OVERRIDE_PATH}" <<EOF
1796 ]
1797 labels = {
1798 "org.opencontainers.image.revision" = ""
1799 EOF
1800 write_ci_base_metadata_labels "${metadata}" >> "${CI_BASE_LABEL_OVERRIDE_PATH}"
1801 cat >> "${CI_BASE_LABEL_OVERRIDE_PATH}" <<EOF
1802 }
1803 }
1804
1805 EOF
1806 done
1807
1808 BAKE_FILES+=(-f "${CI_BASE_LABEL_OVERRIDE_PATH}")
1809 echo "Appended ci_base metadata label override for targets: ${ci_base_targets[*]}"
1810 }
1811
1812 uses_rocm_csrc_cache() {
1813 case "${TARGET}" in
1814 csrc-rocm-ci \
1815 | test-rocm-ci \
1816 | test-rocm-ci-with-wheel \
1817 | test-rocm-ci-with-artifacts \
1818 | export-wheel-rocm \
1819 | smoke-test-rocm-ci)
1820 return 0
1821 ;;
1822 *)
1823 return 1
1824 ;;
1825 esac
1826 }
1827
1828 uses_rocm_rust_cache() {
1829 case "${TARGET}" in
1830 rust-rocm-ci \
1831 | test-rocm-ci \
1832 | test-rocm-ci-with-wheel \
1833 | test-rocm-ci-with-artifacts \
1834 | export-wheel-rocm \
1835 | smoke-test-rocm-ci)
1836 return 0
1837 ;;
1838 *)
1839 return 1
1840 ;;
1841 esac
1842 }
1843
1844 compute_rocm_csrc_content_hash() {
1845 local bake_dir=""
1846 local dockerfile_rocm=""
1847 local content_files="${ROCM_CSRC_CONTENT_FILES:-${DEFAULT_ROCM_CSRC_CONTENT_FILES}}"
1848 local stages="${ROCM_CSRC_DOCKERFILE_STAGES:-${DEFAULT_ROCM_CSRC_DOCKERFILE_STAGES}}"
1849 local -a content_paths=()
1850 local -a content_args=()
1851
1852 bake_dir=$(dirname "${VLLM_BAKE_FILE}")
1853 dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}"
1854 read -r -a content_paths <<< "${content_files}"
1855 mapfile -t content_args < <(
1856 get_content_arg_names "${dockerfile_rocm}" "${stages}" "${ROCM_CSRC_CONTENT_ARGS:-}"
1857 )
1858
1859 {
1860 printf 'csrc-input-files-hash:%s\n' "$(compute_content_hash "${content_paths[@]}")"
1861 printf 'dockerfile:%s\n' "${dockerfile_rocm}"
1862 printf 'resolved-build-args:\n'
1863 hash_dockerfile_arg_values "${dockerfile_rocm}" "${content_args[@]}"
1864 printf 'dockerfile-stages:%s\n' "${stages}"
1865 if content_regular_file "${dockerfile_rocm}"; then
1866 hash_dockerfile_stages "${dockerfile_rocm}" "${stages}"
1867 else
1868 printf 'missing:%s\n' "${dockerfile_rocm}"
1869 fi
1870 } | sha256sum | cut -d' ' -f1
1871 }
1872
1873 compute_rocm_csrc_content_hash_if_needed() {
1874 local cache_repo="${DOCKERHUB_CACHE_REPO:-rocm/vllm-ci-cache}"
1875 local write_scope="${CI_BASE_WRITE_SCOPE:-}"
1876
1877 if [[ "${ROCM_CSRC_CONTENT_CACHE:-1}" == "0" ]] || ! uses_rocm_csrc_cache; then
1878 return 0
1879 fi
1880
1881 ROCM_CSRC_CONTENT_HASH=$(compute_rocm_csrc_content_hash)
1882 ROCM_CSRC_TRUSTED_CONTENT_CACHE_REF="${cache_repo}:csrc-rocm-input-${ROCM_CSRC_CONTENT_HASH}"
1883 ROCM_CSRC_CONTENT_CACHE_REF="${ROCM_CSRC_TRUSTED_CONTENT_CACHE_REF}"
1884 if [[ -n "${write_scope}" ]]; then
1885 ROCM_CSRC_CONTENT_CACHE_REF="${ROCM_CSRC_CONTENT_CACHE_REF}-${write_scope}"
1886 fi
1887 export ROCM_CSRC_CONTENT_HASH
1888 export ROCM_CSRC_TRUSTED_CONTENT_CACHE_REF
1889 export ROCM_CSRC_CONTENT_CACHE_REF
1890 echo "ROCm csrc content cache ref: ${ROCM_CSRC_CONTENT_CACHE_REF}"
1891 }
1892
1893 compute_rocm_rust_content_hash() {
1894 local bake_dir=""
1895 local dockerfile_rocm=""
1896 local content_files="${ROCM_RUST_CONTENT_FILES:-${DEFAULT_ROCM_RUST_CONTENT_FILES}}"
1897 local stages="${ROCM_RUST_DOCKERFILE_STAGES:-${DEFAULT_ROCM_RUST_DOCKERFILE_STAGES}}"
1898 local -a content_paths=()
1899 local -a content_args=()
1900
1901 bake_dir=$(dirname "${VLLM_BAKE_FILE}")
1902 dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}"
1903 read -r -a content_paths <<< "${content_files}"
1904 mapfile -t content_args < <(
1905 get_content_arg_names \
1906 "${dockerfile_rocm}" "${stages}" "${ROCM_RUST_CONTENT_ARGS:-}"
1907 )
1908
1909 {
1910 printf 'rust-input-files-hash:%s\n' "$(compute_content_hash "${content_paths[@]}")"
1911 printf 'dockerfile:%s\n' "${dockerfile_rocm}"
1912 printf 'resolved-build-args:\n'
1913 hash_dockerfile_arg_values "${dockerfile_rocm}" "${content_args[@]}"
1914 printf 'dockerfile-stages:%s\n' "${stages}"
1915 if content_regular_file "${dockerfile_rocm}"; then
1916 hash_dockerfile_stages "${dockerfile_rocm}" "${stages}"
1917 else
1918 printf 'missing:%s\n' "${dockerfile_rocm}"
1919 fi
1920 } | sha256sum | cut -d' ' -f1
1921 }
1922
1923 compute_rocm_rust_content_hash_if_needed() {
1924 local cache_repo="${DOCKERHUB_CACHE_REPO:-rocm/vllm-ci-cache}"
1925 local write_scope="${CI_BASE_WRITE_SCOPE:-}"
1926
1927 if [[ "${ROCM_RUST_CONTENT_CACHE:-1}" == "0" ]] || ! uses_rocm_rust_cache; then
1928 return 0
1929 fi
1930
1931 ROCM_RUST_CONTENT_HASH=$(compute_rocm_rust_content_hash)
1932 ROCM_RUST_TRUSTED_CONTENT_CACHE_REF="${cache_repo}:rust-rocm-input-${ROCM_RUST_CONTENT_HASH}"
1933 ROCM_RUST_CONTENT_CACHE_REF="${ROCM_RUST_TRUSTED_CONTENT_CACHE_REF}"
1934 if [[ -n "${write_scope}" ]]; then
1935 ROCM_RUST_CONTENT_CACHE_REF="${ROCM_RUST_CONTENT_CACHE_REF}-${write_scope}"
1936 fi
1937 export ROCM_RUST_CONTENT_HASH
1938 export ROCM_RUST_TRUSTED_CONTENT_CACHE_REF
1939 export ROCM_RUST_CONTENT_CACHE_REF
1940 echo "ROCm Rust content cache ref: ${ROCM_RUST_CONTENT_CACHE_REF}"
1941 }
1942
1943 write_hcl_string_list_entries() {
1944 local indent="$1"
1945 local value=""
1946 shift
1947
1948 for value in "$@"; do
1949 value="${value//\\/\\\\}"
1950 value="${value//\"/\\\"}"
1951 printf '%s"%s",\n' "${indent}" "${value}"
1952 done
1953 }
1954
1955 hcl_escape_string() {
1956 local value="$1"
1957
1958 value="${value//\\/\\\\}"
1959 value="${value//\"/\\\"}"
1960 printf '%s' "${value}"
1961 }
1962
1963 join_words() {
1964 local IFS=" "
1965 printf '%s' "$*"
1966 }
1967
1968 metadata_pair() {
1969 local key="$1"
1970 local value="${2:-}"
1971
1972 printf '%s\t%s\n' "${key}" "${value}"
1973 }
1974
1975 write_hcl_string_list() {
1976 local indent="$1"
1977 shift
1978
1979 printf '%s[\n' "${indent}"
1980 write_hcl_string_list_entries "${indent} " "$@"
1981 printf '%s]\n' "${indent}"
1982 }
1983
1984 write_rocm_build_arg_override() {
1985 local bake_dir=""
1986 local dockerfile_rocm=""
1987 local -a arg_names=()
1988 local arg_name=""
1989 local arg_value=""
1990
1991 bake_dir=$(dirname "${VLLM_BAKE_FILE}")
1992 dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}"
1993 mapfile -t arg_names < <(
1994 {
1995 get_content_arg_names \
1996 "${dockerfile_rocm}" \
1997 "${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}" \
1998 "${CI_BASE_CONTENT_ARGS:-}"
1999 get_content_arg_names \
2000 "${dockerfile_rocm}" \
2001 "${ROCM_CSRC_DOCKERFILE_STAGES:-${DEFAULT_ROCM_CSRC_DOCKERFILE_STAGES}}" \
2002 "${ROCM_CSRC_CONTENT_ARGS:-}"
2003 get_content_arg_names \
2004 "${dockerfile_rocm}" \
2005 "${ROCM_RUST_DOCKERFILE_STAGES:-${DEFAULT_ROCM_RUST_DOCKERFILE_STAGES}}" \
2006 "${ROCM_RUST_CONTENT_ARGS:-}"
2007 } | awk 'NF && !seen[$0]++'
2008 )
2009
2010 {
2011 cat <<EOF
2012 target "_common-rocm" {
2013 args = {
2014 EOF
2015 for arg_name in "${arg_names[@]}"; do
2016 [[ -n "${arg_name}" ]] || continue
2017 arg_value=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "${arg_name}")
2018 [[ -n "${arg_value}" ]] || continue
2019 printf ' %s = "%s"\n' "${arg_name}" "$(hcl_escape_string "${arg_value}")"
2020 done
2021 cat <<EOF
2022 }
2023 }
2024 EOF
2025 } > "${ROCM_ARG_OVERRIDE_PATH}"
2026
2027 BAKE_FILES+=(-f "${ROCM_ARG_OVERRIDE_PATH}")
2028 echo "Appended resolved ROCm Docker ARG override"
2029 }
2030
2031 write_hcl_string_list_attr() {
2032 local indent="$1"
2033 local attr="$2"
2034 shift 2
2035
2036 printf '%s%s = [\n' "${indent}" "${attr}"
2037 write_hcl_string_list_entries "${indent} " "$@"
2038 printf '%s]\n' "${indent}"
2039 }
2040
2041 validate_cache_export_mode() {
2042 local mode="$1"
2043 local env_name="$2"
2044
2045 case "${mode}" in
2046 min|max)
2047 ;;
2048 *)
2049 echo "Error: ${env_name} must be one of: min, max"
2050 exit 1
2051 ;;
2052 esac
2053 }
2054
2055 validate_content_cache_export_mode() {
2056 local mode="$1"
2057 local env_name="$2"
2058
2059 case "${mode}" in
2060 missing|always|never)
2061 ;;
2062 *)
2063 echo "Error: ${env_name} must be one of: missing, always, never"
2064 exit 1
2065 ;;
2066 esac
2067 }
2068
2069 should_export_content_cache_ref() {
2070 local cache_ref="$1"
2071 local cache_name="$2"
2072 local trusted_ref="${3:-${cache_ref}}"
2073 local mode="${ROCM_CONTENT_CACHE_EXPORT_MODE:-missing}"
2074
2075 case "${mode}" in
2076 always)
2077 echo "${cache_name} content cache export mode is always"
2078 return 0
2079 ;;
2080 never)
2081 echo "${cache_name} content cache export mode is never"
2082 return 1
2083 ;;
2084 missing|"")
2085 if registry_ref_exists_with_retry imagetools "${trusted_ref}"; then
2086 echo "${cache_name} trusted content cache is visible: ${trusted_ref}"
2087 return 1
2088 fi
2089 if [[ "${cache_ref}" != "${trusted_ref}" ]] \
2090 && registry_ref_exists_with_retry imagetools "${cache_ref}"; then
2091 echo "${cache_name} scoped content cache is visible: ${cache_ref}"
2092 return 1
2093 fi
2094 echo "${cache_name} content cache is missing: ${cache_ref}"
2095 return 0
2096 ;;
2097 *)
2098 echo "Error: ROCM_CONTENT_CACHE_EXPORT_MODE must be one of: missing, always, never"
2099 exit 1
2100 ;;
2101 esac
2102 }
2103
2104 write_rocm_cache_override() {
2105 local cache_repo="${DOCKERHUB_CACHE_REPO:-rocm/vllm-ci-cache}"
2106 local content_cache_export_mode="${ROCM_CONTENT_CACHE_EXPORT_MODE:-missing}"
2107 local csrc_cache_to_mode="${ROCM_CSRC_CACHE_TO_MODE:-max}"
2108 local rust_cache_to_mode="${ROCM_RUST_CACHE_TO_MODE:-max}"
2109 local rocm_cache_to_mode="${ROCM_FINAL_CACHE_TO_MODE:-min}"
2110 local -a csrc_content_cache_from=()
2111 local -a rust_content_cache_from=()
2112 local -a combined_content_cache_from=()
2113 local -a csrc_cache_to=()
2114 local -a rust_cache_to=()
2115 local -a rocm_cache_to=()
2116 local -a export_wheel_cache_to=()
2117 local export_csrc_cache=1
2118 local export_rust_cache=1
2119
2120 if ! uses_rocm_csrc_cache && ! uses_rocm_rust_cache; then
2121 return 0
2122 fi
2123
2124 validate_content_cache_export_mode \
2125 "${content_cache_export_mode}" \
2126 "ROCM_CONTENT_CACHE_EXPORT_MODE"
2127 validate_cache_export_mode "${csrc_cache_to_mode}" "ROCM_CSRC_CACHE_TO_MODE"
2128 validate_cache_export_mode "${rust_cache_to_mode}" "ROCM_RUST_CACHE_TO_MODE"
2129 validate_cache_export_mode "${rocm_cache_to_mode}" "ROCM_FINAL_CACHE_TO_MODE"
2130 echo "ROCm content cache export mode: ${content_cache_export_mode}"
2131 echo "ROCm csrc cache export mode: ${csrc_cache_to_mode}"
2132 echo "ROCm Rust fallback cache export mode: ${rust_cache_to_mode}"
2133 echo "ROCm final image cache export mode: ${rocm_cache_to_mode}"
2134
2135 if [[ -n "${ROCM_CSRC_CONTENT_CACHE_REF:-}" ]]; then
2136 csrc_content_cache_from+=(
2137 "type=registry,ref=${ROCM_CSRC_TRUSTED_CONTENT_CACHE_REF}"
2138 )
2139 if [[ "${ROCM_CSRC_CONTENT_CACHE_REF}" != \
2140 "${ROCM_CSRC_TRUSTED_CONTENT_CACHE_REF}" ]]; then
2141 csrc_content_cache_from+=(
2142 "type=registry,ref=${ROCM_CSRC_CONTENT_CACHE_REF}"
2143 )
2144 fi
2145 if should_export_content_cache_ref \
2146 "${ROCM_CSRC_CONTENT_CACHE_REF}" "ROCm csrc" \
2147 "${ROCM_CSRC_TRUSTED_CONTENT_CACHE_REF}"; then
2148 csrc_cache_to+=(
2149 "type=registry,ref=${ROCM_CSRC_CONTENT_CACHE_REF},mode=${csrc_cache_to_mode},ignore-error=true"
2150 )
2151 else
2152 export_csrc_cache=0
2153 fi
2154 fi
2155
2156 if [[ -n "${ROCM_RUST_CONTENT_CACHE_REF:-}" ]]; then
2157 rust_content_cache_from+=(
2158 "type=registry,ref=${ROCM_RUST_TRUSTED_CONTENT_CACHE_REF}"
2159 )
2160 if [[ "${ROCM_RUST_CONTENT_CACHE_REF}" != \
2161 "${ROCM_RUST_TRUSTED_CONTENT_CACHE_REF}" ]]; then
2162 rust_content_cache_from+=(
2163 "type=registry,ref=${ROCM_RUST_CONTENT_CACHE_REF}"
2164 )
2165 fi
2166 # Legacy commit/branch exports are only needed while the exact-input
2167 # content ref is absent. The exact ref itself is refreshed below.
2168 if ! should_export_content_cache_ref \
2169 "${ROCM_RUST_CONTENT_CACHE_REF}" "ROCm Rust" \
2170 "${ROCM_RUST_TRUSTED_CONTENT_CACHE_REF}"; then
2171 export_rust_cache=0
2172 fi
2173 if [[ "${content_cache_export_mode}" != "never" ]]; then
2174 # Refresh the exact-input ref in the original solve regardless of
2175 # which local or remote cache supplied the Rust result.
2176 rust_cache_to+=(
2177 "type=registry,ref=${ROCM_RUST_CONTENT_CACHE_REF},mode=min,ignore-error=true"
2178 )
2179 echo "ROCm Rust exact-input cache will be refreshed (mode=min): ${ROCM_RUST_CONTENT_CACHE_REF}"
2180 fi
2181 fi
2182
2183 combined_content_cache_from=("${csrc_content_cache_from[@]}" "${rust_content_cache_from[@]}")
2184
2185 # Docker Hub cache exports are best-effort. A cache-only target failure can
2186 # otherwise cancel the sibling image target before its manifest is pushed.
2187 if [[ -n "${BUILDKITE_COMMIT:-}" ]]; then
2188 if [[ ${export_csrc_cache} -eq 1 ]]; then
2189 csrc_cache_to+=(
2190 "type=registry,ref=${cache_repo}:csrc-rocm-${BUILDKITE_COMMIT},mode=${csrc_cache_to_mode},ignore-error=true"
2191 )
2192 fi
2193 if [[ ${export_rust_cache} -eq 1 ]]; then
2194 rust_cache_to+=(
2195 "type=registry,ref=${cache_repo}:rust-rocm-${BUILDKITE_COMMIT},mode=${rust_cache_to_mode},ignore-error=true"
2196 )
2197 fi
2198 rocm_cache_to+=(
2199 "type=registry,ref=${cache_repo}:rocm-${BUILDKITE_COMMIT},mode=${rocm_cache_to_mode},ignore-error=true"
2200 )
2201 fi
2202
2203 if [[ -n "${ROCM_CACHE_BRANCH_TAG:-}" ]]; then
2204 if [[ ${export_csrc_cache} -eq 1 ]]; then
2205 csrc_cache_to+=(
2206 "type=registry,ref=${cache_repo}:csrc-rocm-branch-${ROCM_CACHE_BRANCH_TAG},mode=${csrc_cache_to_mode},ignore-error=true"
2207 )
2208 fi
2209 if [[ ${export_rust_cache} -eq 1 ]]; then
2210 rust_cache_to+=(
2211 "type=registry,ref=${cache_repo}:rust-rocm-branch-${ROCM_CACHE_BRANCH_TAG},mode=${rust_cache_to_mode},ignore-error=true"
2212 )
2213 fi
2214 rocm_cache_to+=(
2215 "type=registry,ref=${cache_repo}:rocm-branch-${ROCM_CACHE_BRANCH_TAG},mode=${rocm_cache_to_mode},ignore-error=true"
2216 )
2217 fi
2218
2219 # Standalone image/wheel targets reach rust-build but not the cache-only
2220 # exporter unless it is requested explicitly.
2221 if ((${#rust_cache_to[@]} > 0)); then
2222 case "${TARGET}" in
2223 test-rocm-ci|export-wheel-rocm|smoke-test-rocm-ci)
2224 BAKE_TARGETS=("rust-rocm-ci" "${BAKE_TARGETS[@]}")
2225 ;;
2226 esac
2227 fi
2228
2229 if [[ "${TARGET}" == "test-rocm-ci-with-wheel" ]]; then
2230 export_wheel_cache_to=()
2231 else
2232 export_wheel_cache_to=("${rocm_cache_to[@]}")
2233 fi
2234
2235 {
2236 cat <<EOF
2237 target "csrc-rocm-ci" {
2238 cache-from = concat(
2239 get_cache_from_rocm_csrc(),
2240 EOF
2241 write_hcl_string_list " " "${csrc_content_cache_from[@]}"
2242 cat <<EOF
2243 )
2244 EOF
2245 write_hcl_string_list_attr " " "cache-to" "${csrc_cache_to[@]}"
2246 cat <<EOF
2247 }
2248
2249 target "rust-rocm-ci" {
2250 cache-from = concat(
2251 get_cache_from_rocm_rust(),
2252 EOF
2253 write_hcl_string_list " " "${rust_content_cache_from[@]}"
2254 cat <<EOF
2255 )
2256 EOF
2257 write_hcl_string_list_attr " " "cache-to" "${rust_cache_to[@]}"
2258 cat <<EOF
2259 }
2260
2261 target "test-rocm-ci" {
2262 cache-from = concat(
2263 get_cache_from_rocm(),
2264 EOF
2265 write_hcl_string_list " " "${combined_content_cache_from[@]}"
2266 cat <<EOF
2267 )
2268 EOF
2269 write_hcl_string_list_attr " " "cache-to" "${rocm_cache_to[@]}"
2270 cat <<EOF
2271 }
2272
2273 target "smoke-test-rocm-ci" {
2274 cache-from = concat(
2275 get_cache_from_rocm(),
2276 EOF
2277 write_hcl_string_list " " "${combined_content_cache_from[@]}"
2278 cat <<EOF
2279 )
2280 }
2281
2282 target "export-wheel-rocm" {
2283 cache-from = concat(
2284 get_cache_from_rocm(),
2285 EOF
2286 write_hcl_string_list " " "${combined_content_cache_from[@]}"
2287 cat <<EOF
2288 )
2289 EOF
2290 write_hcl_string_list_attr " " "cache-to" "${export_wheel_cache_to[@]}"
2291 cat <<EOF
2292 }
2293 EOF
2294 } > "${CSRC_CACHE_OVERRIDE_PATH}"
2295
2296 BAKE_FILES+=(-f "${CSRC_CACHE_OVERRIDE_PATH}")
2297 echo "Appended ROCm cache override with non-fatal registry exports"
2298 }
2299
2300 extract_dependency_pins() {
2301 local bake_dir=""
2302 local dockerfile_rocm=""
2303 local physical_dockerfile=""
2304 local var=""
2305 local val=""
2306
2307 bake_dir=$(dirname "${VLLM_BAKE_FILE}")
2308 dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}"
2309 physical_dockerfile="${dockerfile_rocm}"
2310 if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${dockerfile_rocm}" != /* ]]; then
2311 physical_dockerfile="${ROCM_BUILD_CONTEXT_ROOT}/${dockerfile_rocm}"
2312 fi
2313 if [[ ! -f "${physical_dockerfile}" ]]; then
2314 return 0
2315 fi
2316
2317 for var in NIXL_BRANCH UCX_BRANCH ROCSHMEM_BRANCH DEEPEP_BRANCH; do
2318 if [[ -n "${!var:-}" ]]; then
2319 echo "Using provided ${var}: ${!var}"
2320 continue
2321 fi
2322
2323 val=$(
2324 sed -n -E "s/^[[:space:]]*[Aa][Rr][Gg][[:space:]]+${var}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
2325 "${physical_dockerfile}" | head -1
2326 )
2327 if [[ -n "${val}" ]]; then
2328 export "${var}=${val}"
2329 echo "Extracted ${var}=${val} from ${dockerfile_rocm}"
2330 fi
2331 done
2332 }
2333
2334 compute_dependency_cache_keys() {
2335 local bake_dir=""
2336 local dockerfile_rocm=""
2337 local nixl_branch=""
2338 local ucx_branch=""
2339 local rocshmem_branch=""
2340 local deepep_branch=""
2341 local nixl_material=""
2342 local rocshmem_material=""
2343 local deepep_material=""
2344
2345 bake_dir=$(dirname "${VLLM_BAKE_FILE}")
2346 dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}"
2347 nixl_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "NIXL_BRANCH")
2348 ucx_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "UCX_BRANCH")
2349 rocshmem_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "ROCSHMEM_BRANCH")
2350 deepep_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "DEEPEP_BRANCH")
2351
2352 if [[ -n "${nixl_branch}" && -n "${ucx_branch}" ]]; then
2353 nixl_material=$(compose_stage_cache_material "${dockerfile_rocm}" "base build_nixl")
2354 NIXL_CACHE_KEY=$(
2355 compose_dependency_cache_key \
2356 "${nixl_branch}-ucx-${ucx_branch}" \
2357 "${nixl_material}"
2358 )
2359 export NIXL_CACHE_KEY
2360 echo "NIXL dependency cache key: ${NIXL_CACHE_KEY}"
2361 fi
2362
2363 if [[ -n "${rocshmem_branch}" ]]; then
2364 rocshmem_material=$(compose_stage_cache_material "${dockerfile_rocm}" "base build_rocshmem")
2365 ROCSHMEM_CACHE_KEY=$(
2366 compose_dependency_cache_key \
2367 "${rocshmem_branch}" \
2368 "${rocshmem_material}"
2369 )
2370 export ROCSHMEM_CACHE_KEY
2371 echo "ROCShmem dependency cache key: ${ROCSHMEM_CACHE_KEY}"
2372 fi
2373
2374 if [[ -n "${deepep_branch}" && -n "${rocshmem_branch}" ]]; then
2375 deepep_material=$(compose_stage_cache_material "${dockerfile_rocm}" "base build_rocshmem build_deepep")
2376 DEEPEP_CACHE_KEY=$(
2377 compose_dependency_cache_key \
2378 "${deepep_branch}-rocshmem-${rocshmem_branch}" \
2379 "${deepep_material}"
2380 )
2381 export DEEPEP_CACHE_KEY
2382 echo "DeepEP dependency cache key: ${DEEPEP_CACHE_KEY}"
2383 fi
2384 }
2385
2386 compose_stage_cache_material() {
2387 local dockerfile="$1"
2388 local stages="$2"
2389 local -a content_args=()
2390
2391 mapfile -t content_args < <(get_content_arg_names "${dockerfile}" "${stages}" "")
2392 {
2393 printf 'dockerfile:%s\n' "${dockerfile}"
2394 printf 'dockerfile-stages:%s\n' "${stages}"
2395 hash_dockerfile_stages "${dockerfile}" "${stages}"
2396 printf 'resolved-build-args:\n'
2397 hash_dockerfile_arg_values "${dockerfile}" "${content_args[@]}"
2398 }
2399 }
2400
2401 dependency_cache_ref_exists() {
2402 local cache_ref="$1"
2403 registry_ref_exists_with_retry imagetools "${cache_ref}"
2404 }
2405
2406 dependency_cache_ref_for_target() {
2407 local target="$1"
2408 local cache_repo="${DOCKERHUB_CACHE_REPO:-rocm/vllm-ci-cache}"
2409
2410 case "${target}" in
2411 nixl-rocm-ci)
2412 if [[ -n "${NIXL_CACHE_KEY:-}" ]]; then
2413 printf '%s\n' "${cache_repo}:nixl-rocm-${NIXL_CACHE_KEY}"
2414 elif [[ -n "${NIXL_BRANCH:-}" ]]; then
2415 printf '%s\n' "${cache_repo}:nixl-rocm-${NIXL_BRANCH}-ucx-${UCX_BRANCH:-}"
2416 fi
2417 ;;
2418 rocshmem-rocm-ci)
2419 if [[ -n "${ROCSHMEM_CACHE_KEY:-}" ]]; then
2420 printf '%s\n' "${cache_repo}:rocshmem-rocm-${ROCSHMEM_CACHE_KEY}"
2421 elif [[ -n "${ROCSHMEM_BRANCH:-}" ]]; then
2422 printf '%s\n' "${cache_repo}:rocshmem-rocm-${ROCSHMEM_BRANCH}"
2423 fi
2424 ;;
2425 deepep-rocm-ci)
2426 if [[ -n "${DEEPEP_CACHE_KEY:-}" ]]; then
2427 printf '%s\n' "${cache_repo}:deepep-rocm-${DEEPEP_CACHE_KEY}"
2428 elif [[ -n "${DEEPEP_BRANCH:-}" ]]; then
2429 printf '%s\n' "${cache_repo}:deepep-rocm-${DEEPEP_BRANCH}-rocshmem-${ROCSHMEM_BRANCH:-}"
2430 fi
2431 ;;
2432 esac
2433 }
2434
2435 add_dependency_cache_target() {
2436 local target="$1"
2437
2438 if printf '%s\n' "${DEPENDENCY_CACHE_TARGETS[@]}" | grep -qx "${target}"; then
2439 return 0
2440 fi
2441 DEPENDENCY_CACHE_TARGETS+=("${target}")
2442 }
2443
2444 resolve_ci_base_dependency_targets() {
2445 local mode="${ROCM_DEP_CACHE_EXPORT_MODE:-missing}"
2446 local nixl_ref=""
2447 local rocshmem_ref=""
2448 local deepep_ref=""
2449
2450 [[ "${TARGET}" == "ci-base-rocm-ci-with-deps" ]] || return 0
2451
2452 case "${mode}" in
2453 always)
2454 echo "ROCM_DEP_CACHE_EXPORT_MODE=always; exporting all dependency caches serially"
2455 for target in nixl-rocm-ci rocshmem-rocm-ci deepep-rocm-ci; do
2456 if [[ -n "$(dependency_cache_ref_for_target "${target}")" ]]; then
2457 add_dependency_cache_target "${target}"
2458 fi
2459 done
2460 ;;
2461 never)
2462 BAKE_TARGETS=("ci-base-rocm-ci")
2463 DEPENDENCY_CACHE_TARGETS=()
2464 echo "ROCM_DEP_CACHE_EXPORT_MODE=never; building ci_base without dependency cache exports"
2465 return 0
2466 ;;
2467 missing|"")
2468 ;;
2469 *)
2470 echo "Error: ROCM_DEP_CACHE_EXPORT_MODE must be one of: missing, always, never"
2471 exit 1
2472 ;;
2473 esac
2474
2475 if [[ "${mode}" != "always" && -n "${NIXL_CACHE_KEY:-}" ]]; then
2476 nixl_ref=$(dependency_cache_ref_for_target "nixl-rocm-ci")
2477 if dependency_cache_ref_exists "${nixl_ref}"; then
2478 echo "NIXL dependency cache exists: ${nixl_ref}"
2479 else
2480 echo "NIXL dependency cache missing; will seed: ${nixl_ref}"
2481 add_dependency_cache_target "nixl-rocm-ci"
2482 fi
2483 fi
2484
2485 if [[ "${mode}" != "always" && -n "${ROCSHMEM_CACHE_KEY:-}" ]]; then
2486 rocshmem_ref=$(dependency_cache_ref_for_target "rocshmem-rocm-ci")
2487 if dependency_cache_ref_exists "${rocshmem_ref}"; then
2488 echo "ROCShmem dependency cache exists: ${rocshmem_ref}"
2489 else
2490 echo "ROCShmem dependency cache missing; will seed: ${rocshmem_ref}"
2491 add_dependency_cache_target "rocshmem-rocm-ci"
2492 fi
2493 fi
2494
2495 if [[ "${mode}" != "always" && -n "${DEEPEP_CACHE_KEY:-}" ]]; then
2496 deepep_ref=$(dependency_cache_ref_for_target "deepep-rocm-ci")
2497 if dependency_cache_ref_exists "${deepep_ref}"; then
2498 echo "DeepEP dependency cache exists: ${deepep_ref}"
2499 else
2500 echo "DeepEP dependency cache missing; will seed: ${deepep_ref}"
2501 add_dependency_cache_target "deepep-rocm-ci"
2502 fi
2503 fi
2504
2505 # DeepEP inherits from ROCShmem. If ROCShmem is being seeded, seed DeepEP too
2506 # so the pair stays consistent for future ci_base rebuilds.
2507 if printf '%s\n' "${DEPENDENCY_CACHE_TARGETS[@]}" | grep -qx "rocshmem-rocm-ci" \
2508 && ! printf '%s\n' "${DEPENDENCY_CACHE_TARGETS[@]}" | grep -qx "deepep-rocm-ci" \
2509 && [[ -n "${DEEPEP_BRANCH:-}" ]]; then
2510 echo "ROCShmem cache is missing; also seeding DeepEP cache"
2511 add_dependency_cache_target "deepep-rocm-ci"
2512 fi
2513
2514 BAKE_TARGETS=("ci-base-rocm-ci")
2515 if [[ ${#DEPENDENCY_CACHE_TARGETS[@]} -eq 0 ]]; then
2516 echo "All dependency caches exist; building ci_base without dependency cache exports"
2517 else
2518 echo "Resolved dependency cache seed targets: ${DEPENDENCY_CACHE_TARGETS[*]}"
2519 echo "Resolved ci_base bake targets: ${BAKE_TARGETS[*]}"
2520 fi
2521 }
2522
2523 bake_config_targets() {
2524 printf '%s\n' "${DEPENDENCY_CACHE_TARGETS[@]}" "${BAKE_TARGETS[@]}" \
2525 | awk 'NF && !seen[$0]++'
2526 }
2527
2528 print_bake_config() {
2529 local -a print_targets=()
2530
2531 echo "--- :page_facing_up: Resolved bake configuration"
2532 mapfile -t print_targets < <(bake_config_targets)
2533 docker buildx bake "${BAKE_ALLOW_ARGS[@]}" \
2534 "${BAKE_FILES[@]}" --print "${print_targets[@]}" | tee "${BAKE_CONFIG_FILE}"
2535
2536 if command -v buildkite-agent >/dev/null 2>&1 && [[ -n "${BUILDKITE_BUILD_NUMBER:-}" ]]; then
2537 buildkite-agent artifact upload "${BAKE_CONFIG_FILE}" || true
2538 echo "Uploaded ${BAKE_CONFIG_FILE} as Buildkite artifact"
2539 else
2540 echo "Saved bake config to ${BAKE_CONFIG_FILE} (not in Buildkite, skipping upload)"
2541 fi
2542 }
2543
2544 confirm_remote_image_push() {
2545 local image_ref="$1"
2546
2547 if [[ -z "${CI_BASE_CONTENT_HASH:-}" ]]; then
2548 remote_image_exists "${image_ref}"
2549 return
2550 fi
2551
2552 if remote_ci_base_identity_is_current_with_retry "${image_ref}"; then
2553 return 0
2554 fi
2555
2556 echo "Remote image does not have the expected complete ci_base identity."
2557 return 1
2558 }
2559
2560 verify_dependency_cache_ref() {
2561 local cache_ref="$1"
2562 local attempts="${ROCM_DEP_CACHE_VERIFY_ATTEMPTS:-6}"
2563 local delay_secs="${ROCM_DEP_CACHE_VERIFY_DELAY:-5}"
2564 local attempt
2565
2566 for ((attempt = 1; attempt <= attempts; attempt++)); do
2567 if dependency_cache_ref_exists "${cache_ref}"; then
2568 echo "Dependency cache confirmed: ${cache_ref}"
2569 return 0
2570 fi
2571 if [[ ${attempt} -lt ${attempts} ]]; then
2572 echo "Dependency cache not visible yet (${attempt}/${attempts}): ${cache_ref}"
2573 sleep "${delay_secs}"
2574 fi
2575 done
2576
2577 echo "ERROR: dependency cache was not confirmed after upload: ${cache_ref}"
2578 return 1
2579 }
2580
2581 seed_dependency_caches_if_needed() {
2582 local target=""
2583 local cache_ref=""
2584
2585 if [[ "${TARGET}" != "ci-base-rocm-ci-with-deps" ]]; then
2586 return 0
2587 fi
2588 if [[ ${#DEPENDENCY_CACHE_TARGETS[@]} -eq 0 ]]; then
2589 return 0
2590 fi
2591
2592 echo "--- :docker: Seeding ROCm dependency caches"
2593 echo "Dependency cache uploads are required for this build."
2594 echo "Seeding serially to avoid concurrent Docker Hub cache exporters."
2595
2596 for target in "${DEPENDENCY_CACHE_TARGETS[@]}"; do
2597 cache_ref=$(dependency_cache_ref_for_target "${target}")
2598 if [[ -z "${cache_ref}" ]]; then
2599 echo "ERROR: could not resolve dependency cache ref for ${target}"
2600 return 1
2601 fi
2602
2603 echo "--- :docker: Seeding ${target}"
2604 echo "Expected cache ref: ${cache_ref}"
2605 docker buildx bake \
2606 "${BAKE_ALLOW_ARGS[@]}" \
2607 "${BAKE_FILES[@]}" \
2608 --progress "${BUILDKIT_PROGRESS:-plain}" \
2609 "${target}"
2610 verify_dependency_cache_ref "${cache_ref}"
2611 done
2612 }
2613
2614 run_bake() {
2615 local confirmation_ref="${IMAGE_TAG:-}"
2616
2617 if is_ci_base_target && [[ -n "${CI_BASE_IMAGE_TAG_BUILD_REF:-}" ]]; then
2618 confirmation_ref="${CI_BASE_IMAGE_TAG_BUILD_REF}"
2619 fi
2620
2621 echo "--- :docker: Building ${TARGET}"
2622 docker buildx bake \
2623 "${BAKE_ALLOW_ARGS[@]}" \
2624 "${BAKE_FILES[@]}" \
2625 --progress "${BUILDKIT_PROGRESS:-plain}" \
2626 "${BAKE_TARGETS[@]}"
2627
2628 if is_ci_base_target; then
2629 if ! confirm_remote_image_push "${confirmation_ref}"; then
2630 echo "Fresh ci_base image failed identity validation: ${confirmation_ref}" >&2
2631 return 1
2632 fi
2633 refresh_ci_base_tags_from_ref "${confirmation_ref}"
2634 fi
2635 echo "--- :white_check_mark: Build complete"
2636 }
2637
2638 upload_wheel_artifacts_if_present() {
2639 local wheel_dir="./wheel-export"
2640 local artifact_dir="artifacts/vllm-rocm-install"
2641 local archive_name="vllm-rocm-install.tar.gz"
2642 local metadata_dir="${wheel_dir}/.vllm-ci-artifact"
2643 local build_base_digest=""
2644 local expected_native_base_image=""
2645 local native_base_image=""
2646 local native_base_digest=""
2647 local whl=""
2648 local whl_name=""
2649 local -a wheels=()
2650
2651 if ! should_upload_wheel_artifacts; then
2652 return 0
2653 fi
2654
2655 if [[ -d "${wheel_dir}" ]]; then
2656 mapfile -t wheels < <(find "${wheel_dir}" -maxdepth 1 -type f -name '*.whl' -print)
2657 fi
2658 if [[ ${#wheels[@]} -ne 1 ]]; then
2659 echo "Expected exactly one ROCm wheel in ${wheel_dir}; found ${#wheels[@]}" >&2
2660 return 1
2661 fi
2662 whl="${wheels[0]}"
2663 whl_name=$(basename "${whl}")
2664 native_base_image="${CI_BASE_IMAGE_TAG_BUILD_REF:-${CI_BASE_IMAGE:-}}"
2665 if [[ -z "${native_base_image}" ]]; then
2666 echo "Native ROCm artifact requires a ci_base image reference" >&2
2667 return 1
2668 fi
2669 if [[ "${BUILDKITE:-false}" == "true" ]]; then
2670 expected_native_base_image=$(ci_base_tag_with_suffix \
2671 "${CI_BASE_IMAGE_TAG:-rocm/vllm-dev:ci_base}" \
2672 "build-${BUILDKITE_BUILD_ID:-}")
2673 if [[ "${native_base_image}" != "${expected_native_base_image}" ]]; then
2674 echo "Native ROCm artifact requires the exact ci_base build handoff: ${native_base_image}" >&2
2675 return 1
2676 fi
2677 if [[ ! "${CI_BASE_IMAGE:-}" =~ @sha256:[0-9a-f]{64}$ ]]; then
2678 echo "ROCm artifact build base must be digest-pinned: ${CI_BASE_IMAGE:-<empty>}" >&2
2679 return 1
2680 fi
2681 build_base_digest="${CI_BASE_IMAGE##*@}"
2682 if ! native_base_digest=$(resolve_image_digest "${native_base_image}"); then
2683 echo "Could not resolve native ci_base handoff: ${native_base_image}" >&2
2684 return 1
2685 fi
2686 if [[ "${native_base_digest}" != "${build_base_digest}" ]]; then
2687 echo "Native ci_base handoff does not match the artifact build base" >&2
2688 echo " native: ${native_base_image}@${native_base_digest}" >&2
2689 echo " build: ${CI_BASE_IMAGE}" >&2
2690 return 1
2691 fi
2692 fi
2693
2694 echo "--- :package: Uploading ROCm vLLM install artifact"
2695 rm -rf "${artifact_dir}" "${metadata_dir}"
2696 mkdir -p "${artifact_dir}" "${metadata_dir}"
2697
2698 printf '%s\n' "${BUILDKITE_COMMIT:-local}" > "${metadata_dir}/commit.txt"
2699 printf '%s\n' "${native_base_image}" > "${metadata_dir}/native-base-image.txt"
2700 printf '%s\n' "${CI_BASE_IMAGE:-}" > "${metadata_dir}/ci-base-image.txt"
2701 printf '%s\n' "${IMAGE_TAG:-}" > "${metadata_dir}/fallback-image.txt"
2702 printf '%s\n' "${whl_name}" > "${metadata_dir}/wheel-filename.txt"
2703
2704 tar -C "${wheel_dir}" -czf "${artifact_dir}/${archive_name}" .
2705 (
2706 cd "${artifact_dir}"
2707 sha256sum "${archive_name}" > "${archive_name}.sha256"
2708 )
2709 echo "Created ${archive_name}: $(du -sh "${artifact_dir}/${archive_name}" | cut -f1)"
2710 cp "${metadata_dir}"/*.txt "${artifact_dir}/"
2711 cp "${whl}" "${artifact_dir}/${whl_name}"
2712 echo "Copied ${whl_name}: $(du -sh "${artifact_dir}/${whl_name}" | cut -f1)"
2713
2714 if command -v buildkite-agent >/dev/null 2>&1; then
2715 buildkite-agent artifact upload "${artifact_dir}/*" || return 1
2716 echo "ROCm vLLM install artifacts uploaded to ${artifact_dir}/"
2717 elif [[ "${BUILDKITE:-false}" == "true" ]]; then
2718 echo "buildkite-agent not found; cannot upload required ROCm artifacts" >&2
2719 return 1
2720 else
2721 echo "Not in Buildkite, skipping artifact upload"
2722 fi
2723
2724 rm -rf "${wheel_dir}"
2725 }
2726
2727 main() {
2728 init_config "$@"
2729 configure_ci_base_write_scope
2730 print_header
2731 validate_inputs
2732 load_ci_hcl
2733 init_bake_files
2734 if is_ci_base_target; then
2735 prepare_ci_build_context
2736 configure_custom_rocm_stages
2737 fi
2738 compute_ci_base_hash_if_needed
2739 configure_ci_base_image_refs
2740 maybe_skip_existing_image
2741 setup_builder
2742 prepare_git_cache_metadata
2743 # Non-ci_base builds may deepen a shallow checkout above. Derive archival
2744 # version metadata only after that lookup sees the available tag history.
2745 if ! is_ci_base_target; then
2746 prepare_ci_build_context
2747 configure_custom_rocm_stages
2748 fi
2749 extract_dependency_pins
2750 write_rocm_build_arg_override
2751 compute_dependency_cache_keys
2752 write_ci_base_label_override
2753 compute_rocm_csrc_content_hash_if_needed
2754 compute_rocm_rust_content_hash_if_needed
2755 write_rocm_cache_override
2756 # Keep the context override last so every bake target uses the owned tree.
2757 write_build_context_override
2758 resolve_ci_base_dependency_targets
2759 print_bake_config
2760 if [[ "${BAKE_PRINT_ONLY:-0}" == "1" ]]; then
2761 echo "BAKE_PRINT_ONLY=1 set; skipping build"
2762 return 0
2763 fi
2764 if should_upload_wheel_artifacts; then
2765 # wheel-export is an output directory, not a BuildKit cache. Starting
2766 # clean prevents a failed/retried export from packaging a stale wheel.
2767 rm -rf ./wheel-export
2768 fi
2769 if should_export_rocm_smoke; then
2770 # The marker is a build output, not a cache. Never accept stale output
2771 # from an earlier build or retry.
2772 rm -rf ./build/rocm-smoke-export
2773 fi
2774 seed_dependency_caches_if_needed
2775 run_bake
2776 verify_rocm_smoke_export
2777 promote_stable_ci_base_tag
2778 publish_ci_base_handoff_ref
2779 upload_wheel_artifacts_if_present
2780 }
2781
2782 if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
2783 main "$@"
2784 fi