@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 419b51b385

[Bugfix][ROCm][CI] Stabilize build context and source caches (#51721)

Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>

Andreas Karatzas committed Aug 10, 2026 at 21:51 UTC 419b51b38517bd446c6e359d37b1ee8674cd7e84
2 files changed +345 -61
.buildkite/scripts/ci-bake-rocm.sh
+313 -57
@@ -22,7 +22,7 @@ DEFAULT_CI_BASE_METADATA_VERSION="3"
22 # ROCm CI forces REMOTE_VLLM=0, so content identity covers only the selected
23 # local-source stages rather than unreachable remote-fetch alternatives.
24 DEFAULT_ROCM_CSRC_CONTENT_FILES=".dockerignore requirements/common.txt requirements/rocm.txt pyproject.toml setup.py CMakeLists.txt cmake csrc vllm/envs.py vllm/__init__.py tools/build_rust.py"
25 -DEFAULT_ROCM_CSRC_DOCKERFILE_STAGES="base fetch_vllm_0 fetch_vllm build_vllm_dependencies csrc-build"
25 +DEFAULT_ROCM_CSRC_DOCKERFILE_STAGES="base fetch_vllm_0 fetch_vllm build_vllm_dependencies rocm-triton-kernels csrc-build"
26 DEFAULT_ROCM_RUST_CONTENT_FILES=".dockerignore requirements/build/rust.txt rust/Cargo.lock rust/Cargo.toml rust/proto rust/src rust-toolchain.toml tools/build_rust.py tools/install_protoc.sh build_rust.sh"
27 DEFAULT_ROCM_RUST_DOCKERFILE_STAGES="base rust_toolchain_input_0 rust-toolchain-input rust_input_0 rust-input rust-toolchain rust-build"
28 # Docker's 128-character tag limit minus the longest cache prefix
@@ -36,9 +36,14 @@ CI_HCL_PATH=""
36 CI_BASE_LABEL_OVERRIDE_PATH=""
37 CSRC_CACHE_OVERRIDE_PATH=""
38 ROCM_ARG_OVERRIDE_PATH=""
39 +BUILD_CONTEXT_OVERRIDE_PATH=""
40 SCRIPT_TMP_DIR=""
41 BAKE_CONFIG_FILE=""
42 +ROCM_BUILD_CONTEXT_ROOT=""
43 +ROCM_BUILD_CONTEXT_INDEX=""
44 +ROCM_BUILD_CONTEXT_COMMIT=""
45 BAKE_FILES=()
46 +BAKE_ALLOW_ARGS=()
47 BAKE_TARGETS=()
48 DEPENDENCY_CACHE_TARGETS=()
49
@@ -192,16 +197,20 @@ get_buildkite_target_repo_url() {
197 printf 'https://github.com/%s.git\n' "${DEFAULT_REPO_SLUG}"
198 }
199
195 -git_fetch_for_cache() {
200 +git_fetch_with_timeout() {
201 local timeout_secs="${ROCM_CACHE_GIT_FETCH_TIMEOUT:-60}"
202
203 if command -v timeout >/dev/null 2>&1; then
199 - timeout "${timeout_secs}s" git fetch "$@" 2>/dev/null
204 + timeout "${timeout_secs}s" git fetch "$@"
205 else
201 - git fetch "$@" 2>/dev/null
206 + git fetch "$@"
207 fi
208 }
209
210 +git_fetch_for_cache() {
211 + git_fetch_with_timeout "$@" 2>/dev/null
212 +}
213 +
214 hash_string_short() {
215 printf '%s' "$1" | sha256sum | cut -c1-16
216 }
@@ -209,23 +218,51 @@ hash_string_short() {
218 list_content_files() {
219 # Hash the checkout inputs Docker can intentionally consume, not ignored
220 # compiler/test debris left behind on a reused worker.
212 - git ls-files -z --cached --others --exclude-standard -- "$1" \
213 - | LC_ALL=C sort -z
221 + if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" ]]; then
222 + GIT_INDEX_FILE="${ROCM_BUILD_CONTEXT_INDEX}" \
223 + git ls-files -z --cached -- "$1" | LC_ALL=C sort -z
224 + else
225 + git ls-files -z --cached --others --exclude-standard -- "$1" \
226 + | LC_ALL=C sort -z
227 + fi
228 }
229
216 -hash_content_file() {
230 +content_regular_file() {
231 local file="$1"
232 + local physical_file=""
233
219 - if [[ -L "${file}" ]]; then
220 - printf 'symlink:%s\ntarget:%s\n' "${file}" "$(readlink "${file}")"
234 + physical_file="${file}"
235 + if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${file}" != /* ]]; then
236 + physical_file="${ROCM_BUILD_CONTEXT_ROOT}/${file}"
237 + fi
238 + [[ -f "${physical_file}" ]]
239 +}
240 +
241 +hash_content_file() {
242 + local file="$1"
243 + local physical_file=""
244 + local checksum=""
245 + local file_mode=""
246 +
247 + physical_file="${file}"
248 + if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${file}" != /* ]]; then
249 + physical_file="${ROCM_BUILD_CONTEXT_ROOT}/${file}"
250 + fi
251 + if [[ -L "${physical_file}" ]]; then
252 + printf 'symlink:%s\ntarget:' "${file}"
253 + readlink -n -- "${physical_file}" || return $?
254 + printf '\n'
255 return
256 fi
223 - if [[ ! -f "${file}" ]]; then
257 + if [[ ! -f "${physical_file}" ]]; then
258 printf 'missing:%s\n' "${file}"
259 return
260 fi
227 - printf 'file:%s\nmode:%s\n' "${file}" "$(stat -c '%a' "${file}")"
228 - sha256sum "${file}"
261 + file_mode=$(stat -c '%a' "${physical_file}") || return $?
262 + printf 'file:%s\nmode:%s\n' "${file}" "${file_mode}"
263 + checksum=$(sha256sum < "${physical_file}") || return $?
264 + checksum="${checksum%% *}"
265 + printf '%s %s\n' "${checksum}" "${file}"
266 }
267
268 hash_content_directory() {
@@ -239,11 +276,16 @@ hash_content_directory() {
276
277 compute_content_hash() {
278 local path=""
279 + local physical_path=""
280
281 for path in "$@"; do
244 - if [[ -L "${path}" || -f "${path}" ]]; then
282 + physical_path="${path}"
283 + if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${path}" != /* ]]; then
284 + physical_path="${ROCM_BUILD_CONTEXT_ROOT}/${path}"
285 + fi
286 + if [[ -L "${physical_path}" || -f "${physical_path}" ]]; then
287 hash_content_file "${path}" || return $?
246 - elif [[ -d "${path}" ]]; then
288 + elif [[ -d "${physical_path}" ]]; then
289 hash_content_directory "${path}" || return $?
290 else
291 printf 'missing:%s\n' "${path}"
@@ -251,25 +293,69 @@ compute_content_hash() {
293 done | sha256sum | cut -d' ' -f1
294 }
295
254 -normalize_ci_worktree_modes() {
296 +validate_ci_build_context_source() {
297 + local source_root="$1"
298 + local context_commit=""
299 + local source_head=""
300 local entry=""
301 local metadata=""
302 local mode=""
303 local stage=""
304 local path=""
305 local index_modes_file="${SCRIPT_TMP_DIR}/git-index-modes"
261 - local -a regular_files=()
262 - local -a executable_files=()
306 + local info_attributes=""
307
264 - [[ "${BUILDKITE:-false}" == "true" ]] || return 0
265 - [[ "${REMOTE_VLLM:-0}" == "0" ]] || return 0
266 - if ! git ls-files --stage -z > "${index_modes_file}"; then
267 - echo "Failed to read Git modes for the CI Docker context" >&2
308 + if ! context_commit=$(git -C "${source_root}" rev-parse \
309 + --verify "${BUILDKITE_COMMIT:-HEAD}^{commit}"); then
310 + echo "Failed to resolve the CI Docker context revision" >&2
311 + return 1
312 + fi
313 + if ! source_head=$(git -C "${source_root}" rev-parse --verify HEAD); then
314 + echo "Failed to resolve the checked-out CI revision" >&2
315 + return 1
316 + fi
317 + if [[ "${context_commit}" != "${source_head}" ]]; then
318 + echo "BUILDKITE_COMMIT does not match the checked-out CI revision" >&2
319 + return 1
320 + fi
321 + ROCM_BUILD_CONTEXT_COMMIT="${context_commit}"
322 + if ! git -C "${source_root}" diff \
323 + --quiet --no-ext-diff --ignore-submodules=none --; then
324 + echo "Tracked worktree changes cannot be omitted from the CI Docker context" \
325 + >&2
326 + return 1
327 + fi
328 + if ! git -C "${source_root}" diff --cached \
329 + --quiet --no-ext-diff --ignore-submodules=none HEAD --; then
330 + echo "Staged changes cannot be omitted from the CI Docker context" >&2
331 + return 1
332 + fi
333 + # Untracked and ignored worker outputs are intentionally absent: the
334 + # pinned Git tree, rather than mutable checkout contents, is the contract.
335 + if ! info_attributes=$(git -C "${source_root}" \
336 + rev-parse --path-format=absolute --git-path info/attributes); then
337 + echo "Failed to locate repository-local Git attributes" >&2
338 + return 1
339 + fi
340 + if [[ -s "${info_attributes}" ]]; then
341 + echo "Repository-local Git attributes cannot define the CI Docker context" \
342 + >&2
343 return 1
344 fi
345
271 - # Some shared workspaces expose 0644/0755 files as 0664/0775. Restore the
272 - # index modes so both the identity hash and Docker context are canonical.
346 + ROCM_BUILD_CONTEXT_INDEX="${SCRIPT_TMP_DIR}/docker-context.index"
347 + if ! GIT_INDEX_FILE="${ROCM_BUILD_CONTEXT_INDEX}" \
348 + git -C "${source_root}" -c core.splitIndex=false \
349 + read-tree "${context_commit}^{tree}"; then
350 + echo "Failed to create the CI Docker context index" >&2
351 + return 1
352 + fi
353 + if ! GIT_INDEX_FILE="${ROCM_BUILD_CONTEXT_INDEX}" \
354 + git -C "${source_root}" ls-files --stage -z \
355 + > "${index_modes_file}"; then
356 + echo "Failed to read the Git index for the CI Docker context" >&2
357 + return 1
358 + fi
359 while IFS= read -r -d '' entry; do
360 metadata="${entry%%$'\t'*}"
361 mode="${metadata%% *}"
@@ -280,13 +366,12 @@ normalize_ci_worktree_modes() {
366 return 1
367 fi
368 case "${mode}" in
283 - 100644)
284 - regular_files+=("${path}")
369 + 100644|100755|120000)
370 ;;
286 - 100755)
287 - executable_files+=("${path}")
288 - ;;
289 - 120000|160000)
371 + 160000)
372 + echo "Git submodule cannot be materialized in the CI Docker context: ${path}" \
373 + >&2
374 + return 1
375 ;;
376 *)
377 echo "Unsupported Git mode ${mode} for ${path}" >&2
@@ -294,10 +379,130 @@ normalize_ci_worktree_modes() {
379 ;;
380 esac
381 done < "${index_modes_file}"
382 +}
383
298 - ((${#regular_files[@]} == 0)) || chmod 0644 -- "${regular_files[@]}"
299 - ((${#executable_files[@]} == 0)) || chmod 0755 -- "${executable_files[@]}"
300 - echo "Normalized Git file modes for the CI Docker context"
384 +describe_ci_revision() {
385 + git -C "$1" describe --tags --long --abbrev=10 \
386 + --match '*[0-9]*' "$2" 2>/dev/null
387 +}
388 +
389 +write_ci_git_archival_metadata() {
390 + local source_root="$1"
391 + local context_root="$2"
392 + local commit="${ROCM_BUILD_CONTEXT_COMMIT}"
393 + local commit_date=""
394 + local describe=""
395 + local is_shallow="false"
396 +
397 + commit_date=$(git -C "${source_root}" show -s --format=%cI "${commit}") \
398 + || return $?
399 + is_shallow=$(git -C "${source_root}" rev-parse --is-shallow-repository) \
400 + || return $?
401 + if git -C "${source_root}" remote get-url origin >/dev/null 2>&1; then
402 + # AMD agents use shallow, no-tag clones. Reach a release tag so one
403 + # commit cannot acquire different versions from different depths.
404 + # Versioning only needs tags and commits, not historical source trees.
405 + echo "Synchronizing version tags for the canonical CI Docker context"
406 + (cd "${source_root}" && git_fetch_with_timeout --quiet \
407 + --filter=tree:0 --prune origin '+refs/tags/*:refs/tags/*') \
408 + || return $?
409 + if [[ "${is_shallow}" == "true" ]] \
410 + && ! describe_ci_revision "${source_root}" "${commit}" >/dev/null; then
411 + echo "Deepening history to reach a version tag"
412 + (cd "${source_root}" && git_fetch_with_timeout --quiet \
413 + --filter=tree:0 --no-tags --deepen=1000 origin "${commit}") \
414 + || return $?
415 + is_shallow=$(git -C "${source_root}" \
416 + rev-parse --is-shallow-repository) || return $?
417 + if [[ "${is_shallow}" == "true" ]] \
418 + && ! describe_ci_revision "${source_root}" "${commit}" >/dev/null; then
419 + echo "No version tag within 1,000 commits; fetching full history"
420 + (cd "${source_root}" && git_fetch_with_timeout --quiet \
421 + --filter=tree:0 --no-tags --unshallow origin "${commit}") \
422 + || return $?
423 + fi
424 + fi
425 + fi
426 + if ! describe=$(describe_ci_revision "${source_root}" "${commit}"); then
427 + echo "No numeric version tag is reachable from the CI revision" >&2
428 + return 1
429 + fi
430 + if [[ -e "${context_root}/.git_archival.txt" \
431 + || -L "${context_root}/.git_archival.txt" ]]; then
432 + echo "The source tree already contains .git_archival.txt" >&2
433 + return 1
434 + fi
435 + printf 'node: %s\nnode-date: %s\ndescribe-name: %s\n' \
436 + "${commit}" "${commit_date}" "${describe}" \
437 + > "${context_root}/.git_archival.txt" \
438 + && chmod 0644 -- "${context_root}/.git_archival.txt"
439 +}
440 +
441 +write_build_context_override() {
442 + local escaped_context=""
443 +
444 + [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" ]] || return 0
445 + escaped_context=$(hcl_escape_string "${ROCM_BUILD_CONTEXT_ROOT}") || return $?
446 + if ! {
447 + printf 'target "_common-rocm" {\n'
448 + printf ' context = "%s"\n' "${escaped_context}"
449 + printf '}\n'
450 + } > "${BUILD_CONTEXT_OVERRIDE_PATH}"; then
451 + echo "Failed to write the CI Docker context override" >&2
452 + return 1
453 + fi
454 + BAKE_FILES+=(-f "${BUILD_CONTEXT_OVERRIDE_PATH}")
455 +}
456 +
457 +prepare_ci_build_context() {
458 + local source_root=""
459 + local context_root=""
460 +
461 + [[ "${BUILDKITE:-false}" == "true" ]] || return 0
462 + [[ "${REMOTE_VLLM:-0}" == "0" ]] || return 0
463 +
464 + # BuildKit includes file modes in cache keys. Export the pinned revision to
465 + # an owned context instead of changing modes in the shared checkout.
466 + if ! source_root=$(git rev-parse --show-toplevel); then
467 + echo "Failed to locate the CI source checkout" >&2
468 + return 1
469 + fi
470 + validate_ci_build_context_source "${source_root}" || return $?
471 + context_root="${SCRIPT_TMP_DIR}/docker-context"
472 + if ! mkdir -m 0700 -- "${context_root}"; then
473 + echo "Failed to create the owned CI Docker context" >&2
474 + return 1
475 + fi
476 + if ! (
477 + umask 0022
478 + unset GIT_LFS_SKIP_SMUDGE
479 + export GIT_ATTR_NOSYSTEM=1
480 + export GIT_INDEX_FILE="${ROCM_BUILD_CONTEXT_INDEX}"
481 + git -C "${source_root}" \
482 + -c core.attributesFile=/dev/null \
483 + -c core.autocrlf=false \
484 + -c core.eol=lf \
485 + -c core.symlinks=true \
486 + --work-tree="${context_root}" \
487 + checkout-index --all
488 + ); then
489 + echo "Failed to materialize the CI Docker context from the Git index" >&2
490 + return 1
491 + fi
492 +
493 + # setuptools-scm understands Git's stable archive format, so full-source
494 + # wheels retain their exact version without copying mutable Git history.
495 + if ! is_ci_base_target; then
496 + write_ci_git_archival_metadata "${source_root}" "${context_root}" \
497 + || return $?
498 + fi
499 + if [[ -e "${context_root}/.git" ]]; then
500 + echo "Canonical CI Docker context unexpectedly contains .git" >&2
501 + return 1
502 + fi
503 + ROCM_BUILD_CONTEXT_ROOT="${context_root}"
504 + BAKE_ALLOW_ARGS+=(--allow "fs.read=${ROCM_BUILD_CONTEXT_ROOT}")
505 + echo "Using canonical CI Docker context: ${ROCM_BUILD_CONTEXT_ROOT}"
506 }
507
508 compose_dependency_cache_key() {
@@ -312,6 +517,12 @@ compose_dependency_cache_key() {
517 hash_dockerfile_stages() {
518 local dockerfile="$1"
519 local stages="$2"
520 + local physical_dockerfile=""
521 +
522 + physical_dockerfile="${dockerfile}"
523 + if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${dockerfile}" != /* ]]; then
524 + physical_dockerfile="${ROCM_BUILD_CONTEXT_ROOT}/${dockerfile}"
525 + fi
526
527 awk -v wanted_stages="${stages}" '
528 BEGIN {
@@ -335,14 +546,19 @@ hash_dockerfile_stages() {
546 emit {
547 print
548 }
338 - ' "${dockerfile}"
549 + ' "${physical_dockerfile}"
550 }
551
552 discover_dockerfile_stage_args() {
553 local dockerfile="$1"
554 local stages="$2"
555 + local physical_dockerfile=""
556
345 - [[ -f "${dockerfile}" ]] || return 0
557 + physical_dockerfile="${dockerfile}"
558 + if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${dockerfile}" != /* ]]; then
559 + physical_dockerfile="${ROCM_BUILD_CONTEXT_ROOT}/${dockerfile}"
560 + fi
561 + [[ -f "${physical_dockerfile}" ]] || return 0
562
563 awk -v wanted_stages="${stages}" '
564 function add_arg(name) {
@@ -406,7 +622,7 @@ discover_dockerfile_stage_args() {
622 }
623 }
624 }
409 - ' "${dockerfile}"
625 + ' "${physical_dockerfile}"
626 }
627
628 get_content_arg_names() {
@@ -446,7 +662,7 @@ compute_ci_base_content_hash() {
662 || return 1
663 if [[ -n "${stages}" ]]; then
664 printf 'dockerfile-stages:%s\n' "${stages}"
449 - if [[ -f "${dockerfile}" ]]; then
665 + if content_regular_file "${dockerfile}"; then
666 hash_dockerfile_stages "${dockerfile}" "${stages}"
667 else
668 printf 'missing:%s\n' "${dockerfile}"
@@ -459,9 +675,14 @@ compute_ci_base_content_hash() {
675 extract_dockerfile_arg_default() {
676 local dockerfile="$1"
677 local arg_name="$2"
678 + local physical_dockerfile=""
679
680 + physical_dockerfile="${dockerfile}"
681 + if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${dockerfile}" != /* ]]; then
682 + physical_dockerfile="${ROCM_BUILD_CONTEXT_ROOT}/${dockerfile}"
683 + fi
684 sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
464 - "${dockerfile}" | head -1
685 + "${physical_dockerfile}" | head -1
686 }
687
688 resolve_image_digest() {
@@ -523,7 +744,7 @@ resolve_dockerfile_arg_value() {
744 if [[ -z "${value}" && "${env_name}" != "${arg_name}" ]]; then
745 value="${!arg_name:-}"
746 fi
526 - if [[ -z "${value}" && -f "${dockerfile}" ]]; then
747 + if [[ -z "${value}" ]] && content_regular_file "${dockerfile}"; then
748 value=$(extract_dockerfile_arg_default "${dockerfile}" "${arg_name}")
749 fi
750
@@ -741,6 +962,7 @@ init_config() {
962 CI_BASE_LABEL_OVERRIDE_PATH="${SCRIPT_TMP_DIR}/ci-base-label-override.hcl"
963 CSRC_CACHE_OVERRIDE_PATH="${SCRIPT_TMP_DIR}/rocm-csrc-cache-override.hcl"
964 ROCM_ARG_OVERRIDE_PATH="${SCRIPT_TMP_DIR}/rocm-arg-override.hcl"
965 + BUILD_CONTEXT_OVERRIDE_PATH="${SCRIPT_TMP_DIR}/build-context-override.hcl"
966 BAKE_CONFIG_FILE="bake-config-build-${BUILDKITE_BUILD_NUMBER:-local}.json"
967 }
968
@@ -1270,7 +1492,8 @@ prepare_git_cache_metadata() {
1492 if [[ -z "${PARENT_COMMIT:-}" || -z "${VLLM_MERGE_BASE_COMMIT:-}" ]] \
1493 && git rev-parse --is-shallow-repository 2>/dev/null | grep -q "true"; then
1494 echo "Shallow clone detected - deepening for cache key computation"
1273 - git_fetch_for_cache --deepen=1 origin || true
1495 + git_fetch_for_cache --filter=tree:0 --no-tags --deepen=1 \
1496 + origin "$(git rev-parse HEAD)" || true
1497 fi
1498
1499 if [[ -z "${PARENT_COMMIT:-}" ]]; then
@@ -1524,7 +1747,7 @@ compute_rocm_csrc_content_hash() {
1747 printf 'resolved-build-args:\n'
1748 hash_dockerfile_arg_values "${dockerfile_rocm}" "${content_args[@]}"
1749 printf 'dockerfile-stages:%s\n' "${stages}"
1527 - if [[ -f "${dockerfile_rocm}" ]]; then
1750 + if content_regular_file "${dockerfile_rocm}"; then
1751 hash_dockerfile_stages "${dockerfile_rocm}" "${stages}"
1752 else
1753 printf 'missing:%s\n' "${dockerfile_rocm}"
@@ -1574,7 +1797,7 @@ compute_rocm_rust_content_hash() {
1797 printf 'resolved-build-args:\n'
1798 hash_dockerfile_arg_values "${dockerfile_rocm}" "${content_args[@]}"
1799 printf 'dockerfile-stages:%s\n' "${stages}"
1577 - if [[ -f "${dockerfile_rocm}" ]]; then
1800 + if content_regular_file "${dockerfile_rocm}"; then
1801 hash_dockerfile_stages "${dockerfile_rocm}" "${stages}"
1802 else
1803 printf 'missing:%s\n' "${dockerfile_rocm}"
@@ -1736,24 +1959,24 @@ should_export_content_cache_ref() {
1959
1960 case "${mode}" in
1961 always)
1739 - echo "${cache_name} content cache export mode is always; exporting ${cache_ref}"
1962 + echo "${cache_name} content cache export mode is always"
1963 return 0
1964 ;;
1965 never)
1743 - echo "${cache_name} content cache export mode is never; not exporting ${cache_ref}"
1966 + echo "${cache_name} content cache export mode is never"
1967 return 1
1968 ;;
1969 missing|"")
1747 - if remote_image_exists "${trusted_ref}"; then
1748 - echo "${cache_name} trusted content cache exists; not exporting ${cache_ref}"
1970 + if registry_ref_exists_with_retry imagetools "${trusted_ref}"; then
1971 + echo "${cache_name} trusted content cache is visible: ${trusted_ref}"
1972 return 1
1973 fi
1974 if [[ "${cache_ref}" != "${trusted_ref}" ]] \
1752 - && remote_image_exists "${cache_ref}"; then
1753 - echo "${cache_name} scoped content cache exists; not re-exporting ${cache_ref}"
1975 + && registry_ref_exists_with_retry imagetools "${cache_ref}"; then
1976 + echo "${cache_name} scoped content cache is visible: ${cache_ref}"
1977 return 1
1978 fi
1756 - echo "${cache_name} content cache missing; will export ${cache_ref}"
1979 + echo "${cache_name} content cache is missing: ${cache_ref}"
1980 return 0
1981 ;;
1982 *)
@@ -1791,7 +2014,7 @@ write_rocm_cache_override() {
2014 validate_cache_export_mode "${rocm_cache_to_mode}" "ROCM_FINAL_CACHE_TO_MODE"
2015 echo "ROCm content cache export mode: ${content_cache_export_mode}"
2016 echo "ROCm csrc cache export mode: ${csrc_cache_to_mode}"
1794 - echo "ROCm Rust cache export mode: ${rust_cache_to_mode}"
2017 + echo "ROCm Rust fallback cache export mode: ${rust_cache_to_mode}"
2018 echo "ROCm final image cache export mode: ${rocm_cache_to_mode}"
2019
2020 if [[ -n "${ROCM_CSRC_CONTENT_CACHE_REF:-}" ]]; then
@@ -1825,14 +2048,20 @@ write_rocm_cache_override() {
2048 "type=registry,ref=${ROCM_RUST_CONTENT_CACHE_REF}"
2049 )
2050 fi
1828 - if should_export_content_cache_ref \
2051 + # Legacy commit/branch exports are only needed while the exact-input
2052 + # content ref is absent. The exact ref itself is refreshed below.
2053 + if ! should_export_content_cache_ref \
2054 "${ROCM_RUST_CONTENT_CACHE_REF}" "ROCm Rust" \
2055 "${ROCM_RUST_TRUSTED_CONTENT_CACHE_REF}"; then
2056 + export_rust_cache=0
2057 + fi
2058 + if [[ "${content_cache_export_mode}" != "never" ]]; then
2059 + # Refresh the exact-input ref in the original solve regardless of
2060 + # which local or remote cache supplied the Rust result.
2061 rust_cache_to+=(
1832 - "type=registry,ref=${ROCM_RUST_CONTENT_CACHE_REF},mode=${rust_cache_to_mode},ignore-error=true"
2062 + "type=registry,ref=${ROCM_RUST_CONTENT_CACHE_REF},mode=min,ignore-error=true"
2063 )
1834 - else
1835 - export_rust_cache=0
2064 + echo "ROCm Rust exact-input cache will be refreshed (mode=min): ${ROCM_RUST_CONTENT_CACHE_REF}"
2065 fi
2066 fi
2067
@@ -1872,6 +2101,16 @@ write_rocm_cache_override() {
2101 )
2102 fi
2103
2104 + # Standalone image/wheel targets reach rust-build but not the cache-only
2105 + # exporter unless it is requested explicitly.
2106 + if ((${#rust_cache_to[@]} > 0)); then
2107 + case "${TARGET}" in
2108 + test-rocm-ci|export-wheel-rocm)
2109 + BAKE_TARGETS=("rust-rocm-ci" "${BAKE_TARGETS[@]}")
2110 + ;;
2111 + esac
2112 + fi
2113 +
2114 if [[ "${TARGET}" == "test-rocm-ci-with-wheel" ]]; then
2115 export_wheel_cache_to=()
2116 else
@@ -1937,12 +2176,17 @@ EOF
2176 extract_dependency_pins() {
2177 local bake_dir=""
2178 local dockerfile_rocm=""
2179 + local physical_dockerfile=""
2180 local var=""
2181 local val=""
2182
2183 bake_dir=$(dirname "${VLLM_BAKE_FILE}")
2184 dockerfile_rocm="${bake_dir}/Dockerfile.rocm"
1945 - if [[ ! -f "${dockerfile_rocm}" ]]; then
2185 + physical_dockerfile="${dockerfile_rocm}"
2186 + if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${dockerfile_rocm}" != /* ]]; then
2187 + physical_dockerfile="${ROCM_BUILD_CONTEXT_ROOT}/${dockerfile_rocm}"
2188 + fi
2189 + if [[ ! -f "${physical_dockerfile}" ]]; then
2190 return 0
2191 fi
2192
@@ -1954,7 +2198,7 @@ extract_dependency_pins() {
2198
2199 val=$(
2200 sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${var}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
1957 - "${dockerfile_rocm}" | head -1
2201 + "${physical_dockerfile}" | head -1
2202 )
2203 if [[ -n "${val}" ]]; then
2204 export "${var}=${val}"
@@ -2162,7 +2406,8 @@ print_bake_config() {
2406
2407 echo "--- :page_facing_up: Resolved bake configuration"
2408 mapfile -t print_targets < <(bake_config_targets)
2165 - docker buildx bake "${BAKE_FILES[@]}" --print "${print_targets[@]}" | tee "${BAKE_CONFIG_FILE}"
2409 + docker buildx bake "${BAKE_ALLOW_ARGS[@]}" \
2410 + "${BAKE_FILES[@]}" --print "${print_targets[@]}" | tee "${BAKE_CONFIG_FILE}"
2411
2412 if command -v buildkite-agent >/dev/null 2>&1 && [[ -n "${BUILDKITE_BUILD_NUMBER:-}" ]]; then
2413 buildkite-agent artifact upload "${BAKE_CONFIG_FILE}" || true
@@ -2234,6 +2479,7 @@ seed_dependency_caches_if_needed() {
2479 echo "--- :docker: Seeding ${target}"
2480 echo "Expected cache ref: ${cache_ref}"
2481 docker buildx bake \
2482 + "${BAKE_ALLOW_ARGS[@]}" \
2483 "${BAKE_FILES[@]}" \
2484 --progress "${BUILDKIT_PROGRESS:-plain}" \
2485 "${target}"
@@ -2250,6 +2496,7 @@ run_bake() {
2496
2497 echo "--- :docker: Building ${TARGET}"
2498 docker buildx bake \
2499 + "${BAKE_ALLOW_ARGS[@]}" \
2500 "${BAKE_FILES[@]}" \
2501 --progress "${BUILDKIT_PROGRESS:-plain}" \
2502 "${BAKE_TARGETS[@]}"
@@ -2358,14 +2605,21 @@ main() {
2605 configure_ci_base_write_scope
2606 print_header
2607 validate_inputs
2361 - normalize_ci_worktree_modes
2608 load_ci_hcl
2609 init_bake_files
2610 + if is_ci_base_target; then
2611 + prepare_ci_build_context
2612 + fi
2613 compute_ci_base_hash_if_needed
2614 configure_ci_base_image_refs
2615 maybe_skip_existing_image
2616 setup_builder
2617 prepare_git_cache_metadata
2618 + # Non-ci_base builds may deepen a shallow checkout above. Derive archival
2619 + # version metadata only after that lookup sees the available tag history.
2620 + if ! is_ci_base_target; then
2621 + prepare_ci_build_context
2622 + fi
2623 extract_dependency_pins
2624 write_rocm_build_arg_override
2625 compute_dependency_cache_keys
@@ -2373,6 +2627,8 @@ main() {
2627 compute_rocm_csrc_content_hash_if_needed
2628 compute_rocm_rust_content_hash_if_needed
2629 write_rocm_cache_override
2630 + # Keep the context override last so every bake target uses the owned tree.
2631 + write_build_context_override
2632 resolve_ci_base_dependency_targets
2633 print_bake_config
2634 if [[ "${BAKE_PRINT_ONLY:-0}" == "1" ]]; then
docker/Dockerfile.rocm
+32 -4
@@ -5,6 +5,7 @@ ARG VLLM_BRANCH="main"
5 ARG COMMON_WORKDIR=/app
6 ARG BASE_IMAGE=rocm/vllm-dev:base
7 ARG CI_BASE_IMAGE=rocm/vllm-dev:ci_base
8 +ARG ROCM_TRITON_KERNELS_COMMIT=0f380657dbf3ee86eb57558ff71df24f03b5d4e7
9 # NIC backend for MoRI RDMA support.
10 # By default (all), drivers and userspace libraries for all supported NIC types
11 # (ainic and bnxt) are installed; MoRI selects the appropriate one at runtime.
@@ -272,6 +273,20 @@ COPY --from=fetch_vllm \
273 RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
274 uv pip install --system -r requirements/rocm.txt
275
276 +# Fetch only the pinned ROCm Triton kernels package. Keeping this before the
277 +# vLLM source COPYs makes the download reusable when native sources change.
278 +FROM base AS rocm-triton-kernels
279 +ARG ROCM_TRITON_KERNELS_COMMIT
280 +ADD https://codeload.github.com/ROCm/triton/tar.gz/${ROCM_TRITON_KERNELS_COMMIT} \
281 + /tmp/rocm-triton.tar.gz
282 +RUN mkdir -p /opt/rocm-triton-kernels \
283 + && tar --extract --gzip --file /tmp/rocm-triton.tar.gz \
284 + --directory /opt/rocm-triton-kernels --strip-components=4 \
285 + --no-same-owner \
286 + "triton-${ROCM_TRITON_KERNELS_COMMIT}/python/triton_kernels/triton_kernels" \
287 + && test -f /opt/rocm-triton-kernels/__init__.py \
288 + && rm /tmp/rocm-triton.tar.gz
289 +
290 # -----------------------
291 # vLLM native build stages
292 #
@@ -279,6 +294,11 @@ RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
294 # compilation. That keeps unrelated CI/test/docs edits from invalidating the
295 # expensive HIP/C++ build layer.
296 FROM build_vllm_dependencies AS csrc-build
297 +ARG ROCM_TRITON_KERNELS_COMMIT
298 +COPY --from=rocm-triton-kernels \
299 + /opt/rocm-triton-kernels /opt/rocm-triton-kernels
300 +ENV TRITON_KERNELS_SRC_DIR=/opt/rocm-triton-kernels
301 +
302 # pyproject.toml is bind-mounted so it is not persisted in the csrc-build image,
303 # but it still participates in the BuildKit cache key for this RUN step.
304 COPY setup.py CMakeLists.txt ./
@@ -292,13 +312,15 @@ ENV SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+rocm.csrc.build"
312
313 RUN --mount=type=bind,source=pyproject.toml,target=${COMMON_WORKDIR}/vllm/pyproject.toml \
314 --mount=type=cache,id=vllm-rocm-ccache,target=/root/.cache/ccache \
295 - export CCACHE_BASEDIR="$PWD" \
315 + grep -Fq "set(TRITON_KERNELS_TAG \"${ROCM_TRITON_KERNELS_COMMIT}\")" \
316 + cmake/external_projects/triton_kernels.cmake \
317 + && export CCACHE_BASEDIR="$PWD" \
318 && echo "=== ccache stats before ROCm native build ===" \
319 && (ccache --show-stats || true) \
320 && (ccache --zero-stats || true) \
321 && EFFECTIVE_MAX_JOBS="${MAX_JOBS:-$(nproc)}" \
322 && echo "Building ROCm native extension wheel with MAX_JOBS=${EFFECTIVE_MAX_JOBS}" \
301 - && LDFLAGS="-fuse-ld=mold" MAX_JOBS="${EFFECTIVE_MAX_JOBS}" python3 setup.py bdist_wheel --dist-dir=dist \
323 + && PYTHONDONTWRITEBYTECODE=1 LDFLAGS="-fuse-ld=mold" MAX_JOBS="${EFFECTIVE_MAX_JOBS}" python3 setup.py bdist_wheel --dist-dir=dist \
324 && test -d dist \
325 && ls dist/*.whl >/dev/null \
326 && echo "=== ccache stats after ROCm native build ===" \
@@ -322,7 +344,7 @@ RUN cd "${COMMON_WORKDIR}/vllm" \
344 && export VLLM_PRECOMPILED_WHEEL_LOCATION="$(ls /precompiled-wheels/*.whl)" \
345 && export VLLM_DOCKER_BUILD_CONTEXT=1 \
346 && echo "Packaging vLLM ROCm wheel using precompiled extensions from ${VLLM_PRECOMPILED_WHEEL_LOCATION}" \
325 - && python3 setup.py bdist_wheel --dist-dir=dist \
347 + && PYTHONDONTWRITEBYTECODE=1 python3 setup.py bdist_wheel --dist-dir=dist \
348 && test -d dist \
349 && ls dist/*.whl >/dev/null
350 FROM scratch AS export_vllm
@@ -551,6 +573,10 @@ RUN /bin/bash -lc 'set -euo pipefail; \
573 FROM fetch_vllm AS build_vllm_wheel_release
574
575 ARG COMMON_WORKDIR
576 +ARG ROCM_TRITON_KERNELS_COMMIT
577 +COPY --from=rocm-triton-kernels \
578 + /opt/rocm-triton-kernels /opt/rocm-triton-kernels
579 +ENV TRITON_KERNELS_SRC_DIR=/opt/rocm-triton-kernels
580
581 # Drop the pre-built Rust artifacts into the source tree. setup.py detects
582 # them and ships them as-is, skipping the local Rust build.
@@ -590,7 +616,9 @@ RUN if [ "$GIT_REPO_CHECK" != "0" ]; then \
616 # This ensures setuptools_scm sees clean repo state for version detection
617 RUN --mount=type=bind,source=.git,target=vllm/.git \
618 --mount=type=cache,target=/root/.cache/uv \
593 - cd vllm \
619 + grep -Fq "set(TRITON_KERNELS_TAG \"${ROCM_TRITON_KERNELS_COMMIT}\")" \
620 + vllm/cmake/external_projects/triton_kernels.cmake \
621 + && cd vllm \
622 && uv pip install --system setuptools_scm regex \
623 && VLLM_VERSION=$(python3 -c "import setuptools_scm; print(setuptools_scm.get_version())") \
624 && echo "Detected vLLM version: ${VLLM_VERSION}" \