[CI][ROCm] Add opt-in TheRock builds for AMD CI (#56351)

Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: Codex <noreply@openai.com>

Andreas Karatzas committed Sep 16, 2026 at 00:09 UTC 711768fc17a82c530a1d3c3f070daf40a76caba4
5 files changed +188 -20
.buildkite/scripts/ci-bake-rocm.sh
+63 -15
@@ -540,6 +540,26 @@ prepare_ci_build_context() {
540 echo "Using canonical CI Docker context: ${ROCM_BUILD_CONTEXT_ROOT}"
541 }
542
543 +configure_custom_rocm_stages() {
544 + using_custom_rocm_dockerfiles || return 0
545 + local context_root="${ROCM_BUILD_CONTEXT_ROOT:-.}"
546 + local dockerfile="${context_root}/${CI_BASE_DOCKERFILE}"
547 + local stages=""
548 +
549 + (cd "${context_root}" && \
550 + validate_rocm_dockerfile "${CI_BASE_DOCKERFILE}" \
551 + ci_base test export_vllm export_test_smoke csrc-build rust-build) \
552 + || return $?
553 + # Hash all stages for custom layouts, including additional helper stages.
554 + stages=$(rocm_dockerfile_stages "${dockerfile}" | paste -sd ' ')
555 + CI_BASE_DOCKERFILE_STAGES="${CI_BASE_DOCKERFILE_STAGES:-${stages}}"
556 + ROCM_CSRC_DOCKERFILE_STAGES="${ROCM_CSRC_DOCKERFILE_STAGES:-${stages}}"
557 + ROCM_RUST_DOCKERFILE_STAGES="${ROCM_RUST_DOCKERFILE_STAGES:-${stages}}"
558 + if [[ " ${stages} " != *" build_rocshmem "* ]]; then
559 + unset ROCSHMEM_BRANCH ROCSHMEM_CACHE_KEY DEEPEP_CACHE_KEY
560 + fi
561 +}
562 +
563 compose_dependency_cache_key() {
564 local prefix="$1"
565 local material="$2"
@@ -569,11 +589,11 @@ hash_dockerfile_stages() {
589 }
590 emit = 0
591 }
572 - $1 == "FROM" {
592 + toupper($1) == "FROM" {
593 stage = ""
594 for (idx = 1; idx <= NF; idx++) {
595 if (tolower($idx) == "as" && idx < NF) {
576 - stage = $(idx + 1)
596 + stage = tolower($(idx + 1))
597 }
598 }
599 emit = (stage in wanted)
@@ -613,11 +633,11 @@ discover_dockerfile_stage_args() {
633 }
634 {
635 line = $0
616 - if ($1 == "FROM") {
636 + if (toupper($1) == "FROM") {
637 stage = ""
638 for (idx = 1; idx <= NF; idx++) {
639 if (tolower($idx) == "as" && idx < NF) {
620 - stage = $(idx + 1)
640 + stage = tolower($(idx + 1))
641 }
642 }
643 emit = (stage in wanted)
@@ -630,7 +650,7 @@ discover_dockerfile_stage_args() {
650 for (idx = 1; idx <= line_count; idx++) {
651 line = lines[idx]
652 arg_name = line
633 - sub(/^[[:space:]]*ARG[[:space:]]+/, "", arg_name)
653 + sub(/^[[:space:]]*[Aa][Rr][Gg][[:space:]]+/, "", arg_name)
654 if (arg_name != line) {
655 sub(/[=[:space:]].*/, "", arg_name)
656 if (arg_name ~ /^[A-Za-z_][A-Za-z0-9_]*$/) {
@@ -716,7 +736,7 @@ extract_dockerfile_arg_default() {
736 if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${dockerfile}" != /* ]]; then
737 physical_dockerfile="${ROCM_BUILD_CONTEXT_ROOT}/${dockerfile}"
738 fi
719 - sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
739 + sed -n -E "s/^[[:space:]]*[Aa][Rr][Gg][[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
740 "${physical_dockerfile}" | head -1
741 }
742
@@ -1000,7 +1020,21 @@ create_and_bootstrap_builder() {
1020 }
1021
1022 init_config() {
1023 + # shellcheck source=.buildkite/scripts/rocm/build-config.sh
1024 + source "$(dirname "${BASH_SOURCE[0]}")/rocm/build-config.sh"
1025 + configure_rocm_build
1026 +
1027 TARGET="${1:-test-ci}"
1028 + if using_custom_rocm_dockerfiles; then
1029 + if [[ -z "${BASE_IMAGE:-}" ]]; then
1030 + echo "Custom ROCm builds require BASE_IMAGE from the selected base handoff" >&2
1031 + return 1
1032 + fi
1033 + if ! is_ci_base_target && [[ -z "${CI_BASE_IMAGE:-}" ]]; then
1034 + echo "Custom ROCm builds require CI_BASE_IMAGE from the selected ci_base handoff" >&2
1035 + return 1
1036 + fi
1037 + fi
1038 BAKE_TARGETS=("${TARGET}")
1039 DEPENDENCY_CACHE_TARGETS=()
1040 CI_HCL_SOURCE="${CI_HCL_SOURCE:-${CI_HCL_FILE:-${DEFAULT_CI_HCL_SOURCE}}}"
@@ -1010,10 +1044,12 @@ init_config() {
1044 PYTORCH_ROCM_ARCH="${PYTORCH_ROCM_ARCH:-gfx90a;gfx942;gfx950}"
1045 CI_BASE_CONTENT_FILES="${CI_BASE_CONTENT_FILES:-${DEFAULT_CI_BASE_CONTENT_FILES}}"
1046 CI_BASE_DOCKERFILE="${CI_BASE_DOCKERFILE:-${DEFAULT_CI_BASE_DOCKERFILE}}"
1013 - CI_BASE_DOCKERFILE_STAGES="${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}"
1047 + if ! using_custom_rocm_dockerfiles; then
1048 + CI_BASE_DOCKERFILE_STAGES="${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}"
1049 + fi
1050 CI_BASE_METADATA_VERSION="${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}"
1051 CI_BASE_IMAGE_TAG="${CI_BASE_IMAGE_TAG:-rocm/vllm-dev:ci_base}"
1016 - export PYTORCH_ROCM_ARCH
1052 + export PYTORCH_ROCM_ARCH CI_BASE_DOCKERFILE
1053
1054 SCRIPT_TMP_DIR=$(mktemp -d -t ci-bake-rocm.XXXXXX)
1055 CI_HCL_PATH="${SCRIPT_TMP_DIR}/ci.hcl"
@@ -1042,6 +1078,11 @@ print_header() {
1078 }
1079
1080 validate_inputs() {
1081 + if using_custom_rocm_dockerfiles; then
1082 + validate_rocm_dockerfile "${ROCM_BASE_DOCKERFILE}" || return $?
1083 + validate_rocm_dockerfile "${CI_BASE_DOCKERFILE}" \
1084 + ci_base test export_vllm export_test_smoke csrc-build rust-build || return $?
1085 + fi
1086 if [[ ! -f "${VLLM_BAKE_FILE}" ]]; then
1087 echo "Error: vLLM bake file not found at ${VLLM_BAKE_FILE}"
1088 echo "Make sure you're running from the vLLM repository root"
@@ -1069,6 +1110,11 @@ load_ci_hcl() {
1110
1111 init_bake_files() {
1112 BAKE_FILES=(-f "${VLLM_BAKE_FILE}" -f "${CI_HCL_PATH}")
1113 + if using_custom_rocm_dockerfiles; then
1114 + # Custom stacks reuse local cache without replacing the
1115 + # standard ROCm registry caches for the same commit or branch.
1116 + BAKE_FILES+=(--set '*.cache-to=')
1117 + fi
1118 }
1119
1120 compute_ci_base_hash_if_needed() {
@@ -1804,7 +1850,7 @@ compute_rocm_csrc_content_hash() {
1850 local -a content_args=()
1851
1852 bake_dir=$(dirname "${VLLM_BAKE_FILE}")
1807 - dockerfile_rocm="${bake_dir}/Dockerfile.rocm"
1853 + dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}"
1854 read -r -a content_paths <<< "${content_files}"
1855 mapfile -t content_args < <(
1856 get_content_arg_names "${dockerfile_rocm}" "${stages}" "${ROCM_CSRC_CONTENT_ARGS:-}"
@@ -1853,7 +1899,7 @@ compute_rocm_rust_content_hash() {
1899 local -a content_args=()
1900
1901 bake_dir=$(dirname "${VLLM_BAKE_FILE}")
1856 - dockerfile_rocm="${bake_dir}/Dockerfile.rocm"
1902 + dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}"
1903 read -r -a content_paths <<< "${content_files}"
1904 mapfile -t content_args < <(
1905 get_content_arg_names \
@@ -1943,7 +1989,7 @@ write_rocm_build_arg_override() {
1989 local arg_value=""
1990
1991 bake_dir=$(dirname "${VLLM_BAKE_FILE}")
1946 - dockerfile_rocm="${bake_dir}/Dockerfile.rocm"
1992 + dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}"
1993 mapfile -t arg_names < <(
1994 {
1995 get_content_arg_names \
@@ -2259,7 +2305,7 @@ extract_dependency_pins() {
2305 local val=""
2306
2307 bake_dir=$(dirname "${VLLM_BAKE_FILE}")
2262 - dockerfile_rocm="${bake_dir}/Dockerfile.rocm"
2308 + dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}"
2309 physical_dockerfile="${dockerfile_rocm}"
2310 if [[ -n "${ROCM_BUILD_CONTEXT_ROOT:-}" && "${dockerfile_rocm}" != /* ]]; then
2311 physical_dockerfile="${ROCM_BUILD_CONTEXT_ROOT}/${dockerfile_rocm}"
@@ -2275,12 +2321,12 @@ extract_dependency_pins() {
2321 fi
2322
2323 val=$(
2278 - sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${var}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
2324 + sed -n -E "s/^[[:space:]]*[Aa][Rr][Gg][[:space:]]+${var}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
2325 "${physical_dockerfile}" | head -1
2326 )
2327 if [[ -n "${val}" ]]; then
2328 export "${var}=${val}"
2283 - echo "Extracted ${var}=${val} from Dockerfile.rocm"
2329 + echo "Extracted ${var}=${val} from ${dockerfile_rocm}"
2330 fi
2331 done
2332 }
@@ -2297,7 +2343,7 @@ compute_dependency_cache_keys() {
2343 local deepep_material=""
2344
2345 bake_dir=$(dirname "${VLLM_BAKE_FILE}")
2300 - dockerfile_rocm="${bake_dir}/Dockerfile.rocm"
2346 + dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}"
2347 nixl_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "NIXL_BRANCH")
2348 ucx_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "UCX_BRANCH")
2349 rocshmem_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "ROCSHMEM_BRANCH")
@@ -2687,6 +2733,7 @@ main() {
2733 init_bake_files
2734 if is_ci_base_target; then
2735 prepare_ci_build_context
2736 + configure_custom_rocm_stages
2737 fi
2738 compute_ci_base_hash_if_needed
2739 configure_ci_base_image_refs
@@ -2697,6 +2744,7 @@ main() {
2744 # version metadata only after that lookup sees the available tag history.
2745 if ! is_ci_base_target; then
2746 prepare_ci_build_context
2747 + configure_custom_rocm_stages
2748 fi
2749 extract_dependency_pins
2750 write_rocm_build_arg_override
.buildkite/scripts/hardware_ci/run-amd-test.sh
+10
@@ -34,6 +34,9 @@
34 ###############################################################################
35 set -o pipefail
36
37 +# shellcheck source=.buildkite/scripts/rocm/build-config.sh
38 +source "$(dirname "${BASH_SOURCE[0]}")/../rocm/build-config.sh" || exit $?
39 +
40 : "${BUILDKIT_PROGRESS:=plain}"
41 : "${TERM:=xterm-256color}"
42 : "${FORCE_COLOR:=1}"
@@ -106,6 +109,10 @@ clear_ci_orchestration_env() {
109 VLLM_CI_REQUIRE_WORKSPACE_MOUNT \
110 VLLM_TEST_COMMANDS \
111 VLLM_CI_BRANCH \
112 + CI_ROCM_DOCKERFILE_BASE \
113 + CI_ROCM_DOCKERFILE \
114 + ROCM_BASE_DOCKERFILE \
115 + CI_BASE_DOCKERFILE \
116 VLLM_CI_BASE_IMAGE \
117 VLLM_CI_FALLBACK_IMAGE \
118 VLLM_CI_DOCKER_DISABLED \
@@ -225,6 +232,9 @@ prepare_artifact_image() {
232 metadata_file=$(find "${artifact_work_dir}" -name "ci-base-image.txt" -type f | head -1)
233 if [[ -n "${metadata_file}" && -s "${metadata_file}" ]]; then
234 base_image=$(tr -d '[:space:]' < "${metadata_file}")
235 + elif using_custom_rocm_dockerfiles; then
236 + echo "Custom ROCm ci_base metadata is missing; using the full CI image"
237 + return 1
238 fi
239
240 echo "--- Preparing local ROCm test image"
.buildkite/scripts/rocm/build-config.sh new
+85
@@ -0,0 +1,85 @@
1 +#!/usr/bin/env bash
2 +# Shared Dockerfile selection for the normal AMD CI build chain.
3 +# CI_ROCM_DOCKERFILE_BASE and CI_ROCM_DOCKERFILE accept repository-relative paths.
4 +# For Rock, set CI_ROCM_DOCKERFILE_BASE=docker/Dockerfile.rock_base and
5 +# CI_ROCM_DOCKERFILE=docker/Dockerfile.rock.
6 +# Custom recipes must preserve the CI targets and artifact layout. Additional
7 +# COPY inputs also need the corresponding *_CONTENT_FILES overrides.
8 +
9 +rocm_base_dockerfile() {
10 + local default="docker/Dockerfile.rocm_base"
11 + printf '%s\n' "${CI_ROCM_DOCKERFILE_BASE:-${ROCM_BASE_DOCKERFILE:-${default}}}"
12 +}
13 +
14 +rocm_ci_dockerfile() {
15 + local default="docker/Dockerfile.rocm"
16 + printf '%s\n' "${CI_ROCM_DOCKERFILE:-${CI_BASE_DOCKERFILE:-${default}}}"
17 +}
18 +
19 +using_custom_rocm_dockerfiles() {
20 + [[ "$(rocm_base_dockerfile)" != "docker/Dockerfile.rocm_base" \
21 + || "$(rocm_ci_dockerfile)" != "docker/Dockerfile.rocm" ]]
22 +}
23 +
24 +rocm_dockerfile_stages() {
25 + awk '
26 + toupper($1) == "FROM" {
27 + for (idx = 2; idx < NF; idx++) {
28 + if (tolower($idx) == "as") print tolower($(idx + 1))
29 + }
30 + }
31 + ' "$1"
32 +}
33 +
34 +validate_rocm_dockerfile() {
35 + local dockerfile="$1"
36 + local stage=""
37 + local stages=""
38 + shift
39 +
40 + case "${dockerfile}" in
41 + /*|../*|*/../*|*/..|*[[:space:]]*)
42 + echo "ROCm Dockerfile must be a repository-relative path without '..' or whitespace: ${dockerfile}" >&2
43 + return 2
44 + ;;
45 + esac
46 + if [[ ! -f "${dockerfile}" ]]; then
47 + echo "ROCm Dockerfile not found: ${dockerfile}" >&2
48 + return 2
49 + fi
50 + if (($#)) && ! awk '
51 + toupper($1) == "FROM" {
52 + named = 0
53 + for (idx = 2; idx < NF; idx++) {
54 + if (tolower($idx) == "as") named = 1
55 + }
56 + if (!named) exit 1
57 + }
58 + ' "${dockerfile}"; then
59 + echo "ROCm Dockerfile must name every build stage: ${dockerfile}" >&2
60 + return 2
61 + fi
62 + stages=$(rocm_dockerfile_stages "${dockerfile}")
63 + for stage in "$@"; do
64 + if ! grep -Fxq -- "${stage}" <<< "${stages}"; then
65 + echo "ROCm Dockerfile ${dockerfile} is missing required stage: ${stage}" >&2
66 + return 2
67 + fi
68 + done
69 +}
70 +
71 +configure_rocm_build() {
72 + ROCM_BASE_DOCKERFILE=$(rocm_base_dockerfile)
73 + CI_BASE_DOCKERFILE=$(rocm_ci_dockerfile)
74 + export ROCM_BASE_DOCKERFILE CI_BASE_DOCKERFILE
75 + using_custom_rocm_dockerfiles || return 0
76 +
77 + # Custom stacks may supply dependencies without separate build stages.
78 + export ROCM_DEP_CACHE_EXPORT_MODE=never
79 +
80 + # Preserve the standard runtime tags, which are unique to each build,
81 + # while preventing experimental builds from promoting stable images.
82 + export ROCM_BASE_PUSH_STABLE_TAG=0
83 + export CI_BASE_PUSH_STABLE_TAG=0
84 + echo "Custom ROCm Dockerfiles: ${ROCM_BASE_DOCKERFILE} and ${CI_BASE_DOCKERFILE}"
85 +}
.buildkite/scripts/rocm/refresh-base-image.sh
+25 -4
@@ -4,6 +4,10 @@
4
5 set -euo pipefail
6
7 +# shellcheck source=.buildkite/scripts/rocm/build-config.sh
8 +source "$(dirname "${BASH_SOURCE[0]}")/build-config.sh"
9 +configure_rocm_build
10 +
11 DOCKERFILE="${ROCM_BASE_DOCKERFILE:-docker/Dockerfile.rocm_base}"
12 BASE_REPO="${ROCM_BASE_IMAGE_REPO:-rocm/vllm-dev}"
13 CACHE_REPO="${ROCM_BASE_CACHE_REPO:-${DOCKERHUB_CACHE_REPO:-rocm/vllm-ci-cache}}"
@@ -12,7 +16,12 @@ DEFAULT_ROCM_BASE_METADATA_VERSION="2"
16 DEFAULT_ROCM_BASE_CONTENT_FILES="${DOCKERFILE}"
17
18 ROCM_BASE_LAYER_CACHE_REF=""
15 -ROCM_BASE_TRUSTED_LAYER_CACHE_REF="${CACHE_REPO}:rocm-base-main"
19 +BASE_CACHE_PREFIX="rocm-base"
20 +if using_custom_rocm_dockerfiles; then
21 + BASE_CACHE_PREFIX="rocm-base-$(printf '%s\n%s\n' \
22 + "${ROCM_BASE_DOCKERFILE}" "${CI_BASE_DOCKERFILE}" | sha256sum | cut -c1-12)"
23 +fi
24 +ROCM_BASE_TRUSTED_LAYER_CACHE_REF="${CACHE_REPO}:${BASE_CACHE_PREFIX}-main"
25 ROCM_BASE_STABLE_TAG_UPDATED=0
26 declare -a ROCM_BASE_CACHE_ARGS=()
27
@@ -120,7 +129,7 @@ configure_rocm_base_layer_cache() {
129 fi
130
131 scope=$(rocm_base_layer_cache_scope)
123 - ROCM_BASE_LAYER_CACHE_REF="${CACHE_REPO}:rocm-base-${scope}"
132 + ROCM_BASE_LAYER_CACHE_REF="${CACHE_REPO}:${BASE_CACHE_PREFIX}-${scope}"
133 ROCM_BASE_CACHE_ARGS+=(
134 --cache-from "type=registry,ref=${ROCM_BASE_LAYER_CACHE_REF}"
135 )
@@ -139,7 +148,7 @@ configure_rocm_base_layer_cache() {
148 extract_arg_default() {
149 local arg_name="$1"
150
142 - sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
151 + sed -n -E "s/^[[:space:]]*[Aa][Rr][Gg][[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
152 "${DOCKERFILE}" | head -1
153 }
154
@@ -419,11 +428,16 @@ build_base_image() {
428 read -r -a content_paths <<< "${content_files}"
429 content_files_hash="$(compute_content_hash "${content_paths[@]}")"
430 base_hash=$(compute_base_content_hash "${use_sccache}" "${base_image_digest}")
422 - rocm_version="$(rocm_version_from_base_image "${base_image_arg}")"
431 + rocm_version="$(extract_arg_default ROCM_SDK_VERSION)"
432 + rocm_version="${rocm_version:-$(rocm_version_from_base_image "${base_image_arg}")}"
433 triton_arg="$(extract_arg_default TRITON_BRANCH)"
434 pytorch_arg="$(extract_arg_default PYTORCH_BRANCH)"
435 pytorch_vision_arg="$(extract_arg_default PYTORCH_VISION_BRANCH)"
436 pytorch_audio_arg="$(extract_arg_default PYTORCH_AUDIO_BRANCH)"
437 + triton_arg="${triton_arg:-$(extract_arg_default TRITON_VERSION)}"
438 + pytorch_arg="${pytorch_arg:-$(extract_arg_default TORCH_VERSION)}"
439 + pytorch_vision_arg="${pytorch_vision_arg:-$(extract_arg_default TORCHVISION_VERSION)}"
440 + pytorch_audio_arg="${pytorch_audio_arg:-$(extract_arg_default TORCHAUDIO_VERSION)}"
441 fa_arg="$(extract_arg_default FA_BRANCH)"
442 aiter_arg="$(extract_arg_default AITER_BRANCH)"
443 mori_arg="$(extract_arg_default MORI_BRANCH)"
@@ -569,6 +583,13 @@ build_base_image() {
583 }
584
585 main() {
586 + if using_custom_rocm_dockerfiles; then
587 + if [[ "${ROCM_BASE_REFRESH_SKIP:-0}" == "1" ]]; then
588 + echo "Custom ROCm Dockerfiles require ROCM_BASE_REFRESH_SKIP=0" >&2
589 + return 2
590 + fi
591 + validate_rocm_dockerfile "${DOCKERFILE}" || return $?
592 + fi
593 metadata_set "rocm-base-refresh" "0"
594
595 if [[ "${ROCM_BASE_REFRESH_SKIP:-0}" == "1" ]]; then
docker/docker-bake-rocm.hcl
+5 -1
@@ -33,6 +33,10 @@ variable "CI_BASE_CONTENT_HASH" {
33 default = ""
34 }
35
36 +variable "CI_BASE_DOCKERFILE" {
37 + default = "docker/Dockerfile.rocm"
38 +}
39 +
40 # REMOTE_VLLM=0: use local source via Docker build context (ONBUILD COPY ./ vllm/)
41 # REMOTE_VLLM=1: clone from GitHub at VLLM_BRANCH (standalone builds without local source)
42 variable "REMOTE_VLLM" {
@@ -74,7 +78,7 @@ group "default" {
78 }
79
80 target "_common-rocm" {
77 - dockerfile = "docker/Dockerfile.rocm"
81 + dockerfile = CI_BASE_DOCKERFILE
82 context = "."
83 args = {
84 max_jobs = MAX_JOBS