| 1 | #!/usr/bin/env bash |
| 2 | # Select the content-addressed ROCm base for this checkout, building it only |
| 3 | # when the exact content ref is absent. |
| 4 | |
| 5 | set -euo pipefail |
| 6 | |
| 7 | # shellcheck source=.buildkite/scripts/rocm/build-config.sh |
| 8 | source "$(dirname "${BASH_SOURCE[0]}")/build-config.sh" |
| 9 | configure_rocm_build |
| 10 | |
| 11 | DOCKERFILE="${ROCM_BASE_DOCKERFILE:-docker/Dockerfile.rocm_base}" |
| 12 | BASE_REPO="${ROCM_BASE_IMAGE_REPO:-rocm/vllm-dev}" |
| 13 | CACHE_REPO="${ROCM_BASE_CACHE_REPO:-${DOCKERHUB_CACHE_REPO:-rocm/vllm-ci-cache}}" |
| 14 | BUILDER_NAME="${ROCM_BASE_BUILDER_NAME:-vllm-rocm-base-builder}" |
| 15 | DEFAULT_ROCM_BASE_METADATA_VERSION="2" |
| 16 | DEFAULT_ROCM_BASE_CONTENT_FILES="${DOCKERFILE}" |
| 17 | |
| 18 | ROCM_BASE_LAYER_CACHE_REF="" |
| 19 | BASE_CACHE_PREFIX="rocm-base" |
| 20 | if using_custom_rocm_dockerfiles; then |
| 21 | BASE_CACHE_PREFIX="rocm-base-$(printf '%s\n%s\n' \ |
| 22 | "${ROCM_BASE_DOCKERFILE}" "${CI_BASE_DOCKERFILE}" | sha256sum | cut -c1-12)" |
| 23 | fi |
| 24 | ROCM_BASE_TRUSTED_LAYER_CACHE_REF="${CACHE_REPO}:${BASE_CACHE_PREFIX}-main" |
| 25 | ROCM_BASE_STABLE_TAG_UPDATED=0 |
| 26 | declare -a ROCM_BASE_CACHE_ARGS=() |
| 27 | |
| 28 | metadata_set() { |
| 29 | local key="$1" |
| 30 | local value="$2" |
| 31 | |
| 32 | [[ -n "${value}" ]] || return 0 |
| 33 | if command -v buildkite-agent >/dev/null 2>&1; then |
| 34 | buildkite-agent meta-data set "${key}" "${value}" |
| 35 | elif [[ "${BUILDKITE:-false}" == "true" ]]; then |
| 36 | echo "buildkite-agent is required to publish ROCm base metadata" >&2 |
| 37 | return 1 |
| 38 | fi |
| 39 | } |
| 40 | |
| 41 | compute_content_hash() { |
| 42 | local path="" |
| 43 | local file="" |
| 44 | |
| 45 | for path in "$@"; do |
| 46 | if [[ -d "${path}" ]]; then |
| 47 | while IFS= read -r -d '' file; do |
| 48 | printf 'file:%s\n' "${file}" |
| 49 | sha256sum "${file}" |
| 50 | done < <(find "${path}" -type f -print0 | sort -z) |
| 51 | elif [[ -f "${path}" ]]; then |
| 52 | printf 'file:%s\n' "${path}" |
| 53 | sha256sum "${path}" |
| 54 | else |
| 55 | printf 'missing:%s\n' "${path}" |
| 56 | fi |
| 57 | done | sha256sum | cut -d' ' -f1 |
| 58 | } |
| 59 | |
| 60 | clean_docker_tag() { |
| 61 | local input="$1" |
| 62 | echo "${input}" | sed 's/[^a-zA-Z0-9._-]/_/g' | cut -c1-128 |
| 63 | } |
| 64 | |
| 65 | tag_component() { |
| 66 | local input="$1" |
| 67 | local max_chars="${2:-24}" |
| 68 | |
| 69 | clean_docker_tag "${input:-unknown}" | cut -c1-"${max_chars}" |
| 70 | } |
| 71 | |
| 72 | normalize_repo_slug() { |
| 73 | local repo_slug="${1:-}" |
| 74 | |
| 75 | repo_slug="${repo_slug%/}" |
| 76 | repo_slug="${repo_slug%.git}" |
| 77 | repo_slug="${repo_slug#https://github.com/}" |
| 78 | repo_slug="${repo_slug#http://github.com/}" |
| 79 | repo_slug="${repo_slug#ssh://git@github.com/}" |
| 80 | repo_slug="${repo_slug#git@github.com:}" |
| 81 | repo_slug="${repo_slug#github.com/}" |
| 82 | printf '%s\n' "${repo_slug}" |
| 83 | } |
| 84 | |
| 85 | is_trusted_main_build() { |
| 86 | local actual_repo="" |
| 87 | local trusted_repo="" |
| 88 | |
| 89 | [[ "${BUILDKITE:-false}" == "true" ]] || return 1 |
| 90 | [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" ]] || return 1 |
| 91 | [[ "${BUILDKITE_BRANCH:-}" == "${ROCM_BASE_STABLE_BRANCH:-main}" ]] || return 1 |
| 92 | actual_repo=$(normalize_repo_slug "${BUILDKITE_REPO:-}") |
| 93 | trusted_repo=$(normalize_repo_slug \ |
| 94 | "${ROCM_BASE_STABLE_REPO_SLUG:-vllm-project/vllm}") |
| 95 | [[ -n "${actual_repo}" && "${actual_repo}" == "${trusted_repo}" ]] |
| 96 | } |
| 97 | |
| 98 | rocm_base_layer_cache_scope() { |
| 99 | local pull_request="${BUILDKITE_PULL_REQUEST:-false}" |
| 100 | local branch="${BUILDKITE_PULL_REQUEST_HEAD_BRANCH:-${BUILDKITE_BRANCH:-local}}" |
| 101 | local identity="" |
| 102 | local repo_slug="" |
| 103 | |
| 104 | if is_trusted_main_build; then |
| 105 | printf 'main\n' |
| 106 | return 0 |
| 107 | fi |
| 108 | if [[ "${pull_request}" != "false" && -n "${pull_request}" ]]; then |
| 109 | repo_slug=$(normalize_repo_slug "${BUILDKITE_REPO:-local}") |
| 110 | identity=$(printf '%s\n' "${repo_slug:-local}" | sha256sum | cut -c1-12) |
| 111 | printf 'pr-%s-%s\n' \ |
| 112 | "$(tag_component "${pull_request}" 32)" "${identity}" |
| 113 | return 0 |
| 114 | fi |
| 115 | |
| 116 | identity=$(printf '%s\n%s\n' "${BUILDKITE_REPO:-local}" "${branch}" \ |
| 117 | | sha256sum | cut -c1-12) |
| 118 | printf 'preview-%s-%s\n' "$(tag_component "${branch}" 24)" "${identity}" |
| 119 | } |
| 120 | |
| 121 | configure_rocm_base_layer_cache() { |
| 122 | local scope="" |
| 123 | |
| 124 | ROCM_BASE_CACHE_ARGS=() |
| 125 | if [[ "${ROCM_BASE_NO_CACHE:-0}" == "1" ]]; then |
| 126 | ROCM_BASE_CACHE_ARGS+=(--no-cache) |
| 127 | ROCM_BASE_LAYER_CACHE_REF="disabled" |
| 128 | return 0 |
| 129 | fi |
| 130 | |
| 131 | scope=$(rocm_base_layer_cache_scope) |
| 132 | ROCM_BASE_LAYER_CACHE_REF="${CACHE_REPO}:${BASE_CACHE_PREFIX}-${scope}" |
| 133 | ROCM_BASE_CACHE_ARGS+=( |
| 134 | --cache-from "type=registry,ref=${ROCM_BASE_LAYER_CACHE_REF}" |
| 135 | ) |
| 136 | if [[ "${ROCM_BASE_LAYER_CACHE_REF}" != \ |
| 137 | "${ROCM_BASE_TRUSTED_LAYER_CACHE_REF}" ]]; then |
| 138 | ROCM_BASE_CACHE_ARGS+=( |
| 139 | --cache-from "type=registry,ref=${ROCM_BASE_TRUSTED_LAYER_CACHE_REF}" |
| 140 | ) |
| 141 | fi |
| 142 | ROCM_BASE_CACHE_ARGS+=( |
| 143 | --cache-to \ |
| 144 | "type=registry,ref=${ROCM_BASE_LAYER_CACHE_REF},mode=max,ignore-error=true" |
| 145 | ) |
| 146 | } |
| 147 | |
| 148 | extract_arg_default() { |
| 149 | local arg_name="$1" |
| 150 | |
| 151 | sed -n -E "s/^[[:space:]]*[Aa][Rr][Gg][[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \ |
| 152 | "${DOCKERFILE}" | head -1 |
| 153 | } |
| 154 | |
| 155 | resolve_image_digest() { |
| 156 | local image_ref="$1" |
| 157 | local attempts="${ROCM_IMAGE_DIGEST_ATTEMPTS:-4}" |
| 158 | local delay_secs="${ROCM_IMAGE_DIGEST_RETRY_DELAY:-2}" |
| 159 | local attempt=0 |
| 160 | local output="" |
| 161 | local digest="" |
| 162 | local status=0 |
| 163 | |
| 164 | if [[ "${image_ref}" =~ @(sha256:[0-9a-f]{64})$ ]]; then |
| 165 | printf '%s\n' "${BASH_REMATCH[1]}" |
| 166 | return 0 |
| 167 | fi |
| 168 | if [[ ! "${attempts}" =~ ^[1-9][0-9]*$ \ |
| 169 | || ! "${delay_secs}" =~ ^[0-9]+$ ]]; then |
| 170 | echo "Invalid image digest retry configuration" >&2 |
| 171 | return 1 |
| 172 | fi |
| 173 | |
| 174 | for ((attempt = 1; attempt <= attempts; attempt++)); do |
| 175 | status=0 |
| 176 | output=$(docker buildx imagetools inspect "${image_ref}" 2>&1) || status=$? |
| 177 | digest=$(awk '$1 == "Digest:" { print $2; exit }' <<< "${output}") |
| 178 | if ((status == 0)) && [[ "${digest}" =~ ^sha256:[0-9a-f]{64}$ ]]; then |
| 179 | printf '%s\n' "${digest}" |
| 180 | return 0 |
| 181 | fi |
| 182 | if ((attempt < attempts)); then |
| 183 | printf \ |
| 184 | 'Image digest lookup failed for %s (%d/%d, status %d); retrying\n' \ |
| 185 | "${image_ref}" "${attempt}" "${attempts}" "${status}" >&2 |
| 186 | sleep "${delay_secs}" |
| 187 | fi |
| 188 | done |
| 189 | |
| 190 | printf 'Failed to resolve digest for %s (status %d)\n%s\n' \ |
| 191 | "${image_ref}" "${status}" "${output:-<no output>}" >&2 |
| 192 | return 1 |
| 193 | } |
| 194 | |
| 195 | trusted_base_content_ref() { |
| 196 | local base_hash="$1" |
| 197 | local metadata_version="$2" |
| 198 | |
| 199 | printf '%s:base-v%s-%s\n' "${BASE_REPO}" "${metadata_version}" "${base_hash}" |
| 200 | } |
| 201 | |
| 202 | scoped_base_content_ref() { |
| 203 | local base_hash="$1" |
| 204 | local metadata_version="$2" |
| 205 | |
| 206 | printf '%s:base-v%s-preview-%s\n' \ |
| 207 | "${BASE_REPO}" "${metadata_version}" "${base_hash}" |
| 208 | } |
| 209 | |
| 210 | find_matching_base_content_ref() { |
| 211 | local expected_hash="$1" |
| 212 | local expected_version="$2" |
| 213 | local image_ref="" |
| 214 | local digest="" |
| 215 | local immutable_ref="" |
| 216 | local label_values="" |
| 217 | local remote_hash="" |
| 218 | local remote_version="" |
| 219 | local attempts="${ROCM_IMAGE_DIGEST_ATTEMPTS:-4}" |
| 220 | local delay_secs="${ROCM_IMAGE_DIGEST_RETRY_DELAY:-2}" |
| 221 | local attempt=0 |
| 222 | local inspect_status=0 |
| 223 | shift 2 |
| 224 | |
| 225 | if [[ ! "${attempts}" =~ ^[1-9][0-9]*$ \ |
| 226 | || ! "${delay_secs}" =~ ^[0-9]+$ ]]; then |
| 227 | echo "Invalid image identity retry configuration" >&2 |
| 228 | return 1 |
| 229 | fi |
| 230 | |
| 231 | for image_ref in "$@"; do |
| 232 | if ! digest=$(resolve_image_digest "${image_ref}"); then |
| 233 | continue |
| 234 | fi |
| 235 | immutable_ref="${image_ref}@${digest}" |
| 236 | label_values="" |
| 237 | for ((attempt = 1; attempt <= attempts; attempt++)); do |
| 238 | inspect_status=0 |
| 239 | label_values=$(docker buildx imagetools inspect "${immutable_ref}" \ |
| 240 | --format '{{ index .Image.Config.Labels "vllm.rocm_base.content_hash" }} {{ index .Image.Config.Labels "vllm.rocm_base.metadata_version" }}' \ |
| 241 | 2>/dev/null) || inspect_status=$? |
| 242 | remote_hash="" |
| 243 | remote_version="" |
| 244 | if ((inspect_status == 0)) && [[ -n "${label_values}" ]]; then |
| 245 | read -r remote_hash remote_version _ <<< "${label_values}" |
| 246 | if [[ "${remote_hash}" == "${expected_hash}" \ |
| 247 | && "${remote_version}" == "${expected_version}" ]]; then |
| 248 | printf '%s\n' "${immutable_ref}" |
| 249 | return 0 |
| 250 | fi |
| 251 | fi |
| 252 | if ((attempt < attempts)); then |
| 253 | printf \ |
| 254 | 'ROCm base identity lookup incomplete or mismatched for %s (%d/%d); retrying\n' \ |
| 255 | "${immutable_ref}" "${attempt}" "${attempts}" >&2 |
| 256 | sleep "${delay_secs}" |
| 257 | fi |
| 258 | done |
| 259 | if ((inspect_status == 0)) \ |
| 260 | && [[ -n "${remote_hash}" && -n "${remote_version}" ]]; then |
| 261 | echo "ROCm base content ref has unexpected identity: ${immutable_ref}" >&2 |
| 262 | echo " expected hash/version: ${expected_hash} / ${expected_version}" >&2 |
| 263 | echo " found hash/version: ${remote_hash} / ${remote_version}" >&2 |
| 264 | # A content-addressed ref with the wrong identity is corrupt. The |
| 265 | # stable alias is allowed to describe older content and is only a |
| 266 | # migration/recovery candidate. |
| 267 | if [[ "${image_ref}" != "${BASE_REPO}:base" ]]; then |
| 268 | return 2 |
| 269 | fi |
| 270 | fi |
| 271 | echo "Failed to read complete ROCm base identity labels: ${immutable_ref}" >&2 |
| 272 | done |
| 273 | return 1 |
| 274 | } |
| 275 | |
| 276 | rocm_version_from_base_image() { |
| 277 | local base_image="$1" |
| 278 | local version="" |
| 279 | |
| 280 | version="$(sed -n -E 's/.*:([0-9]+\.[0-9]+(\.[0-9]+)?)-.*/\1/p' <<<"${base_image}")" |
| 281 | tag_component "${version:-${base_image}}" 16 |
| 282 | } |
| 283 | |
| 284 | should_push_stable_tag() { |
| 285 | if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then |
| 286 | return 1 |
| 287 | fi |
| 288 | |
| 289 | if [[ "${ROCM_BASE_PUSH_STABLE_TAG:-}" == "0" ]]; then |
| 290 | return 1 |
| 291 | fi |
| 292 | |
| 293 | is_trusted_main_build |
| 294 | } |
| 295 | |
| 296 | trusted_main_tip_matches_build() { |
| 297 | local branch="${ROCM_BASE_STABLE_BRANCH:-main}" |
| 298 | local build_commit="${BUILDKITE_COMMIT:-}" |
| 299 | local remote_tip="" |
| 300 | |
| 301 | is_trusted_main_build || return 1 |
| 302 | if [[ ! "${build_commit}" =~ ^[0-9a-fA-F]{40}$ ]]; then |
| 303 | echo "Skipping ROCm stable tag: Buildkite commit is missing or invalid" >&2 |
| 304 | return 1 |
| 305 | fi |
| 306 | remote_tip=$(git ls-remote --exit-code "${BUILDKITE_REPO}" \ |
| 307 | "refs/heads/${branch}" 2>/dev/null | awk 'NR == 1 { print $1 }') |
| 308 | if [[ ! "${remote_tip}" =~ ^[0-9a-fA-F]{40}$ ]]; then |
| 309 | echo "Skipping ROCm stable tag: could not resolve remote ${branch} tip" >&2 |
| 310 | return 1 |
| 311 | fi |
| 312 | if [[ "${remote_tip,,}" != "${build_commit,,}" ]]; then |
| 313 | echo "Skipping ROCm stable tag: ${branch} changed after ${build_commit} (${remote_tip})" >&2 |
| 314 | return 1 |
| 315 | fi |
| 316 | return 0 |
| 317 | } |
| 318 | |
| 319 | promote_stable_base_tag() { |
| 320 | local source_ref="$1" |
| 321 | local stable_tag="$2" |
| 322 | |
| 323 | ROCM_BASE_STABLE_TAG_UPDATED=0 |
| 324 | should_push_stable_tag && trusted_main_tip_matches_build || return 0 |
| 325 | |
| 326 | docker buildx imagetools create --prefer-index=false \ |
| 327 | -t "${stable_tag}" "${source_ref}" |
| 328 | ROCM_BASE_STABLE_TAG_UPDATED=1 |
| 329 | } |
| 330 | |
| 331 | setup_builder() { |
| 332 | echo "--- :buildkite: Setting up buildx builder for ROCm base" |
| 333 | if docker buildx inspect "${BUILDER_NAME}" >/dev/null 2>&1; then |
| 334 | docker buildx use "${BUILDER_NAME}" |
| 335 | else |
| 336 | docker buildx create --name "${BUILDER_NAME}" --driver docker-container --use |
| 337 | fi |
| 338 | docker buildx inspect --bootstrap |
| 339 | } |
| 340 | |
| 341 | compute_base_content_hash() { |
| 342 | local use_sccache="$1" |
| 343 | local base_image_digest="$2" |
| 344 | local content_files="${ROCM_BASE_CONTENT_FILES:-${DEFAULT_ROCM_BASE_CONTENT_FILES}}" |
| 345 | local sccache_arg="" |
| 346 | local sccache_value="" |
| 347 | local -a content_paths=() |
| 348 | |
| 349 | read -r -a content_paths <<< "${content_files}" |
| 350 | |
| 351 | { |
| 352 | printf 'content-files-hash:%s\n' "$(compute_content_hash "${content_paths[@]}")" |
| 353 | printf 'dockerfile:%s\n' "${DOCKERFILE}" |
| 354 | printf 'base-image-digest:%s\n' "${base_image_digest}" |
| 355 | printf 'use-sccache:%s\n' "${use_sccache}" |
| 356 | if [[ "${use_sccache}" == "1" ]]; then |
| 357 | # These values can change the installed binary or final image |
| 358 | # configuration. SCCACHE_ENDPOINT is transport-only and therefore |
| 359 | # deliberately excluded from image identity. |
| 360 | for sccache_arg in \ |
| 361 | SCCACHE_DOWNLOAD_URL \ |
| 362 | SCCACHE_BUCKET_NAME \ |
| 363 | SCCACHE_REGION_NAME \ |
| 364 | SCCACHE_S3_NO_CREDENTIALS; do |
| 365 | sccache_value="${!sccache_arg:-}" |
| 366 | if [[ -z "${sccache_value}" ]]; then |
| 367 | sccache_value=$(extract_arg_default "${sccache_arg}") |
| 368 | fi |
| 369 | printf 'arg:%s=%s\n' \ |
| 370 | "${sccache_arg}" "${sccache_value:-<empty>}" |
| 371 | done |
| 372 | fi |
| 373 | } | sha256sum | cut -d' ' -f1 |
| 374 | } |
| 375 | |
| 376 | build_base_image() { |
| 377 | local use_sccache="${ROCM_BASE_USE_SCCACHE:-${USE_SCCACHE:-0}}" |
| 378 | local base_hash="" |
| 379 | local base_image_arg="" |
| 380 | local base_image_digest="" |
| 381 | local pinned_base_image="" |
| 382 | local rocm_version="" |
| 383 | local triton_arg="" |
| 384 | local pytorch_arg="" |
| 385 | local pytorch_vision_arg="" |
| 386 | local pytorch_audio_arg="" |
| 387 | local fa_arg="" |
| 388 | local aiter_arg="" |
| 389 | local mori_arg="" |
| 390 | local python_version_arg="" |
| 391 | local pytorch_rocm_arch_arg="" |
| 392 | local dependency_summary="" |
| 393 | local stable_tag="${BASE_REPO}:base" |
| 394 | local trusted_content_tag="" |
| 395 | local scoped_content_tag="" |
| 396 | local preview_content_tag="" |
| 397 | local writable_content_tag="" |
| 398 | local build_ref="" |
| 399 | local immutable_ref="" |
| 400 | local selected_digest="" |
| 401 | local stable_digest="" |
| 402 | local reuse_status=0 |
| 403 | local content_files="${ROCM_BASE_CONTENT_FILES:-${DEFAULT_ROCM_BASE_CONTENT_FILES}}" |
| 404 | local content_files_hash="" |
| 405 | local metadata_version="${ROCM_BASE_METADATA_VERSION:-${DEFAULT_ROCM_BASE_METADATA_VERSION}}" |
| 406 | local -a sccache_args=() |
| 407 | local -a content_paths=() |
| 408 | |
| 409 | if [[ ! -f "${DOCKERFILE}" ]]; then |
| 410 | echo "Error: ROCm base Dockerfile not found: ${DOCKERFILE}" >&2 |
| 411 | exit 1 |
| 412 | fi |
| 413 | if [[ ! "${metadata_version}" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,15}$ ]]; then |
| 414 | echo "Invalid ROCm base metadata version: ${metadata_version}" >&2 |
| 415 | return 1 |
| 416 | fi |
| 417 | if [[ "${use_sccache}" != "0" && "${use_sccache}" != "1" ]]; then |
| 418 | echo "ROCm base USE_SCCACHE must be 0 or 1: ${use_sccache}" >&2 |
| 419 | return 1 |
| 420 | fi |
| 421 | |
| 422 | base_image_arg="$(extract_arg_default BASE_IMAGE)" |
| 423 | if ! base_image_digest=$(resolve_image_digest "${base_image_arg}"); then |
| 424 | echo "Failed to resolve ROCm base input image: ${base_image_arg}" >&2 |
| 425 | return 1 |
| 426 | fi |
| 427 | pinned_base_image="${base_image_arg%@*}@${base_image_digest}" |
| 428 | read -r -a content_paths <<< "${content_files}" |
| 429 | content_files_hash="$(compute_content_hash "${content_paths[@]}")" |
| 430 | base_hash=$(compute_base_content_hash "${use_sccache}" "${base_image_digest}") |
| 431 | rocm_version="$(extract_arg_default ROCM_SDK_VERSION)" |
| 432 | rocm_version="${rocm_version:-$(rocm_version_from_base_image "${base_image_arg}")}" |
| 433 | triton_arg="$(extract_arg_default TRITON_BRANCH)" |
| 434 | pytorch_arg="$(extract_arg_default PYTORCH_BRANCH)" |
| 435 | pytorch_vision_arg="$(extract_arg_default PYTORCH_VISION_BRANCH)" |
| 436 | pytorch_audio_arg="$(extract_arg_default PYTORCH_AUDIO_BRANCH)" |
| 437 | triton_arg="${triton_arg:-$(extract_arg_default TRITON_VERSION)}" |
| 438 | pytorch_arg="${pytorch_arg:-$(extract_arg_default TORCH_VERSION)}" |
| 439 | pytorch_vision_arg="${pytorch_vision_arg:-$(extract_arg_default TORCHVISION_VERSION)}" |
| 440 | pytorch_audio_arg="${pytorch_audio_arg:-$(extract_arg_default TORCHAUDIO_VERSION)}" |
| 441 | fa_arg="$(extract_arg_default FA_BRANCH)" |
| 442 | aiter_arg="$(extract_arg_default AITER_BRANCH)" |
| 443 | mori_arg="$(extract_arg_default MORI_BRANCH)" |
| 444 | python_version_arg="$(extract_arg_default PYTHON_VERSION)" |
| 445 | pytorch_rocm_arch_arg="$(extract_arg_default PYTORCH_ROCM_ARCH)" |
| 446 | dependency_summary="base=${base_image_arg},rocm=${rocm_version},python=${python_version_arg},pytorch=${pytorch_arg},torchvision=${pytorch_vision_arg},torchaudio=${pytorch_audio_arg},triton=${triton_arg},flash-attn=${fa_arg},aiter=${aiter_arg},mori=${mori_arg},pytorch-rocm-arch=${pytorch_rocm_arch_arg}" |
| 447 | trusted_content_tag=$(trusted_base_content_ref "${base_hash}" "${metadata_version}") |
| 448 | scoped_content_tag=$(scoped_base_content_ref "${base_hash}" "${metadata_version}") |
| 449 | preview_content_tag="${scoped_content_tag}" |
| 450 | # Preview writes share an exact-content namespace so stacked PRs can reuse |
| 451 | # identical images. They may import the trusted ref, but never overwrite it. |
| 452 | if is_trusted_main_build; then |
| 453 | scoped_content_tag="${trusted_content_tag}" |
| 454 | writable_content_tag="${trusted_content_tag}" |
| 455 | else |
| 456 | writable_content_tag="${scoped_content_tag}" |
| 457 | fi |
| 458 | |
| 459 | configure_rocm_base_layer_cache |
| 460 | |
| 461 | if [[ "${use_sccache}" == "1" ]]; then |
| 462 | for env_name in \ |
| 463 | SCCACHE_DOWNLOAD_URL \ |
| 464 | SCCACHE_ENDPOINT \ |
| 465 | SCCACHE_BUCKET_NAME \ |
| 466 | SCCACHE_REGION_NAME \ |
| 467 | SCCACHE_S3_NO_CREDENTIALS; do |
| 468 | if [[ -n "${!env_name:-}" ]]; then |
| 469 | sccache_args+=(--build-arg "${env_name}=${!env_name}") |
| 470 | fi |
| 471 | done |
| 472 | fi |
| 473 | |
| 474 | echo "--- :docker: Preparing ROCm base image" |
| 475 | echo "Dockerfile: ${DOCKERFILE}" |
| 476 | echo "Trusted content tag: ${trusted_content_tag}" |
| 477 | echo "Writable content tag: ${writable_content_tag}" |
| 478 | echo "Stable tag: ${stable_tag} ($(should_push_stable_tag && echo enabled || echo disabled))" |
| 479 | echo "Content hash: ${base_hash}" |
| 480 | echo "Dependency summary: ${dependency_summary}" |
| 481 | echo "USE_SCCACHE: ${use_sccache}" |
| 482 | echo "BuildKit layer cache: ${ROCM_BASE_LAYER_CACHE_REF}" |
| 483 | |
| 484 | if [[ "${ROCM_BASE_REFRESH_FORCE:-0}" != "1" \ |
| 485 | && "${ROCM_BASE_NO_CACHE:-0}" != "1" ]]; then |
| 486 | reuse_status=0 |
| 487 | if [[ "${trusted_content_tag}" == "${scoped_content_tag}" ]]; then |
| 488 | immutable_ref=$(find_matching_base_content_ref \ |
| 489 | "${base_hash}" "${metadata_version}" \ |
| 490 | "${trusted_content_tag}" "${preview_content_tag}" \ |
| 491 | "${stable_tag}") || reuse_status=$? |
| 492 | else |
| 493 | immutable_ref=$(find_matching_base_content_ref \ |
| 494 | "${base_hash}" "${metadata_version}" \ |
| 495 | "${trusted_content_tag}" "${scoped_content_tag}" \ |
| 496 | "${stable_tag}") \ |
| 497 | || reuse_status=$? |
| 498 | fi |
| 499 | else |
| 500 | reuse_status=1 |
| 501 | echo "Forced/no-cache ROCm base build; bypassing content image reuse" |
| 502 | fi |
| 503 | |
| 504 | if ((reuse_status > 1)); then |
| 505 | return "${reuse_status}" |
| 506 | fi |
| 507 | |
| 508 | if [[ ${reuse_status} -eq 0 && -n "${immutable_ref}" ]]; then |
| 509 | echo "Reusing ROCm base image with matching content: ${immutable_ref}" |
| 510 | if [[ "${immutable_ref%@*}" != "${writable_content_tag}" \ |
| 511 | && ( "${immutable_ref%@*}" == "${stable_tag}" \ |
| 512 | || "${immutable_ref%@*}" == "${preview_content_tag}" ) ]]; then |
| 513 | echo "Backfilling ROCm base content tag: ${writable_content_tag}" |
| 514 | docker buildx imagetools create --prefer-index=false \ |
| 515 | -t "${writable_content_tag}" "${immutable_ref}" |
| 516 | if ! immutable_ref=$(find_matching_base_content_ref \ |
| 517 | "${base_hash}" "${metadata_version}" \ |
| 518 | "${writable_content_tag}"); then |
| 519 | echo "Backfilled ROCm base content tag failed identity validation" >&2 |
| 520 | return 1 |
| 521 | fi |
| 522 | fi |
| 523 | else |
| 524 | build_ref="${writable_content_tag}" |
| 525 | echo "No reusable ROCm base content image found; building ${build_ref}" |
| 526 | setup_builder |
| 527 | docker buildx build \ |
| 528 | "${ROCM_BASE_CACHE_ARGS[@]}" \ |
| 529 | --pull \ |
| 530 | --provenance=false \ |
| 531 | --progress "${BUILDKIT_PROGRESS:-plain}" \ |
| 532 | --file "${DOCKERFILE}" \ |
| 533 | --build-arg "BASE_IMAGE=${pinned_base_image}" \ |
| 534 | --build-arg "USE_SCCACHE=${use_sccache}" \ |
| 535 | "${sccache_args[@]}" \ |
| 536 | --label "org.opencontainers.image.source=https://github.com/vllm-project/vllm" \ |
| 537 | --label "org.opencontainers.image.vendor=vLLM" \ |
| 538 | --label "org.opencontainers.image.title=vLLM ROCm base" \ |
| 539 | --label "vllm.rocm_base.metadata_version=${metadata_version}" \ |
| 540 | --label "vllm.rocm_base.content_hash=${base_hash}" \ |
| 541 | --label "vllm.rocm_base.content_files_hash=${content_files_hash}" \ |
| 542 | --label "vllm.rocm_base.dockerfile=${DOCKERFILE}" \ |
| 543 | --label "vllm.rocm_base.dependency_summary=${dependency_summary}" \ |
| 544 | --label "vllm.rocm_base.base_image=${pinned_base_image}" \ |
| 545 | --label "vllm.rocm_base.base_image_digest=${base_image_digest}" \ |
| 546 | --label "vllm.rocm_base.dependency.rocm=${rocm_version}" \ |
| 547 | --label "vllm.rocm_base.dependency.python=${python_version_arg}" \ |
| 548 | --label "vllm.rocm_base.dependency.pytorch=${pytorch_arg}" \ |
| 549 | --label "vllm.rocm_base.dependency.torchvision=${pytorch_vision_arg}" \ |
| 550 | --label "vllm.rocm_base.dependency.torchaudio=${pytorch_audio_arg}" \ |
| 551 | --label "vllm.rocm_base.dependency.triton=${triton_arg}" \ |
| 552 | --label "vllm.rocm_base.dependency.flash_attention=${fa_arg}" \ |
| 553 | --label "vllm.rocm_base.dependency.aiter=${aiter_arg}" \ |
| 554 | --label "vllm.rocm_base.dependency.mori=${mori_arg}" \ |
| 555 | --label "vllm.rocm_base.pytorch_rocm_arch=${pytorch_rocm_arch_arg}" \ |
| 556 | -t "${build_ref}" \ |
| 557 | --push \ |
| 558 | . |
| 559 | if ! immutable_ref=$(find_matching_base_content_ref \ |
| 560 | "${base_hash}" "${metadata_version}" "${build_ref}"); then |
| 561 | echo "Published ROCm base image failed identity validation: ${build_ref}" >&2 |
| 562 | return 1 |
| 563 | fi |
| 564 | fi |
| 565 | |
| 566 | selected_digest="${immutable_ref##*@}" |
| 567 | metadata_set "rocm-base-image" "${immutable_ref}" |
| 568 | metadata_set "rocm-base-push-stable-tag" "0" |
| 569 | stable_digest=$(resolve_image_digest "${stable_tag}" 2>/dev/null || true) |
| 570 | if [[ -n "${stable_digest}" && "${stable_digest}" == "${selected_digest}" ]]; then |
| 571 | echo "--- :white_check_mark: Existing ROCm base is current" |
| 572 | echo "Stable ${stable_tag} already resolves to ${selected_digest}" |
| 573 | return 0 |
| 574 | fi |
| 575 | |
| 576 | promote_stable_base_tag "${immutable_ref}" "${stable_tag}" |
| 577 | |
| 578 | metadata_set "rocm-base-push-stable-tag" "${ROCM_BASE_STABLE_TAG_UPDATED}" |
| 579 | metadata_set "rocm-base-refresh" "1" |
| 580 | |
| 581 | echo "--- :white_check_mark: ROCm base image published" |
| 582 | echo "Use BASE_IMAGE=${immutable_ref} for downstream ROCm CI builds" |
| 583 | } |
| 584 | |
| 585 | main() { |
| 586 | if using_custom_rocm_dockerfiles; then |
| 587 | if [[ "${ROCM_BASE_REFRESH_SKIP:-0}" == "1" ]]; then |
| 588 | echo "Custom ROCm Dockerfiles require ROCM_BASE_REFRESH_SKIP=0" >&2 |
| 589 | return 2 |
| 590 | fi |
| 591 | validate_rocm_dockerfile "${DOCKERFILE}" || return $? |
| 592 | fi |
| 593 | metadata_set "rocm-base-refresh" "0" |
| 594 | |
| 595 | if [[ "${ROCM_BASE_REFRESH_SKIP:-0}" == "1" ]]; then |
| 596 | local stable_ref="${BASE_REPO}:base" |
| 597 | local stable_digest="" |
| 598 | |
| 599 | if ! stable_digest=$(resolve_image_digest "${stable_ref}"); then |
| 600 | echo "Could not pin explicitly skipped ROCm base: ${stable_ref}" >&2 |
| 601 | return 1 |
| 602 | fi |
| 603 | metadata_set "rocm-base-image" "${stable_ref}@${stable_digest}" |
| 604 | metadata_set "rocm-base-push-stable-tag" "0" |
| 605 | echo "ROCM_BASE_REFRESH_SKIP=1; pinned current stable base at ${stable_digest}" |
| 606 | return 0 |
| 607 | fi |
| 608 | build_base_image |
| 609 | } |
| 610 | |
| 611 | if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then |
| 612 | main "$@" |
| 613 | fi |