@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 7dc30f5a66

[ROCm][CI] Build and publish TheRock nightly docker images (#55014)

Signed-off-by: Rohan Potdar <rohan.potdar@amd.com> Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: OpenAI Codex <noreply@openai.com>

Rohan Potdar committed Sep 3, 2026 at 21:34 UTC 7dc30f5a663d1d3fcadf73ffd297682b56a69197
2 files changed +219 -34
.buildkite/release-pipeline.yaml
+138
@@ -959,6 +959,144 @@ steps:
959 DOCKER_BUILDKIT: "1"
960 DOCKERHUB_USERNAME: "vllmbot"
961
962 + # =============================================================================
963 + # ROCk Release Pipeline (x86_64 only)
964 + # =============================================================================
965 + #
966 + # Same shape as the ROCm pipeline above (Jobs 1 and 6 only), but built from
967 + # docker/Dockerfile.rock_base and docker/Dockerfile.rock, which install the
968 + # ROCm SDK from TheRock wheels instead of the ROCm apt repos.
969 + #
970 + # Note: ROCm SDK version is determined by ROCM_SDK_VERSION in
971 + # docker/Dockerfile.rock_base
972 + #
973 + # =============================================================================
974 +
975 + - group: "Build ROCk Image"
976 + key: "build-rock-image"
977 + depends_on: ~
978 + steps:
979 + # ROCk Job 1: Build ROCk Base Image (with ECR caching)
980 + - label: ":rocm: Build ROCk Base Image"
981 + id: build-rock-base-image
982 + depends_on: ~
983 + agents:
984 + queue: cpu_queue_release
985 + timeout_in_minutes: 360
986 + commands:
987 + - |
988 + set -euo pipefail
989 +
990 + # The SDK version, torch wheels and component branches all live in
991 + # the Dockerfile, so its hash is the cache key.
992 + CACHE_KEY=$$(sha256sum docker/Dockerfile.rock_base | cut -c1-16)
993 + ECR_CACHE_TAG="public.ecr.aws/q9t5s3a7/vllm-release-repo:$${CACHE_KEY}-rock-base"
994 +
995 + echo "========================================"
996 + echo "ROCk Base Build Configuration"
997 + echo "========================================"
998 + echo " CACHE_KEY: $${CACHE_KEY}"
999 + echo " ECR_CACHE_TAG: $${ECR_CACHE_TAG}"
1000 + echo "========================================"
1001 +
1002 + # Login to ECR
1003 + aws ecr-public get-login-password --region us-east-1 | \
1004 + docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
1005 +
1006 + if docker manifest inspect "$${ECR_CACHE_TAG}" > /dev/null 2>&1; then
1007 + echo "ECR image cache HIT"
1008 + else
1009 + echo "CACHE MISS - Building from scratch..."
1010 +
1011 + DOCKER_BUILDKIT=1 docker buildx build \
1012 + --file docker/Dockerfile.rock_base \
1013 + --tag "$${ECR_CACHE_TAG}" \
1014 + --build-arg USE_SCCACHE=1 \
1015 + --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
1016 + --build-arg SCCACHE_REGION_NAME=us-west-2 \
1017 + --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
1018 + --push \
1019 + .
1020 + fi
1021 +
1022 + # Save ECR tag for downstream jobs
1023 + buildkite-agent meta-data set "rock-base-image-tag" "$${ECR_CACHE_TAG}"
1024 + env:
1025 + DOCKER_BUILDKIT: "1"
1026 +
1027 + # ROCk Job 2: Build ROCk Release Docker Image
1028 + - label: ":docker: Build release image - x86_64 - ROCk"
1029 + id: build-rock-release-image
1030 + depends_on:
1031 + - step: block-build-release-images
1032 + allow_failure: true
1033 + - step: build-rock-base-image
1034 + allow_failure: false
1035 + agents:
1036 + queue: cpu_queue_release
1037 + timeout_in_minutes: 60
1038 + commands:
1039 + - |
1040 + set -euo pipefail
1041 +
1042 + # Login to ECR
1043 + aws ecr-public get-login-password --region us-east-1 | \
1044 + docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
1045 +
1046 + ECR_IMAGE_TAG="$$(buildkite-agent meta-data get rock-base-image-tag 2>/dev/null || echo '')"
1047 + if [ -z "$${ECR_IMAGE_TAG}" ]; then
1048 + echo "ERROR: rock-base-image-tag metadata not found"
1049 + echo "This should have been set by the build-rock-base-image job"
1050 + exit 1
1051 + fi
1052 +
1053 + docker pull "$${ECR_IMAGE_TAG}"
1054 +
1055 + # Pass the base image ECR tag to downstream steps (nightly publish)
1056 + buildkite-agent meta-data set "rock-base-ecr-tag" "$${ECR_IMAGE_TAG}"
1057 +
1058 + echo "========================================"
1059 + echo "Building vLLM ROCk release image with:"
1060 + echo " BASE_IMAGE: $${ECR_IMAGE_TAG}"
1061 + echo " BUILDKITE_COMMIT: $${BUILDKITE_COMMIT}"
1062 + echo "========================================"
1063 +
1064 + DOCKER_BUILDKIT=1 docker build \
1065 + --build-arg max_jobs=16 \
1066 + --build-arg BASE_IMAGE="$${ECR_IMAGE_TAG}" \
1067 + --build-arg USE_SCCACHE=1 \
1068 + --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
1069 + --build-arg SCCACHE_REGION_NAME=us-west-2 \
1070 + --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
1071 + --build-arg INSTALL_LMCACHE=true \
1072 + --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rock \
1073 + --target vllm-openai \
1074 + --progress plain \
1075 + -f docker/Dockerfile.rock .
1076 +
1077 + docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rock
1078 + env:
1079 + DOCKER_BUILDKIT: "1"
1080 +
1081 + - label: "Publish nightly ROCk image to DockerHub"
1082 + depends_on:
1083 + - build-rock-release-image
1084 + if: build.env("NIGHTLY") == "1"
1085 + agents:
1086 + queue: small_cpu_queue_release
1087 + commands:
1088 + - "bash .buildkite/scripts/push-nightly-builds-rocm.sh rock rocm714"
1089 + # Clean up old nightly builds (keep only last 14)
1090 + - "bash .buildkite/scripts/cleanup-nightly-builds.sh nightly-rocm714- vllm/vllm-openai-rocm"
1091 + - "bash .buildkite/scripts/cleanup-nightly-builds.sh base-nightly-rocm714- vllm/vllm-openai-rocm"
1092 + plugins:
1093 + - docker-login#v3.0.0:
1094 + username: vllmbot
1095 + password-env: DOCKERHUB_TOKEN
1096 + env:
1097 + DOCKER_BUILDKIT: "1"
1098 + DOCKERHUB_USERNAME: "vllmbot"
1099 +
1100 # =============================================================================
1101 # Publish to DockerHub and PyPI (at the end so all builds complete first)
1102 # =============================================================================
.buildkite/scripts/push-nightly-builds-rocm.sh
+81 -34
@@ -2,61 +2,108 @@
2 # SPDX-License-Identifier: Apache-2.0
3 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
4 #
5 -# Push ROCm nightly base image and nightly image from ECR
6 -# to Docker Hub as vllm/vllm-openai-rocm:base-nightly and vllm/vllm-openai-rocm:nightly
7 -# and vllm/vllm-openai-rocm:base-nightly-<commit> and vllm/vllm-openai-rocm:nightly-<commit>.
8 -# Run when NIGHTLY=1 after build-rocm-release-image has pushed to ECR.
5 +# Push a ROCm-family nightly base image and nightly image from ECR to Docker Hub
6 +# under vllm/vllm-openai-rocm, as base-nightly[-<variant>], nightly[-<variant>]
7 +# and their -<commit> forms.
8 +# Run when NIGHTLY=1 after the matching build-*-release-image step has pushed to ECR.
9 +#
10 +# Usage: push-nightly-builds-rocm.sh [ECR_TAG_SUFFIX TAG_VARIANT]
11 +# Pass both arguments or neither. With none, the default ROCm apt build is
12 +# published as :nightly. The TheRock build passes "rock" (the source ECR tag
13 +# suffix) and "rocm714" (the Docker Hub tag flavor), giving :nightly-rocm714.
14 #
15 # Local testing (no push to Docker Hub):
11 -# BUILDKITE_COMMIT=<commit-with-rocm-image-in-ecr> DRY_RUN=1 bash .buildkite/scripts/push-nightly-builds-rocm.sh
16 +# BASE_ECR_IMAGE=<full-base-image-reference> \
17 +# BUILDKITE_COMMIT=<commit-with-image-in-ecr> DRY_RUN=1 \
18 +# bash .buildkite/scripts/push-nightly-builds-rocm.sh
19 # Requires: AWS CLI configured (for ECR public login), Docker. For full run: Docker Hub login.
20
14 -set -ex
21 +set -euxo pipefail
22 +
23 +usage() {
24 + echo "Usage: $0 [ECR_TAG_SUFFIX TAG_VARIANT]" >&2
25 +}
26 +
27 +DOCKERHUB_REPO="vllm/vllm-openai-rocm"
28
16 -# Use BUILDKITE_COMMIT from env (required; set to a commit that has ROCm image in ECR for local test)
17 -BUILDKITE_COMMIT="${BUILDKITE_COMMIT:?Set BUILDKITE_COMMIT to the commit SHA that has the ROCm image in ECR (e.g. from a previous release pipeline run)}"
29 +case "$#" in
30 + 0)
31 + ECR_TAG_SUFFIX="rocm"
32 + TAG_VARIANT=""
33 + ;;
34 + 2)
35 + if [[ -z "$1" || -z "$2" ]]; then
36 + usage
37 + exit 2
38 + fi
39 + ECR_TAG_SUFFIX="$1"
40 + TAG_VARIANT="$2"
41 + ;;
42 + *)
43 + usage
44 + exit 2
45 + ;;
46 +esac
47 +
48 +# Use BUILDKITE_COMMIT from env (required; set to a commit that has the image in ECR for local test)
49 +BUILDKITE_COMMIT="${BUILDKITE_COMMIT:?Set BUILDKITE_COMMIT to the commit SHA that has the image in ECR (e.g. from a previous release pipeline run)}"
50 DRY_RUN="${DRY_RUN:-0}"
51
20 -# Get the base image ECR tag (set by build-rocm-release-image pipeline step)
21 -BASE_ORIG_TAG="$(buildkite-agent meta-data get rocm-base-ecr-tag 2>/dev/null || echo "")"
22 -if [ -z "$BASE_ORIG_TAG" ]; then
23 - echo "WARNING: rocm-base-ecr-tag metadata not found, falling back to commit-based tag"
24 - BASE_ORIG_TAG="public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm-base"
52 +ECR_REPO="public.ecr.aws/q9t5s3a7/vllm-release-repo"
53 +
54 +# Get the base image ECR tag (set by the build-*-release-image pipeline step)
55 +BASE_METADATA_KEY="${ECR_TAG_SUFFIX}-base-ecr-tag"
56 +if [[ -n "${BASE_ECR_IMAGE:-}" ]]; then
57 + if [[ "$DRY_RUN" != "1" ]]; then
58 + echo "ERROR: BASE_ECR_IMAGE may only be used with DRY_RUN=1" >&2
59 + exit 1
60 + fi
61 + BASE_ORIG_TAG="$BASE_ECR_IMAGE"
62 +elif ! BASE_ORIG_TAG="$(buildkite-agent meta-data get "$BASE_METADATA_KEY")"; then
63 + echo "ERROR: Failed to read required Buildkite metadata '$BASE_METADATA_KEY'" >&2
64 + exit 1
65 +fi
66 +if [[ -z "$BASE_ORIG_TAG" ]]; then
67 + echo "ERROR: Required Buildkite metadata '$BASE_METADATA_KEY' is empty" >&2
68 + echo "Set BASE_ECR_IMAGE to the full ECR base image reference for local testing" >&2
69 + exit 1
70 fi
71
27 -ORIG_TAG="${BUILDKITE_COMMIT}-rocm"
28 -BASE_TAG_NAME="base-nightly"
29 -TAG_NAME="nightly"
30 -BASE_TAG_NAME_COMMIT="base-nightly-${BUILDKITE_COMMIT}"
31 -TAG_NAME_COMMIT="nightly-${BUILDKITE_COMMIT}"
72 +ORIG_TAG="${BUILDKITE_COMMIT}-${ECR_TAG_SUFFIX}"
73 +VARIANT_SUFFIX=""
74 +[[ -n "$TAG_VARIANT" ]] && VARIANT_SUFFIX="-$TAG_VARIANT"
75 +BASE_TAG_NAME="base-nightly${VARIANT_SUFFIX}"
76 +TAG_NAME="nightly${VARIANT_SUFFIX}"
77 +BASE_TAG_NAME_COMMIT="base-nightly${VARIANT_SUFFIX}-${BUILDKITE_COMMIT}"
78 +TAG_NAME_COMMIT="nightly${VARIANT_SUFFIX}-${BUILDKITE_COMMIT}"
79
33 -echo "Pushing ROCm base image from ECR: $BASE_ORIG_TAG"
34 -echo "Pushing ROCm release image from ECR tag: $ORIG_TAG to Docker Hub as $TAG_NAME and $TAG_NAME_COMMIT"
80 +echo "Pushing base image from ECR: $BASE_ORIG_TAG"
81 +echo "Pushing release image from ECR tag: $ORIG_TAG to Docker Hub as $DOCKERHUB_REPO:$TAG_NAME and $DOCKERHUB_REPO:$TAG_NAME_COMMIT"
82 [[ "$DRY_RUN" == "1" ]] && echo "[DRY_RUN] Skipping push to Docker Hub"
83
37 -# Login to ECR and pull the image built by build-rocm-release-image
84 +# Login to ECR and pull the images built by build-*-release-image
85 aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
86 docker pull "$BASE_ORIG_TAG"
40 -docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:"$ORIG_TAG"
87 +docker pull "${ECR_REPO}:${ORIG_TAG}"
88
89 # Tag for Docker Hub (base-nightly and base-nightly-<commit>, nightly and nightly-<commit>)
43 -docker tag "$BASE_ORIG_TAG" vllm/vllm-openai-rocm:"$BASE_TAG_NAME"
44 -docker tag "$BASE_ORIG_TAG" vllm/vllm-openai-rocm:"$BASE_TAG_NAME_COMMIT"
45 -docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:"$ORIG_TAG" vllm/vllm-openai-rocm:"$TAG_NAME"
46 -docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:"$ORIG_TAG" vllm/vllm-openai-rocm:"$TAG_NAME_COMMIT"
90 +docker tag "$BASE_ORIG_TAG" "$DOCKERHUB_REPO":"$BASE_TAG_NAME"
91 +docker tag "$BASE_ORIG_TAG" "$DOCKERHUB_REPO":"$BASE_TAG_NAME_COMMIT"
92 +docker tag "${ECR_REPO}:${ORIG_TAG}" "$DOCKERHUB_REPO":"$TAG_NAME"
93 +docker tag "${ECR_REPO}:${ORIG_TAG}" "$DOCKERHUB_REPO":"$TAG_NAME_COMMIT"
94
95 if [[ "$DRY_RUN" == "1" ]]; then
49 - echo "[DRY_RUN] Would push vllm/vllm-openai-rocm:$BASE_TAG_NAME and vllm/vllm-openai-rocm:$BASE_TAG_NAME_COMMIT"
50 - echo "[DRY_RUN] Would push vllm/vllm-openai-rocm:$TAG_NAME and vllm/vllm-openai-rocm:$TAG_NAME_COMMIT"
96 + echo "[DRY_RUN] Would push $DOCKERHUB_REPO:$BASE_TAG_NAME and $DOCKERHUB_REPO:$BASE_TAG_NAME_COMMIT"
97 + echo "[DRY_RUN] Would push $DOCKERHUB_REPO:$TAG_NAME and $DOCKERHUB_REPO:$TAG_NAME_COMMIT"
98 echo "[DRY_RUN] Local tags created. Exiting without push."
99 exit 0
100 fi
101
102 # Push to Docker Hub (docker-login plugin runs before this step in CI)
56 -docker push vllm/vllm-openai-rocm:"$BASE_TAG_NAME"
57 -docker push vllm/vllm-openai-rocm:"$BASE_TAG_NAME_COMMIT"
58 -docker push vllm/vllm-openai-rocm:"$TAG_NAME"
59 -docker push vllm/vllm-openai-rocm:"$TAG_NAME_COMMIT"
103 +docker push "$DOCKERHUB_REPO":"$BASE_TAG_NAME"
104 +docker push "$DOCKERHUB_REPO":"$BASE_TAG_NAME_COMMIT"
105 +docker push "$DOCKERHUB_REPO":"$TAG_NAME"
106 +docker push "$DOCKERHUB_REPO":"$TAG_NAME_COMMIT"
107
61 -echo "Pushed vllm/vllm-openai-rocm:$BASE_TAG_NAME and vllm/vllm-openai-rocm:$BASE_TAG_NAME_COMMIT"
62 -echo "Pushed vllm/vllm-openai-rocm:$TAG_NAME and vllm/vllm-openai-rocm:$TAG_NAME_COMMIT"
108 +echo "Pushed $DOCKERHUB_REPO:$BASE_TAG_NAME and $DOCKERHUB_REPO:$BASE_TAG_NAME_COMMIT"
109 +echo "Pushed $DOCKERHUB_REPO:$TAG_NAME and $DOCKERHUB_REPO:$TAG_NAME_COMMIT"