@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 299d2b5655

[CI] Annotate built Docker image tags on the Buildkite build page (#48101)

Signed-off-by: khluu <khluu000@gmail.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>

Kevin H. Luu committed Jul 9, 2026 at 07:02 UTC 299d2b565517044847f7300c11b9eb4c41931c36
8 files changed +124 -77
.buildkite/image_build/image_build.sh
+8
@@ -79,12 +79,18 @@ setup_buildx_builder() {
79 docker buildx ls | grep -E '^\*|^NAME' || docker buildx ls
80 }
81
82 +annotate_image_tags() {
83 + .buildkite/scripts/annotate-image-build.sh \
84 + "${IMAGE_TAG:-}" "${IMAGE_TAG_LATEST:-}"
85 +}
86 +
87 check_and_skip_if_image_exists() {
88 if [[ -n "${IMAGE_TAG:-}" ]]; then
89 echo "--- :mag: Checking if image exists"
90 if docker manifest inspect "${IMAGE_TAG}" >/dev/null 2>&1; then
91 echo "Image already exists: ${IMAGE_TAG}"
92 echo "Skipping build"
93 + annotate_image_tags
94 exit 0
95 fi
96 echo "Image not found, proceeding with build"
@@ -254,3 +260,5 @@ echo "--- :docker: Building ${TARGET}"
260 docker --debug buildx bake -f "${VLLM_BAKE_FILE_PATH}" -f "${CI_HCL_PATH}" --progress plain "${TARGET}"
261
262 echo "--- :white_check_mark: Build complete"
263 +
264 +annotate_image_tags
.buildkite/image_build/image_build_arm64.sh
+19 -19
@@ -9,30 +9,30 @@ fi
9 REGISTRY=$1
10 REPO=$2
11 BUILDKITE_COMMIT=$3
12 +IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-arm64"
13
14 # authenticate with AWS ECR
15 aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
16
17 # skip build if image already exists
17 -if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64) ]]; then
18 - echo "Image not found, proceeding with build..."
19 -else
18 +if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then
19 echo "Image found"
21 - exit 0
20 +else
21 + echo "Image not found, proceeding with build..."
22 + # build for arm64 GPU targets: Grace/GH200 (sm_90) and DGX Spark/GB10
23 + # (sm_121, family-covered by 12.0 under CUDA 13)
24 + docker build --file docker/Dockerfile \
25 + --platform linux/arm64 \
26 + --build-arg max_jobs=16 \
27 + --build-arg nvcc_threads=4 \
28 + --build-arg torch_cuda_arch_list="9.0 12.0" \
29 + --build-arg USE_SCCACHE=1 \
30 + --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
31 + --tag "$IMAGE" \
32 + --target test \
33 + --progress plain .
34 + # push
35 + docker push "$IMAGE"
36 fi
37
24 -# build for arm64 GPU targets: Grace/GH200 (sm_90) and DGX Spark/GB10
25 -# (sm_121, family-covered by 12.0 under CUDA 13)
26 -docker build --file docker/Dockerfile \
27 - --platform linux/arm64 \
28 - --build-arg max_jobs=16 \
29 - --build-arg nvcc_threads=4 \
30 - --build-arg torch_cuda_arch_list="9.0 12.0" \
31 - --build-arg USE_SCCACHE=1 \
32 - --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
33 - --tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64 \
34 - --target test \
35 - --progress plain .
36 -
37 -# push
38 -docker push "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64
38 +.buildkite/scripts/annotate-image-build.sh "$IMAGE"
.buildkite/image_build/image_build_cpu.sh
+15 -15
@@ -9,26 +9,26 @@ fi
9 REGISTRY=$1
10 REPO=$2
11 BUILDKITE_COMMIT=$3
12 +IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-cpu"
13
14 # authenticate with AWS ECR
15 aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
16
17 # skip build if image already exists
17 -if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-cpu) ]]; then
18 - echo "Image not found, proceeding with build..."
19 -else
18 +if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then
19 echo "Image found"
21 - exit 0
20 +else
21 + echo "Image not found, proceeding with build..."
22 + # build
23 + docker build --file docker/Dockerfile.cpu \
24 + --build-arg max_jobs=16 \
25 + --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
26 + --build-arg VLLM_CPU_X86=true \
27 + --tag "$IMAGE" \
28 + --target vllm-test \
29 + --progress plain .
30 + # push
31 + docker push "$IMAGE"
32 fi
33
24 -# build
25 -docker build --file docker/Dockerfile.cpu \
26 - --build-arg max_jobs=16 \
27 - --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
28 - --build-arg VLLM_CPU_X86=true \
29 - --tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-cpu \
30 - --target vllm-test \
31 - --progress plain .
32 -
33 -# push
34 -docker push "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-cpu
34 +.buildkite/scripts/annotate-image-build.sh "$IMAGE"
.buildkite/image_build/image_build_cpu_arm64.sh
+14 -14
@@ -9,25 +9,25 @@ fi
9 REGISTRY=$1
10 REPO=$2
11 BUILDKITE_COMMIT=$3
12 +IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-arm64-cpu"
13
14 # authenticate with AWS ECR
15 aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
16
17 # skip build if image already exists
17 -if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64-cpu) ]]; then
18 - echo "Image not found, proceeding with build..."
19 -else
18 +if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then
19 echo "Image found"
21 - exit 0
20 +else
21 + echo "Image not found, proceeding with build..."
22 + # build
23 + docker build --file docker/Dockerfile.cpu \
24 + --build-arg max_jobs=16 \
25 + --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
26 + --tag "$IMAGE" \
27 + --target vllm-test \
28 + --progress plain .
29 + # push
30 + docker push "$IMAGE"
31 fi
32
24 -# build
25 -docker build --file docker/Dockerfile.cpu \
26 - --build-arg max_jobs=16 \
27 - --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
28 - --tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64-cpu \
29 - --target vllm-test \
30 - --progress plain .
31 -
32 -# push
33 -docker push "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64-cpu
33 +.buildkite/scripts/annotate-image-build.sh "$IMAGE"
.buildkite/image_build/image_build_hpu.sh
+15 -15
@@ -9,26 +9,26 @@ fi
9 REGISTRY=$1
10 REPO=$2
11 BUILDKITE_COMMIT=$3
12 +IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-hpu"
13
14 # authenticate with AWS ECR
15 aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
16
17 # skip build if image already exists
17 -if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-hpu) ]]; then
18 - echo "Image not found, proceeding with build..."
19 -else
18 +if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then
19 echo "Image found"
21 - exit 0
20 +else
21 + echo "Image not found, proceeding with build..."
22 + # build
23 + docker build \
24 + --file tests/pytorch_ci_hud_benchmark/Dockerfile.hpu \
25 + --build-arg max_jobs=16 \
26 + --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
27 + --tag "$IMAGE" \
28 + --progress plain \
29 + https://github.com/vllm-project/vllm-gaudi.git
30 + # push
31 + docker push "$IMAGE"
32 fi
33
24 -# build
25 -docker build \
26 - --file tests/pytorch_ci_hud_benchmark/Dockerfile.hpu \
27 - --build-arg max_jobs=16 \
28 - --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
29 - --tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-hpu \
30 - --progress plain \
31 - https://github.com/vllm-project/vllm-gaudi.git
32 -
33 -# push
34 -docker push "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-hpu
34 +.buildkite/scripts/annotate-image-build.sh "$IMAGE"
.buildkite/image_build/image_build_torch_nightly.sh
+3
@@ -40,6 +40,7 @@ docker buildx ls
40 echo "--- :mag: Checking if image already exists"
41 if docker manifest inspect "$IMAGE_TAG" >/dev/null 2>&1; then
42 echo "Image found: $IMAGE_TAG — skipping build"
43 + .buildkite/scripts/annotate-image-build.sh "$IMAGE_TAG"
44 exit 0
45 fi
46 echo "Image not found, proceeding with build..."
@@ -66,3 +67,5 @@ docker buildx build --file docker/Dockerfile \
67 --progress plain .
68
69 echo "--- :white_check_mark: Torch nightly image build complete: $IMAGE_TAG"
70 +
71 +.buildkite/scripts/annotate-image-build.sh "$IMAGE_TAG"
.buildkite/image_build/image_build_xpu.sh
+14 -14
@@ -9,26 +9,26 @@ fi
9 REGISTRY=$1
10 REPO=$2
11 BUILDKITE_COMMIT=$3
12 +IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-xpu"
13
14 # authenticate with AWS ECR
15 aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
16 aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 936637512419.dkr.ecr.us-east-1.amazonaws.com || true
17
18 # skip build if image already exists
18 -if ! docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-xpu &> /dev/null; then
19 - echo "Image not found, proceeding with build..."
20 -else
19 +if docker manifest inspect "$IMAGE" &> /dev/null; then
20 echo "Image found"
22 - exit 0
21 +else
22 + echo "Image not found, proceeding with build..."
23 + # build
24 + docker build \
25 + --file docker/Dockerfile.xpu \
26 + --build-arg max_jobs=16 \
27 + --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
28 + --tag "$IMAGE" \
29 + --progress plain .
30 + # push
31 + docker push "$IMAGE"
32 fi
33
25 -# build
26 -docker build \
27 - --file docker/Dockerfile.xpu \
28 - --build-arg max_jobs=16 \
29 - --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
30 - --tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-xpu \
31 - --progress plain .
32 -
33 -# push
34 -docker push "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-xpu
34 +.buildkite/scripts/annotate-image-build.sh "$IMAGE"
.buildkite/scripts/annotate-image-build.sh new
+36
@@ -0,0 +1,36 @@
1 +#!/bin/bash
2 +# SPDX-License-Identifier: Apache-2.0
3 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
4 +#
5 +# Append the Docker image tag(s) an image-build step pushed to a Buildkite
6 +# annotation, so the built image tags show up on the build page instead of
7 +# being buried in the job logs.
8 +#
9 +# Usage: annotate-image-build.sh <image_tag> [<image_tag> ...]
10 +set -euo pipefail
11 +
12 +# buildkite-agent only exists on Buildkite agents; no-op elsewhere so the
13 +# image build scripts stay runnable locally.
14 +if ! command -v buildkite-agent >/dev/null 2>&1; then
15 + echo "buildkite-agent not found; skipping image tag annotation"
16 + exit 0
17 +fi
18 +
19 +label="${BUILDKITE_LABEL:-Image build}"
20 +content=""
21 +for image in "$@"; do
22 + [[ -n "$image" ]] || continue
23 + content+="- **${label}**: \`${image}\`"$'\n'
24 +done
25 +
26 +if [[ -z "$content" ]]; then
27 + echo "No image tags provided; nothing to annotate"
28 + exit 0
29 +fi
30 +
31 +# Best-effort: a flaky annotation must never fail an otherwise successful
32 +# (and expensive) image build.
33 +if ! printf '%s' "$content" | \
34 + buildkite-agent annotate --append --style 'info' --context 'docker-images'; then
35 + echo "warning: failed to annotate build with image tags"
36 +fi