[CI] Gate non-default release wheel builds (#48772)
Signed-off-by: khluu <khluu000@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
Kevin H. Luu committed
Jul 17, 2026 at 02:16 UTC
867ff69733dd32467a18fed288592373225d8db1
4 files changed
+69
-30
.buildkite/release-pipeline.yaml
+56
-23
@@ -31,16 +31,16 @@ steps:
31
- text: "What is the release version?"
32
key: release-version
33
34
- - group: "Build Python wheels"
34
+ - group: "Build CUDA 13.0 Python wheels"
35
key: "build-wheels"
36
steps:
37
- - label: "Build wheel - aarch64 - CUDA 12.9"
37
+ - label: "Build wheel - aarch64 - CUDA 13.0"
38
depends_on: ~
39
- id: build-wheel-arm64-cuda-12-9
39
+ id: build-wheel-arm64-cuda-13-0
40
agents:
41
queue: arm64_cpu_queue_release
42
commands:
43
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64_CU129}\" --build-arg BUILD_OS=manylinux --build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda12.9 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
43
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64}\" --build-arg BUILD_OS=manylinux --build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda13.0 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
44
- "mkdir artifacts"
45
- "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
46
- "bash .buildkite/scripts/upload-nightly-wheels.sh"
@@ -48,13 +48,37 @@ steps:
48
env:
49
DOCKER_BUILDKIT: "1"
50
51
- - label: "Build wheel - aarch64 - CUDA 13.0"
51
+ - label: "Build wheel - x86_64 - CUDA 13.0"
52
depends_on: ~
53
- id: build-wheel-arm64-cuda-13-0
53
+ id: build-wheel-x86-cuda-13-0
54
+ agents:
55
+ queue: cpu_queue_release
56
+ commands:
57
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86}\" --build-arg BUILD_OS=manylinux --build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda13.0 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
58
+ - "mkdir artifacts"
59
+ - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
60
+ - "bash .buildkite/scripts/upload-nightly-wheels.sh"
61
+ - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)" release-wheels'
62
+ env:
63
+ DOCKER_BUILDKIT: "1"
64
+
65
+ - block: "Unblock to build additional Python wheels"
66
+ depends_on: ~
67
+ key: block-build-additional-wheels
68
+ if: build.env("NIGHTLY") != "1"
69
+
70
+ - group: "Build additional Python wheels"
71
+ key: "build-additional-wheels"
72
+ depends_on: block-build-additional-wheels
73
+ allow_dependency_failure: true
74
+ steps:
75
+ - label: "Build wheel - aarch64 - CUDA 12.9"
76
+ depends_on: ~
77
+ id: build-wheel-arm64-cuda-12-9
78
agents:
79
queue: arm64_cpu_queue_release
80
commands:
57
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64}\" --build-arg BUILD_OS=manylinux --build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda13.0 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
81
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64_CU129}\" --build-arg BUILD_OS=manylinux --build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda12.9 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
82
- "mkdir artifacts"
83
- "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
84
- "bash .buildkite/scripts/upload-nightly-wheels.sh"
@@ -133,20 +157,6 @@ steps:
157
env:
158
DOCKER_BUILDKIT: "1"
159
136
- - label: "Build wheel - x86_64 - CUDA 13.0"
137
- depends_on: ~
138
- id: build-wheel-x86-cuda-13-0
139
- agents:
140
- queue: cpu_queue_release
141
- commands:
142
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86}\" --build-arg BUILD_OS=manylinux --build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda13.0 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
143
- - "mkdir artifacts"
144
- - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
145
- - "bash .buildkite/scripts/upload-nightly-wheels.sh"
146
- - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)" release-wheels'
147
- env:
148
- DOCKER_BUILDKIT: "1"
149
-
160
- label: "Build wheel - x86_64 - CPU"
161
depends_on: ~
162
id: build-wheel-x86-cpu
@@ -162,12 +172,26 @@ steps:
172
DOCKER_BUILDKIT: "1"
173
174
- label: "Generate and upload wheel indices"
175
+ key: generate-wheel-indices
176
depends_on: "build-wheels"
177
allow_dependency_failure: true
178
+ if: build.env("NIGHTLY") != "1"
179
+ agents:
180
+ queue: cpu_queue_release
181
+ commands:
182
+ - "UPDATE_VERSION_INDEX=0 bash .buildkite/scripts/generate-and-upload-nightly-index.sh"
183
+
184
+ - label: "Regenerate indices with additional wheels"
185
+ key: generate-additional-wheel-indices
186
+ depends_on:
187
+ - build-wheels
188
+ - build-additional-wheels
189
+ - generate-wheel-indices
190
+ allow_dependency_failure: true
191
agents:
192
queue: cpu_queue_release
193
commands:
170
- - "bash .buildkite/scripts/generate-and-upload-nightly-index.sh"
194
+ - 'UPDATE_NIGHTLY_INDEX="$${NIGHTLY:-0}" bash .buildkite/scripts/generate-and-upload-nightly-index.sh'
195
196
- block: "Unblock to build release Docker images"
197
depends_on: ~
@@ -566,10 +590,17 @@ steps:
590
#
591
# =============================================================================
592
593
+ - block: "Unblock ROCm wheel/image prerequisites"
594
+ depends_on: ~
595
+ key: block-build-rocm
596
+ if: build.env("NIGHTLY") != "1"
597
+
598
# ROCm Job 1: Build ROCm Base Wheels (with S3 caching)
599
- label: ":rocm: Build ROCm Base Image & Wheels"
600
id: build-rocm-base-wheels
572
- depends_on: ~
601
+ depends_on:
602
+ - step: block-build-rocm
603
+ allow_failure: true
604
agents:
605
queue: cpu_queue_release
606
commands:
@@ -974,6 +1005,8 @@ steps:
1005
depends_on:
1006
- input-release-version
1007
- build-wheels
1008
+ - build-additional-wheels
1009
+ - generate-additional-wheel-indices
1010
1011
- label: "Upload release wheels to PyPI"
1012
depends_on:
.buildkite/scripts/generate-and-upload-nightly-index.sh
+5
-3
@@ -45,8 +45,10 @@ $PYTHON .buildkite/scripts/generate-nightly-index.py --version "$SUBPATH" --curr
45
echo "Uploading indices to $S3_COMMIT_PREFIX"
46
aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "$S3_COMMIT_PREFIX"
47
48
-# copy to /nightly/ only if it is on the main branch and not a PR
49
-if [[ "$BUILDKITE_BRANCH" == "main" && "$BUILDKITE_PULL_REQUEST" == "false" ]]; then
48
+# copy to /nightly/ only when enabled for a main branch build that is not a PR
49
+if [[ "${UPDATE_NIGHTLY_INDEX:-1}" == "1" && \
50
+ "$BUILDKITE_BRANCH" == "main" && \
51
+ "$BUILDKITE_PULL_REQUEST" == "false" ]]; then
52
echo "Uploading indices to overwrite /nightly/"
53
aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/nightly/"
54
fi
@@ -67,7 +69,7 @@ pure_version="${version%%+*}"
69
echo "Pure version (without variant): $pure_version"
70
71
# re-generate and copy to /<pure_version>/ only if it does not have "dev" in the version
70
-if [[ "$version" != *"dev"* ]]; then
72
+if [[ "${UPDATE_VERSION_INDEX:-1}" == "1" && "$version" != *"dev"* ]]; then
73
echo "Re-generating indices for /$pure_version/"
74
rm -rf "${INDICES_OUTPUT_DIR:?}"
75
mkdir -p "$INDICES_OUTPUT_DIR"
.buildkite/scripts/upload-rocm-wheels.sh
+3
-3
@@ -113,8 +113,8 @@ $PYTHON .buildkite/scripts/generate-nightly-index.py \
113
echo "Uploading indices to $S3_COMMIT_PREFIX"
114
aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "$S3_COMMIT_PREFIX"
115
116
-# Update rocm/nightly/ if on main branch and not a PR
117
-if [[ "$BUILDKITE_BRANCH" == "main" && "$BUILDKITE_PULL_REQUEST" == "false" ]] || [[ "$NIGHTLY" == "1" ]]; then
116
+# Only scheduled nightly builds should update the moving nightly index.
117
+if [[ "${NIGHTLY:-0}" == "1" ]]; then
118
echo "Updating rocm/nightly/ index..."
119
aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/rocm/nightly/"
120
fi
@@ -147,7 +147,7 @@ echo ""
147
echo "Install command (by commit):"
148
echo " pip install vllm --extra-index-url https://${BUCKET}.s3.amazonaws.com/$ROCM_SUBPATH/"
149
echo ""
150
-if [[ "$BUILDKITE_BRANCH" == "main" ]] || [[ "$NIGHTLY" == "1" ]]; then
150
+if [[ "${NIGHTLY:-0}" == "1" ]]; then
151
echo "Install command (nightly):"
152
echo " pip install vllm --extra-index-url https://${BUCKET}.s3.amazonaws.com/rocm/nightly/"
153
fi
docs/contributing/ci/nightly_builds.md
+5
-1
@@ -6,7 +6,11 @@ vLLM maintains a per-commit wheel repository (commonly referred to as "nightly")
6
7
### Wheel Building
8
9
-Wheels are built in the `Release` pipeline (`.buildkite/release-pipeline.yaml`) after a PR is merged into the main branch, with multiple variants:
9
+Wheels are built in the `Release` pipeline
10
+(`.buildkite/release-pipeline.yaml`) after a PR is merged into the main branch.
11
+Regular builds produce the CUDA 13.0 wheels for x86_64 and aarch64. Additional
12
+wheel variants and ROCm builds can be unblocked on demand and run automatically
13
+when `NIGHTLY=1`:
14
15
- **Backend variants**: `cpu` and `cuXXX` (e.g., `cu129`, `cu130`).
16
- **Architecture variants**: `x86_64` and `aarch64`.