A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 01:30 UTC
[CI] Implement uploading to PyPI and GitHub in the release pipeline, enable release image building for CUDA 13.0 (#31032)
Shengqi Chen committed
Jan 17, 2026 at 12:52 UTC
8e61425ee6d0bd03d3669c148eba8b263d101273
3 files changed
+184
-23
.buildkite/release-pipeline.yaml
+81
-23
@@ -1,6 +1,6 @@
1
steps:
2
# aarch64 + CUDA builds
3
- - label: "Build arm64 wheel - CUDA 12.9"
3
+ - label: "Build wheel - aarch64 - CUDA 12.9"
4
depends_on: ~
5
id: build-wheel-arm64-cuda-12-9
6
agents:
@@ -11,11 +11,11 @@ steps:
11
- "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0' --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
12
- "mkdir artifacts"
13
- "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
14
- - "bash .buildkite/scripts/upload-wheels.sh"
14
+ - "bash .buildkite/scripts/upload-nightly-wheels.sh"
15
env:
16
DOCKER_BUILDKIT: "1"
17
18
- - label: "Build arm64 wheel - CUDA 13.0"
18
+ - label: "Build wheel - aarch64 - CUDA 13.0"
19
depends_on: ~
20
id: build-wheel-arm64-cuda-13-0
21
agents:
@@ -26,12 +26,12 @@ steps:
26
- "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0' --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu22.04 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
27
- "mkdir artifacts"
28
- "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
29
- - "bash .buildkite/scripts/upload-wheels.sh manylinux_2_35"
29
+ - "bash .buildkite/scripts/upload-nightly-wheels.sh manylinux_2_35"
30
env:
31
DOCKER_BUILDKIT: "1"
32
33
# aarch64 build
34
- - label: "Build arm64 CPU wheel"
34
+ - label: "Build wheel - aarch64 - CPU"
35
depends_on: ~
36
id: build-wheel-arm64-cpu
37
agents:
@@ -40,39 +40,39 @@ steps:
40
- "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --build-arg VLLM_BUILD_ACL=ON --tag vllm-ci:build-image --target vllm-build --progress plain -f docker/Dockerfile.cpu ."
41
- "mkdir artifacts"
42
- "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
43
- - "bash .buildkite/scripts/upload-wheels.sh manylinux_2_35"
43
+ - "bash .buildkite/scripts/upload-nightly-wheels.sh manylinux_2_35"
44
env:
45
DOCKER_BUILDKIT: "1"
46
47
# x86 + CUDA builds
48
- - label: "Build wheel - CUDA 12.9"
48
+ - label: "Build wheel - x86_64 - CUDA 12.9"
49
depends_on: ~
50
- id: build-wheel-cuda-12-9
50
+ id: build-wheel-x86-cuda-12-9
51
agents:
52
queue: cpu_queue_postmerge
53
commands:
54
- "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
55
- "mkdir artifacts"
56
- "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
57
- - "bash .buildkite/scripts/upload-wheels.sh manylinux_2_31"
57
+ - "bash .buildkite/scripts/upload-nightly-wheels.sh manylinux_2_31"
58
env:
59
DOCKER_BUILDKIT: "1"
60
61
- - label: "Build wheel - CUDA 13.0"
61
+ - label: "Build wheel - x86_64 - CUDA 13.0"
62
depends_on: ~
63
- id: build-wheel-cuda-13-0
63
+ id: build-wheel-x86-cuda-13-0
64
agents:
65
queue: cpu_queue_postmerge
66
commands:
67
- "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu22.04 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
68
- "mkdir artifacts"
69
- "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
70
- - "bash .buildkite/scripts/upload-wheels.sh manylinux_2_35"
70
+ - "bash .buildkite/scripts/upload-nightly-wheels.sh manylinux_2_35"
71
env:
72
DOCKER_BUILDKIT: "1"
73
74
# x86 CPU wheel build
75
- - label: "Build x86 CPU wheel"
75
+ - label: "Build wheel - x86_64 - CPU"
76
depends_on: ~
77
id: build-wheel-x86-cpu
78
agents:
@@ -81,12 +81,12 @@ steps:
81
- "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --build-arg VLLM_CPU_AVX512BF16=true --build-arg VLLM_CPU_AVX512VNNI=true --build-arg VLLM_CPU_AMXBF16=true --tag vllm-ci:build-image --target vllm-build --progress plain -f docker/Dockerfile.cpu ."
82
- "mkdir artifacts"
83
- "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
84
- - "bash .buildkite/scripts/upload-wheels.sh manylinux_2_35"
84
+ - "bash .buildkite/scripts/upload-nightly-wheels.sh manylinux_2_35"
85
env:
86
DOCKER_BUILDKIT: "1"
87
88
- # Build release images (12.9)
89
- - label: "Build release image (x86)"
88
+ # Build release images (CUDA 12.9)
89
+ - label: "Build release image - x86_64 - CUDA 12.9"
90
depends_on: ~
91
id: build-release-image-x86
92
agents:
@@ -99,7 +99,7 @@ steps:
99
- "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT"
100
- "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT"
101
102
- - label: "Build release image (arm64)"
102
+ - label: "Build release image - aarch64 - CUDA 12.9"
103
depends_on: ~
104
id: build-release-image-arm64
105
agents:
@@ -109,34 +109,92 @@ steps:
109
- "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg FLASHINFER_AOT_COMPILE=true --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0' --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) --target vllm-openai --progress plain -f docker/Dockerfile ."
110
- "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)"
111
112
- # Add job to create multi-arch manifest
113
- - label: "Create multi-arch manifest"
112
+ - label: "Create multi-arch manifest - CUDA 12.9"
113
depends_on:
114
- build-release-image-x86
115
- build-release-image-arm64
116
id: create-multi-arch-manifest
117
agents:
119
- queue: cpu_queue_postmerge
118
+ queue: small_cpu_queue_postmerge
119
commands:
120
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
121
- "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64 --amend"
122
- "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT"
123
125
- - label: "Annotate release workflow"
124
+ - label: "Annotate release workflow - CUDA 12.9"
125
depends_on:
126
- create-multi-arch-manifest
127
id: annotate-release-workflow
128
agents:
130
- queue: cpu_queue_postmerge
129
+ queue: small_cpu_queue_postmerge
130
commands:
131
- "bash .buildkite/scripts/annotate-release.sh"
132
133
+ - block: "Build CUDA 13.0 release images"
134
+ key: block-release-image-build-cuda-13-0
135
+ depends_on: ~
136
+
137
+ - label: "Build release image - x86_64 - CUDA 13.0"
138
+ depends_on: block-release-image-build-cuda-13-0
139
+ id: build-release-image-x86-cuda-13-0
140
+ agents:
141
+ queue: cpu_queue_postmerge
142
+ commands:
143
+ - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
144
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg FLASHINFER_AOT_COMPILE=true --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu22.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130 --target vllm-openai --progress plain -f docker/Dockerfile ."
145
+ - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130"
146
+ # re-tag to default image tag and push, just in case arm64 build fails
147
+ - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130"
148
+ - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130"
149
+
150
+ - label: "Build release image - aarch64 - CUDA 13.0"
151
+ depends_on: block-release-image-build-cuda-13-0
152
+ id: build-release-image-arm64-cuda-13-0
153
+ agents:
154
+ queue: arm64_cpu_queue_postmerge
155
+ commands:
156
+ - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
157
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg FLASHINFER_AOT_COMPILE=true --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0' --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu22.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130 --target vllm-openai --progress plain -f docker/Dockerfile ."
158
+ - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130"
159
+
160
+ - label: "Create multi-arch manifest - CUDA 13.0"
161
+ depends_on:
162
+ - build-release-image-x86-cuda-13-0
163
+ - build-release-image-arm64-cuda-13-0
164
+ id: create-multi-arch-manifest-cuda-13-0
165
+ agents:
166
+ queue: small_cpu_queue_postmerge
167
+ commands:
168
+ - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
169
+ - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-cu130 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-cu130 --amend"
170
+ - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130"
171
+
172
- input: "Provide Release version here"
173
id: input-release-version
174
fields:
175
- text: "What is the release version?"
176
key: release-version
177
178
+ - block: "Confirm update release wheels to PyPI (experimental, use with caution)?"
179
+ key: block-upload-release-wheels
180
+ depends_on:
181
+ - input-release-version
182
+ - build-wheel-x86-cuda-12-9
183
+ - build-wheel-x86-cuda-13-0
184
+ - build-wheel-x86-cpu
185
+ - build-wheel-arm64-cuda-12-9
186
+ - build-wheel-arm64-cuda-13-0
187
+ - build-wheel-arm64-cpu
188
+
189
+ - label: "Upload release wheels to PyPI and GitHub"
190
+ depends_on:
191
+ - block-upload-release-wheels
192
+ id: upload-release-wheels
193
+ agents:
194
+ queue: small_cpu_queue_postmerge
195
+ commands:
196
+ - "bash .buildkite/scripts/upload-release-wheels.sh"
197
+
198
- block: "Build CPU release image"
199
key: block-cpu-release-image-build
200
depends_on: ~
@@ -192,7 +250,7 @@ steps:
250
- create-multi-arch-manifest
251
if: build.env("NIGHTLY") == "1"
252
agents:
195
- queue: cpu_queue_postmerge
253
+ queue: small_cpu_queue_postmerge
254
commands:
255
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
256
- "docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64"
.buildkite/scripts/upload-nightly-wheels.sh
renamed
.buildkite/scripts/upload-release-wheels.sh
new
+103
@@ -0,0 +1,103 @@
1
+#!/usr/bin/env bash
2
+
3
+set -e
4
+
5
+BUCKET="vllm-wheels"
6
+SUBPATH=$BUILDKITE_COMMIT
7
+S3_COMMIT_PREFIX="s3://$BUCKET/$SUBPATH/"
8
+
9
+RELEASE_VERSION=$(buildkite-agent meta-data get release-version)
10
+echo "Release version from Buildkite: $RELEASE_VERSION"
11
+GIT_VERSION=$(git describe --exact-match --tags $BUILDKITE_COMMIT 2>/dev/null)
12
+if [ -z "$GIT_VERSION" ]; then
13
+ echo "[FATAL] Not on a git tag, cannot create release."
14
+ exit 1
15
+else
16
+ echo "Git version for commit $BUILDKITE_COMMIT: $GIT_VERSION"
17
+fi
18
+# sanity check for version mismatch
19
+if [ "v$RELEASE_VERSION" != "$GIT_VERSION" ]; then
20
+ if [ "$FORCE_RELEASE_IGNORE_VERSION_MISMATCH" == "true" ]; then
21
+ echo "[WARNING] Force release and ignore version mismatch"
22
+ else
23
+ echo "[FATAL] Release version from Buildkite does not match Git version."
24
+ exit 1
25
+ fi
26
+fi
27
+
28
+# check pypi token
29
+if [ -z "$PYPI_TOKEN" ]; then
30
+ echo "[FATAL] PYPI_TOKEN is not set."
31
+ exit 1
32
+else
33
+ export TWINE_USERNAME="__token__"
34
+ export TWINE_PASSWORD="$PYPI_TOKEN"
35
+fi
36
+
37
+# check github token
38
+if [ -z "$GITHUB_TOKEN" ]; then
39
+ echo "[FATAL] GITHUB_TOKEN is not set."
40
+ exit 1
41
+else
42
+ export GH_TOKEN="$GITHUB_TOKEN"
43
+fi
44
+
45
+set -x # avoid printing secrets above
46
+
47
+# download gh CLI from github
48
+# Get latest gh CLI version from GitHub API
49
+GH_VERSION=$(curl -s https://api.github.com/repos/cli/cli/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/' | sed 's/^v//')
50
+if [ -z "$GH_VERSION" ]; then
51
+ echo "[FATAL] Failed to get latest gh CLI version from GitHub"
52
+ exit 1
53
+fi
54
+echo "Downloading gh CLI version: $GH_VERSION"
55
+GH_TARBALL="gh_${GH_VERSION}_linux_amd64.tar.gz"
56
+GH_URL="https://github.com/cli/cli/releases/download/v${GH_VERSION}/${GH_TARBALL}"
57
+GH_INSTALL_DIR="/tmp/gh-install"
58
+mkdir -p "$GH_INSTALL_DIR"
59
+pushd "$GH_INSTALL_DIR"
60
+curl -L -o "$GH_TARBALL" "$GH_URL"
61
+tar -xzf "$GH_TARBALL"
62
+GH_BIN=$(realpath $(find . -name "gh" -type f -executable | head -n 1))
63
+if [ -z "$GH_BIN" ]; then
64
+ echo "[FATAL] Failed to find gh CLI executable"
65
+ exit 1
66
+fi
67
+echo "gh CLI downloaded successfully, version: $($GH_BIN --version)"
68
+echo "Last 5 releases on GitHub:" # as a sanity check of gh and GH_TOKEN
69
+command "$GH_BIN" release list --limit 5
70
+popd
71
+
72
+# install twine from pypi
73
+python3 -m venv /tmp/vllm-release-env
74
+source /tmp/vllm-release-env/bin/activate
75
+pip install twine
76
+python3 -m twine --version
77
+
78
+# copy release wheels to local directory
79
+DIST_DIR=/tmp/vllm-release-dist
80
+echo "Existing wheels on S3:"
81
+aws s3 ls "$S3_COMMIT_PREFIX"
82
+echo "Copying wheels to local directory"
83
+mkdir -p $DIST_DIR
84
+# include only wheels for the release version, ignore all files with "dev" or "rc" in the name
85
+aws s3 cp --recursive --exclude "*" --include "vllm-${RELEASE_VERSION}*.whl" --exclude "*dev*" --exclude "*rc*" "$S3_COMMIT_PREFIX" $DIST_DIR
86
+echo "Wheels copied to local directory"
87
+# generate source tarball
88
+git archive --format=tar.gz --output="$DIST_DIR/vllm-${RELEASE_VERSION}.tar.gz" $BUILDKITE_COMMIT
89
+ls -la $DIST_DIR
90
+
91
+
92
+# upload wheels to PyPI (only default variant, i.e. files without '+' in the name)
93
+PYPI_WHEEL_FILES=$(find $DIST_DIR -name "vllm-${RELEASE_VERSION}*.whl" -not -name "*+*")
94
+if [ -z "$PYPI_WHEEL_FILES" ]; then
95
+ echo "No default variant wheels found, quitting..."
96
+ exit 1
97
+fi
98
+python3 -m twine check $PYPI_WHEEL_FILES
99
+python3 -m twine --non-interactive --verbose upload $PYPI_WHEEL_FILES
100
+echo "Wheels uploaded to PyPI"
101
+
102
+# create release on GitHub with the release version and all wheels
103
+command "$GH_BIN" release create $GIT_VERSION -d --latest --notes-from-tag --verify-tag $DIST_DIR/*.whl