A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 18:45 UTC
[Build] Switch default CUDA to 13.0, update CUDA architecture lists, clean up stale build-args (#39878)
Signed-off-by: Shengqi Chen <harry-chen@outlook.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Shengqi Chen committed
Apr 23, 2026 at 15:51 UTC
3ed5231c6a7a9042a6ac4000e569ad2d85a21b9a
10 files changed
+98
-92
.buildkite/release-pipeline.yaml
+62
-57
@@ -1,3 +1,13 @@
1
+# CUDA architecture lists — following PyTorch RELEASE.md
2
+# (https://github.com/pytorch/pytorch/blob/main/RELEASE.md)
3
+# SM86 included for broader Ampere coverage; SM89 for marlin fp8 support
4
+env:
5
+ CUDA_ARCH_X86: "7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX"
6
+ # aarch64 only architectures: 8.7 for Orin, 11.0 for Thor (since CUDA 13)
7
+ CUDA_ARCH_AARCH64: "8.0 8.7 8.9 9.0 10.0 11.0 12.0+PTX"
8
+ CUDA_ARCH_X86_CU129: "7.5 8.0 8.6 8.9 9.0 10.0 12.0"
9
+ CUDA_ARCH_AARCH64_CU129: "8.0 8.7 8.9 9.0 10.0 12.0"
10
+
11
steps:
12
- input: "Provide Release version here"
13
id: input-release-version
@@ -14,12 +24,10 @@ steps:
24
agents:
25
queue: arm64_cpu_queue_release
26
commands:
17
- # #NOTE: torch_cuda_arch_list is derived from upstream PyTorch build files here:
18
- # https://github.com/pytorch/pytorch/blob/main/.ci/aarch64_linux/aarch64_ci_build.sh#L7
19
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0' --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
27
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64_CU129}\" --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
28
- "mkdir artifacts"
29
- "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
22
- - "bash .buildkite/scripts/upload-nightly-wheels.sh"
30
+ - "bash .buildkite/scripts/upload-nightly-wheels.sh manylinux_2_31"
31
env:
32
DOCKER_BUILDKIT: "1"
33
@@ -29,9 +37,7 @@ steps:
37
agents:
38
queue: arm64_cpu_queue_release
39
commands:
32
- # #NOTE: torch_cuda_arch_list is derived from upstream PyTorch build files here:
33
- # https://github.com/pytorch/pytorch/blob/main/.ci/aarch64_linux/aarch64_ci_build.sh#L7
34
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0' --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu22.04 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
40
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64}\" --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu22.04 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
41
- "mkdir artifacts"
42
- "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
43
- "bash .buildkite/scripts/upload-nightly-wheels.sh manylinux_2_35"
@@ -57,7 +63,7 @@ steps:
63
agents:
64
queue: cpu_queue_release
65
commands:
60
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
66
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86_CU129}\" --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
67
- "mkdir artifacts"
68
- "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
69
- "bash .buildkite/scripts/upload-nightly-wheels.sh manylinux_2_31"
@@ -70,7 +76,7 @@ steps:
76
agents:
77
queue: cpu_queue_release
78
commands:
73
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu22.04 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
79
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86}\" --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu22.04 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
80
- "mkdir artifacts"
81
- "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
82
- "bash .buildkite/scripts/upload-nightly-wheels.sh manylinux_2_35"
@@ -108,96 +114,95 @@ steps:
114
depends_on: block-build-release-images
115
allow_dependency_failure: true
116
steps:
111
- - label: "Build release image - x86_64 - CUDA 12.9"
117
+ - label: "Build release image - x86_64 - CUDA 13.0"
118
depends_on: ~
119
id: build-release-image-x86
120
agents:
121
queue: cpu_queue_release
122
commands:
123
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
118
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg FLASHINFER_AOT_COMPILE=true --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) --target vllm-openai --progress plain -f docker/Dockerfile ."
124
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86}\" --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu22.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) --target vllm-openai --progress plain -f docker/Dockerfile ."
125
- "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)"
126
# re-tag to default image tag and push, just in case arm64 build fails
127
- "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT"
128
- "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT"
129
124
- - label: "Build release image - aarch64 - CUDA 12.9"
130
+ - label: "Build release image - aarch64 - CUDA 13.0"
131
depends_on: ~
132
id: build-release-image-arm64
133
agents:
134
queue: arm64_cpu_queue_release
135
commands:
136
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
131
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg FLASHINFER_AOT_COMPILE=true --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0' --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) --target vllm-openai --progress plain -f docker/Dockerfile ."
137
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64}\" --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu22.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) --target vllm-openai --progress plain -f docker/Dockerfile ."
138
- "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)"
139
134
- - label: "Build release image - x86_64 - CUDA 13.0"
140
+ - label: "Build release image - x86_64 - CUDA 12.9"
141
depends_on: ~
136
- id: build-release-image-x86-cuda-13-0
142
+ id: build-release-image-x86-cuda-12-9
143
agents:
144
queue: cpu_queue_release
145
commands:
146
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
141
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu22.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130 --target vllm-openai --progress plain -f docker/Dockerfile ."
142
- - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130"
147
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86_CU129}\" --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129 --target vllm-openai --progress plain -f docker/Dockerfile ."
148
+ - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129"
149
# re-tag to default image tag and push, just in case arm64 build fails
144
- - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130"
145
- - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130"
150
+ - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129"
151
+ - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129"
152
147
- - label: "Build release image - aarch64 - CUDA 13.0"
153
+ - label: "Build release image - aarch64 - CUDA 12.9"
154
depends_on: ~
149
- id: build-release-image-arm64-cuda-13-0
155
+ id: build-release-image-arm64-cuda-12-9
156
agents:
157
queue: arm64_cpu_queue_release
158
commands:
159
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
154
- # compute capability 12.0 for RTX-50 series / RTX PRO 6000 Blackwell, 12.1 for DGX Spark
155
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0 12.1' --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu22.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130 --target vllm-openai --progress plain -f docker/Dockerfile ."
156
- - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130"
160
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64_CU129}\" --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129 --target vllm-openai --progress plain -f docker/Dockerfile ."
161
+ - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129"
162
158
- - label: "Build release image - x86_64 - CUDA 12.9 - Ubuntu 24.04"
163
+ - label: "Build release image - x86_64 - CUDA 13.0 - Ubuntu 24.04"
164
depends_on: ~
165
id: build-release-image-x86-ubuntu2404
166
agents:
167
queue: cpu_queue_release
168
commands:
169
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
165
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg FLASHINFER_AOT_COMPILE=true --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0' --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ."
170
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86}\" --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu24.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ."
171
- "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404"
172
- "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404"
173
- "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404"
174
170
- - label: "Build release image - aarch64 - CUDA 12.9 - Ubuntu 24.04"
175
+ - label: "Build release image - aarch64 - CUDA 13.0 - Ubuntu 24.04"
176
depends_on: ~
177
id: build-release-image-arm64-ubuntu2404
178
agents:
179
queue: arm64_cpu_queue_release
180
commands:
181
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
177
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg FLASHINFER_AOT_COMPILE=true --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0' --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ."
182
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64}\" --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu24.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ."
183
- "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404"
184
180
- - label: "Build release image - x86_64 - CUDA 13.0 - Ubuntu 24.04"
185
+ - label: "Build release image - x86_64 - CUDA 12.9 - Ubuntu 24.04"
186
depends_on: ~
182
- id: build-release-image-x86-cuda-13-0-ubuntu2404
187
+ id: build-release-image-x86-cuda-12-9-ubuntu2404
188
agents:
189
queue: cpu_queue_release
190
commands:
191
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
187
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg FLASHINFER_AOT_COMPILE=true --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0 12.1' --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu24.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ."
188
- - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130-ubuntu2404"
189
- - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130-ubuntu2404"
190
- - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130-ubuntu2404"
192
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86_CU129}\" --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ."
193
+ - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404"
194
+ - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404"
195
+ - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404"
196
192
- - label: "Build release image - aarch64 - CUDA 13.0 - Ubuntu 24.04"
197
+ - label: "Build release image - aarch64 - CUDA 12.9 - Ubuntu 24.04"
198
depends_on: ~
194
- id: build-release-image-arm64-cuda-13-0-ubuntu2404
199
+ id: build-release-image-arm64-cuda-12-9-ubuntu2404
200
agents:
201
queue: arm64_cpu_queue_release
202
commands:
203
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
199
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg FLASHINFER_AOT_COMPILE=true --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0 12.1' --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu24.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ."
200
- - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130-ubuntu2404"
204
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64_CU129}\" --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ."
205
+ - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404"
206
207
- block: "Build release image for x86_64 CPU"
208
key: block-cpu-release-image-build
@@ -238,7 +243,7 @@ steps:
243
- group: "Publish release images"
244
key: "publish-release-images"
245
steps:
241
- - label: "Create multi-arch manifest - CUDA 12.9"
246
+ - label: "Create multi-arch manifest - CUDA 13.0"
247
depends_on:
248
- build-release-image-x86
249
- build-release-image-arm64
@@ -250,7 +255,7 @@ steps:
255
- "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64 --amend"
256
- "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT"
257
253
- - label: "Annotate release workflow - CUDA 12.9"
258
+ - label: "Annotate release workflow - CUDA 13.0"
259
depends_on:
260
- create-multi-arch-manifest
261
id: annotate-release-workflow
@@ -259,19 +264,19 @@ steps:
264
commands:
265
- "bash .buildkite/scripts/annotate-release.sh"
266
262
- - label: "Create multi-arch manifest - CUDA 13.0"
267
+ - label: "Create multi-arch manifest - CUDA 12.9"
268
depends_on:
264
- - build-release-image-x86-cuda-13-0
265
- - build-release-image-arm64-cuda-13-0
266
- id: create-multi-arch-manifest-cuda-13-0
269
+ - build-release-image-x86-cuda-12-9
270
+ - build-release-image-arm64-cuda-12-9
271
+ id: create-multi-arch-manifest-cuda-12-9
272
agents:
273
queue: small_cpu_queue_release
274
commands:
275
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
271
- - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-cu130 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-cu130 --amend"
272
- - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130"
276
+ - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-cu129 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-cu129 --amend"
277
+ - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129"
278
274
- - label: "Create multi-arch manifest - CUDA 12.9 - Ubuntu 24.04"
279
+ - label: "Create multi-arch manifest - CUDA 13.0 - Ubuntu 24.04"
280
depends_on:
281
- build-release-image-x86-ubuntu2404
282
- build-release-image-arm64-ubuntu2404
@@ -283,17 +288,17 @@ steps:
288
- "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-ubuntu2404 --amend"
289
- "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404"
290
286
- - label: "Create multi-arch manifest - CUDA 13.0 - Ubuntu 24.04"
291
+ - label: "Create multi-arch manifest - CUDA 12.9 - Ubuntu 24.04"
292
depends_on:
288
- - build-release-image-x86-cuda-13-0-ubuntu2404
289
- - build-release-image-arm64-cuda-13-0-ubuntu2404
290
- id: create-multi-arch-manifest-cuda-13-0-ubuntu2404
293
+ - build-release-image-x86-cuda-12-9-ubuntu2404
294
+ - build-release-image-arm64-cuda-12-9-ubuntu2404
295
+ id: create-multi-arch-manifest-cuda-12-9-ubuntu2404
296
agents:
297
queue: small_cpu_queue_release
298
commands:
299
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
295
- - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-cu130-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-cu130-ubuntu2404 --amend"
296
- - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130-ubuntu2404"
300
+ - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-cu129-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-cu129-ubuntu2404 --amend"
301
+ - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404"
302
303
- label: "Publish nightly multi-arch image to DockerHub"
304
depends_on:
@@ -313,16 +318,16 @@ steps:
318
DOCKER_BUILDKIT: "1"
319
DOCKERHUB_USERNAME: "vllmbot"
320
316
- - label: "Publish nightly multi-arch image to DockerHub - CUDA 13.0"
321
+ - label: "Publish nightly multi-arch image to DockerHub - CUDA 12.9"
322
depends_on:
318
- - create-multi-arch-manifest-cuda-13-0
323
+ - create-multi-arch-manifest-cuda-12-9
324
if: build.env("NIGHTLY") == "1"
325
agents:
326
queue: small_cpu_queue_release
327
commands:
323
- - "bash .buildkite/scripts/push-nightly-builds.sh cu130"
328
+ - "bash .buildkite/scripts/push-nightly-builds.sh cu129"
329
# Clean up old nightly builds (keep only last 14)
325
- - "bash .buildkite/scripts/cleanup-nightly-builds.sh cu130-nightly-"
330
+ - "bash .buildkite/scripts/cleanup-nightly-builds.sh cu129-nightly-"
331
plugins:
332
- docker-login#v3.0.0:
333
username: vllmbot
.buildkite/scripts/annotate-release.sh
+22
-22
@@ -13,12 +13,12 @@ ROCM_BASE_CACHE_KEY=$(.buildkite/scripts/cache-rocm-base-wheels.sh key)
13
buildkite-agent annotate --style 'info' --context 'release-workflow' << EOF
14
To download the wheel (by commit):
15
\`\`\`
16
-aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}-cp38-abi3-manylinux_2_31_x86_64.whl .
17
-aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}-cp38-abi3-manylinux_2_31_aarch64.whl .
16
+aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}-cp38-abi3-manylinux_2_35_x86_64.whl .
17
+aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}-cp38-abi3-manylinux_2_35_aarch64.whl .
18
19
-(Optional) For CUDA 13.0:
20
-aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}+cu130-cp38-abi3-manylinux_2_35_x86_64.whl .
21
-aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}+cu130-cp38-abi3-manylinux_2_35_aarch64.whl .
19
+(Optional) For CUDA 12.9:
20
+aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}+cu129-cp38-abi3-manylinux_2_31_x86_64.whl .
21
+aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}+cu129-cp38-abi3-manylinux_2_31_aarch64.whl .
22
23
(Optional) For CPU:
24
aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}+cpu-cp38-abi3-manylinux_2_35_x86_64.whl .
@@ -33,8 +33,8 @@ To download and upload the image:
33
34
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-x86_64
35
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-aarch64
36
-docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-x86_64-cu130
37
-docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-aarch64-cu130
36
+docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-x86_64-cu129
37
+docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-aarch64-cu129
38
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base
39
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm
40
docker pull public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v${RELEASE_VERSION}
@@ -50,11 +50,11 @@ docker tag vllm/vllm-openai:x86_64 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64
50
docker push vllm/vllm-openai:latest-x86_64
51
docker push vllm/vllm-openai:v${RELEASE_VERSION}-x86_64
52
53
-docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-x86_64-cu130 vllm/vllm-openai:x86_64-cu130
54
-docker tag vllm/vllm-openai:x86_64-cu130 vllm/vllm-openai:latest-x86_64-cu130
55
-docker tag vllm/vllm-openai:x86_64-cu130 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu130
56
-docker push vllm/vllm-openai:latest-x86_64-cu130
57
-docker push vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu130
53
+docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-x86_64-cu129 vllm/vllm-openai:x86_64-cu129
54
+docker tag vllm/vllm-openai:x86_64-cu129 vllm/vllm-openai:latest-x86_64-cu129
55
+docker tag vllm/vllm-openai:x86_64-cu129 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129
56
+docker push vllm/vllm-openai:latest-x86_64-cu129
57
+docker push vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129
58
59
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-aarch64 vllm/vllm-openai:aarch64
60
docker tag vllm/vllm-openai:aarch64 vllm/vllm-openai:latest-aarch64
@@ -62,11 +62,11 @@ docker tag vllm/vllm-openai:aarch64 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64
62
docker push vllm/vllm-openai:latest-aarch64
63
docker push vllm/vllm-openai:v${RELEASE_VERSION}-aarch64
64
65
-docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-aarch64-cu130 vllm/vllm-openai:aarch64-cu130
66
-docker tag vllm/vllm-openai:aarch64-cu130 vllm/vllm-openai:latest-aarch64-cu130
67
-docker tag vllm/vllm-openai:aarch64-cu130 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu130
68
-docker push vllm/vllm-openai:latest-aarch64-cu130
69
-docker push vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu130
65
+docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-aarch64-cu129 vllm/vllm-openai:aarch64-cu129
66
+docker tag vllm/vllm-openai:aarch64-cu129 vllm/vllm-openai:latest-aarch64-cu129
67
+docker tag vllm/vllm-openai:aarch64-cu129 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129
68
+docker push vllm/vllm-openai:latest-aarch64-cu129
69
+docker push vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129
70
71
## ROCm
72
@@ -104,11 +104,11 @@ docker manifest create vllm/vllm-openai:v${RELEASE_VERSION} vllm/vllm-openai:v${
104
docker manifest push vllm/vllm-openai:latest
105
docker manifest push vllm/vllm-openai:v${RELEASE_VERSION}
106
107
-docker manifest rm vllm/vllm-openai:latest-cu130
108
-docker manifest create vllm/vllm-openai:latest-cu130 vllm/vllm-openai:latest-x86_64-cu130 vllm/vllm-openai:latest-aarch64-cu130
109
-docker manifest create vllm/vllm-openai:v${RELEASE_VERSION}-cu130 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu130 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu130
110
-docker manifest push vllm/vllm-openai:latest-cu130
111
-docker manifest push vllm/vllm-openai:v${RELEASE_VERSION}-cu130
107
+docker manifest rm vllm/vllm-openai:latest-cu129
108
+docker manifest create vllm/vllm-openai:latest-cu129 vllm/vllm-openai:latest-x86_64-cu129 vllm/vllm-openai:latest-aarch64-cu129
109
+docker manifest create vllm/vllm-openai:v${RELEASE_VERSION}-cu129 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129
110
+docker manifest push vllm/vllm-openai:latest-cu129
111
+docker manifest push vllm/vllm-openai:v${RELEASE_VERSION}-cu129
112
113
docker manifest rm vllm/vllm-openai-cpu:latest || true
114
docker manifest create vllm/vllm-openai-cpu:latest vllm/vllm-openai-cpu:latest-x86_64 vllm/vllm-openai-cpu:latest-arm64
.buildkite/scripts/check-ray-compatibility.sh
+1
-1
@@ -29,7 +29,7 @@ if python3 -c "import torch; assert torch.version.hip" 2>/dev/null; then
29
TORCH_INDEX_URL=""
30
fi
31
else
32
- TORCH_INDEX_URL="https://download.pytorch.org/whl/cu129"
32
+ TORCH_INDEX_URL="https://download.pytorch.org/whl/cu130"
33
fi
34
echo ">>> Using PyTorch index: ${TORCH_INDEX_URL:-PyPI default}"
35
.buildkite/scripts/generate-and-upload-nightly-index.sh
+1
-1
@@ -9,7 +9,7 @@ set -ex
9
10
BUCKET="vllm-wheels"
11
INDICES_OUTPUT_DIR="indices"
12
-DEFAULT_VARIANT_ALIAS="cu129" # align with vLLM_MAIN_CUDA_VERSION in vllm/envs.py
12
+DEFAULT_VARIANT_ALIAS="cu130" # align with vLLM_MAIN_CUDA_VERSION in vllm/envs.py
13
PYTHON="${PYTHON_PROG:-python3}" # try to read from env var, otherwise use python3
14
SUBPATH=$BUILDKITE_COMMIT
15
S3_COMMIT_PREFIX="s3://$BUCKET/$SUBPATH/"
.github/workflows/scripts/build.sh
+1
-1
@@ -14,7 +14,7 @@ $python_executable -m pip install -r requirements/build/cuda.txt -r requirements
14
# Limit the number of parallel jobs to avoid OOM
15
export MAX_JOBS=1
16
# Make sure release wheels are built for the following architectures
17
-export TORCH_CUDA_ARCH_LIST="7.0 7.5 8.0 8.6 8.9 9.0+PTX"
17
+export TORCH_CUDA_ARCH_LIST="7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX"
18
19
bash tools/check_repo.sh
20
CMakeLists.txt
+6
-3
@@ -94,12 +94,15 @@ find_package(Torch REQUIRED)
94
# This check must happen after find_package(Torch) because that's when CMAKE_CUDA_COMPILER_VERSION gets defined
95
if(DEFINED CMAKE_CUDA_COMPILER_VERSION AND
96
CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 13.0)
97
- set(CUDA_SUPPORTED_ARCHS "7.5;8.0;8.6;8.7;8.9;9.0;10.0;11.0;12.0;12.1")
97
+ # starting from CUDA 12.9 and Blackwell (10.0), we use family-specific targets (10.0f, 12.0f, etc)
98
+ # to support the whole generation without specifying all sub-architectures
99
+ # see: https://developer.nvidia.com/blog/nvidia-blackwell-and-nvidia-cuda-12-9-introduce-family-specific-architecture-features/
100
+ set(CUDA_SUPPORTED_ARCHS "7.5;8.0;8.6;8.7;8.9;9.0;10.0;11.0;12.0")
101
elseif(DEFINED CMAKE_CUDA_COMPILER_VERSION AND
102
CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 12.8)
100
- set(CUDA_SUPPORTED_ARCHS "7.0;7.2;7.5;8.0;8.6;8.7;8.9;9.0;10.0;10.1;12.0;12.1")
103
+ set(CUDA_SUPPORTED_ARCHS "7.5;8.0;8.6;8.7;8.9;9.0;10.0;10.1;10.3;12.0;12.1")
104
else()
102
- set(CUDA_SUPPORTED_ARCHS "7.0;7.2;7.5;8.0;8.6;8.7;8.9;9.0")
105
+ set(CUDA_SUPPORTED_ARCHS "7.0;7.5;8.0;8.6;8.7;8.9;9.0")
106
endif()
107
108
#
docker/Dockerfile
+2
-2
@@ -188,7 +188,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
188
# Explicitly set the list to avoid issues with torch 2.2
189
# See https://github.com/pytorch/pytorch/pull/123243
190
# From versions.json: .torch.cuda_arch_list
191
-ARG torch_cuda_arch_list='7.0 7.5 8.0 8.9 9.0 10.0 12.0'
191
+ARG torch_cuda_arch_list='7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX'
192
ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list}
193
#################### BUILD BASE IMAGE ####################
194
@@ -765,7 +765,7 @@ ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
765
ENV UV_HTTP_TIMEOUT=500
766
767
# install kv_connectors if requested
768
-ARG torch_cuda_arch_list='7.0 7.5 8.0 8.9 9.0 10.0 12.0'
768
+ARG torch_cuda_arch_list='7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX'
769
ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list}
770
RUN --mount=type=cache,target=/root/.cache/uv \
771
--mount=type=bind,source=requirements/kv_connectors.txt,target=/tmp/kv_connectors.txt,ro \
docker/docker-bake.hcl
-2
@@ -88,7 +88,6 @@ target "test-ubuntu2404" {
88
args = {
89
UBUNTU_VERSION = "24.04"
90
GDRCOPY_OS_VERSION = "Ubuntu24_04"
91
- FLASHINFER_AOT_COMPILE = "true"
91
}
92
output = ["type=docker"]
93
}
@@ -100,7 +99,6 @@ target "openai-ubuntu2404" {
99
args = {
100
UBUNTU_VERSION = "24.04"
101
GDRCOPY_OS_VERSION = "Ubuntu24_04"
103
- FLASHINFER_AOT_COMPILE = "true"
102
}
103
output = ["type=docker"]
104
}
docker/versions.json
+1
-1
@@ -32,7 +32,7 @@
32
"default": "false"
33
},
34
"TORCH_CUDA_ARCH_LIST": {
35
- "default": "7.0 7.5 8.0 8.9 9.0 10.0 12.0"
35
+ "default": "7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX"
36
},
37
"MAX_JOBS": {
38
"default": "2"
vllm/envs.py
+2
-2
@@ -79,7 +79,7 @@ if TYPE_CHECKING:
79
VLLM_MEDIA_CONNECTOR: str = "http"
80
VLLM_MM_HASHER_ALGORITHM: str = "blake3"
81
VLLM_TARGET_DEVICE: str = "cuda"
82
- VLLM_MAIN_CUDA_VERSION: str = "12.9"
82
+ VLLM_MAIN_CUDA_VERSION: str = "13.0"
83
VLLM_FLOAT32_MATMUL_PRECISION: Literal["highest", "high", "medium"] = "highest"
84
VLLM_BATCH_INVARIANT: bool = False
85
MAX_JOBS: str | None = None
@@ -493,7 +493,7 @@ environment_variables: dict[str, Callable[[], Any]] = {
493
"VLLM_TARGET_DEVICE": lambda: os.getenv("VLLM_TARGET_DEVICE", "cuda").lower(),
494
# Main CUDA version of vLLM. This follows PyTorch but can be overridden.
495
"VLLM_MAIN_CUDA_VERSION": lambda: (
496
- os.getenv("VLLM_MAIN_CUDA_VERSION", "").lower() or "12.9"
496
+ os.getenv("VLLM_MAIN_CUDA_VERSION", "").lower() or "13.0"
497
),
498
# Controls PyTorch float32 matmul precision mode within vLLM workers.
499
# Valid options mirror torch.set_float32_matmul_precision