A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 02:00 UTC
Add Ubuntu 24.04 support for Docker builds (#35386)
Signed-off-by: aasgaonkar <aasgaonkar@nvidia.com>
amey asgaonkar committed
Mar 24, 2026 at 13:34 UTC
0c1809c8065f51646c9ca6ce7911831a763a5d18
4 files changed
+108
-2
.buildkite/release-pipeline.yaml
+68
@@ -140,6 +140,50 @@ steps:
140
- "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0 12.1' --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu22.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130 --target vllm-openai --progress plain -f docker/Dockerfile ."
141
- "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130"
142
143
+ - label: "Build release image - x86_64 - CUDA 12.9 - Ubuntu 24.04"
144
+ depends_on: ~
145
+ id: build-release-image-x86-ubuntu2404
146
+ agents:
147
+ queue: cpu_queue_postmerge
148
+ commands:
149
+ - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
150
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg FLASHINFER_AOT_COMPILE=true --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0' --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ."
151
+ - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404"
152
+ - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404"
153
+ - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404"
154
+
155
+ - label: "Build release image - aarch64 - CUDA 12.9 - Ubuntu 24.04"
156
+ depends_on: ~
157
+ id: build-release-image-arm64-ubuntu2404
158
+ agents:
159
+ queue: arm64_cpu_queue_postmerge
160
+ commands:
161
+ - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
162
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg FLASHINFER_AOT_COMPILE=true --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0' --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ."
163
+ - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404"
164
+
165
+ - label: "Build release image - x86_64 - CUDA 13.0 - Ubuntu 24.04"
166
+ depends_on: ~
167
+ id: build-release-image-x86-cuda-13-0-ubuntu2404
168
+ agents:
169
+ queue: cpu_queue_postmerge
170
+ commands:
171
+ - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
172
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg FLASHINFER_AOT_COMPILE=true --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0 12.1' --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu24.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ."
173
+ - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130-ubuntu2404"
174
+ - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130-ubuntu2404"
175
+ - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130-ubuntu2404"
176
+
177
+ - label: "Build release image - aarch64 - CUDA 13.0 - Ubuntu 24.04"
178
+ depends_on: ~
179
+ id: build-release-image-arm64-cuda-13-0-ubuntu2404
180
+ agents:
181
+ queue: arm64_cpu_queue_postmerge
182
+ commands:
183
+ - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
184
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg FLASHINFER_AOT_COMPILE=true --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0 12.1' --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu24.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ."
185
+ - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130-ubuntu2404"
186
+
187
- block: "Build release image for x86_64 CPU"
188
key: block-cpu-release-image-build
189
depends_on: ~
@@ -212,6 +256,30 @@ steps:
256
- "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-cu130 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-cu130 --amend"
257
- "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130"
258
259
+ - label: "Create multi-arch manifest - CUDA 12.9 - Ubuntu 24.04"
260
+ depends_on:
261
+ - build-release-image-x86-ubuntu2404
262
+ - build-release-image-arm64-ubuntu2404
263
+ id: create-multi-arch-manifest-ubuntu2404
264
+ agents:
265
+ queue: small_cpu_queue_postmerge
266
+ commands:
267
+ - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
268
+ - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-ubuntu2404 --amend"
269
+ - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404"
270
+
271
+ - label: "Create multi-arch manifest - CUDA 13.0 - Ubuntu 24.04"
272
+ depends_on:
273
+ - build-release-image-x86-cuda-13-0-ubuntu2404
274
+ - build-release-image-arm64-cuda-13-0-ubuntu2404
275
+ id: create-multi-arch-manifest-cuda-13-0-ubuntu2404
276
+ agents:
277
+ queue: small_cpu_queue_postmerge
278
+ commands:
279
+ - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
280
+ - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-cu130-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-cu130-ubuntu2404 --amend"
281
+ - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130-ubuntu2404"
282
+
283
- label: "Publish nightly multi-arch image to DockerHub"
284
depends_on:
285
- create-multi-arch-manifest
docker/Dockerfile
+7
-2
@@ -24,6 +24,7 @@
24
25
ARG CUDA_VERSION=12.9.1
26
ARG PYTHON_VERSION=3.12
27
+ARG UBUNTU_VERSION=22.04
28
29
# By parameterizing the base images, we allow third-party to use their own
30
# base images. One use case is hermetic builds with base images stored in
@@ -38,7 +39,7 @@ ARG PYTHON_VERSION=3.12
39
# version are not backwards compatible with OSes that use an earlier version.
40
ARG BUILD_BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu20.04
41
# Using cuda base image with minimal dependencies necessary for JIT compilation (FlashInfer, DeepGEMM, EP kernels)
41
-ARG FINAL_BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-base-ubuntu22.04
42
+ARG FINAL_BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-base-ubuntu${UBUNTU_VERSION}
43
44
# By parameterizing the Deadsnakes repository URL, we allow third-party to use
45
# their own mirror. When doing so, we don't benefit from the transparent
@@ -111,6 +112,10 @@ RUN apt-get update -y \
112
gcc-10 \
113
g++-10 \
114
&& update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 110 --slave /usr/bin/g++ g++ /usr/bin/g++-10 \
115
+ # Install python dev headers if available (needed for cmake FindPython on Ubuntu 24.04
116
+ # which ships cmake 3.28 and requires Development.SABIModule; silently skipped on
117
+ # Ubuntu 20.04/22.04 where python3.x-dev is not available without a PPA)
118
+ && (apt-get install -y --no-install-recommends python${PYTHON_VERSION}-dev 2>/dev/null || true) \
119
&& rm -rf /var/lib/apt/lists/* \
120
&& curl -LsSf https://astral.sh/uv/install.sh | sh \
121
&& $HOME/.local/bin/uv venv /opt/venv --python ${PYTHON_VERSION} \
@@ -507,7 +512,6 @@ RUN apt-get update -y \
512
software-properties-common \
513
curl \
514
sudo \
510
- python3-pip \
515
ffmpeg \
516
libsm6 \
517
libxext6 \
@@ -535,6 +539,7 @@ RUN apt-get update -y \
539
&& update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \
540
&& update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION} \
541
&& ln -sf /usr/bin/python${PYTHON_VERSION}-config /usr/bin/python3-config \
542
+ && rm -f /usr/lib/python${PYTHON_VERSION}/EXTERNALLY-MANAGED \
543
&& curl -sS ${GET_PIP_URL} | python${PYTHON_VERSION} \
544
&& python3 --version && python3 -m pip --version
545
docker/docker-bake.hcl
+30
@@ -33,6 +33,10 @@ group "default" {
33
targets = ["openai"]
34
}
35
36
+group "all" {
37
+ targets = ["openai", "openai-ubuntu2404"]
38
+}
39
+
40
# Base targets
41
42
target "_common" {
@@ -74,3 +78,29 @@ target "openai" {
78
tags = ["vllm:openai"]
79
output = ["type=docker"]
80
}
81
+
82
+# Ubuntu 24.04 targets
83
+
84
+target "test-ubuntu2404" {
85
+ inherits = ["_common", "_labels"]
86
+ target = "test"
87
+ tags = ["vllm:test-ubuntu24.04"]
88
+ args = {
89
+ UBUNTU_VERSION = "24.04"
90
+ GDRCOPY_OS_VERSION = "Ubuntu24_04"
91
+ FLASHINFER_AOT_COMPILE = "true"
92
+ }
93
+ output = ["type=docker"]
94
+}
95
+
96
+target "openai-ubuntu2404" {
97
+ inherits = ["_common", "_labels"]
98
+ target = "vllm-openai"
99
+ tags = ["vllm:openai-ubuntu24.04"]
100
+ args = {
101
+ UBUNTU_VERSION = "24.04"
102
+ GDRCOPY_OS_VERSION = "Ubuntu24_04"
103
+ FLASHINFER_AOT_COMPILE = "true"
104
+ }
105
+ output = ["type=docker"]
106
+}
docker/versions.json
+3
@@ -7,6 +7,9 @@
7
"PYTHON_VERSION": {
8
"default": "3.12"
9
},
10
+ "UBUNTU_VERSION": {
11
+ "default": "22.04"
12
+ },
13
"BUILD_BASE_IMAGE": {
14
"default": "nvidia/cuda:12.9.1-devel-ubuntu20.04"
15
},