A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[ci] fix wheel names for arm wheels (#24898)
Signed-off-by: simon-mo <simon.mo@hey.com>
Simon Mo committed
Sep 15, 2025 at 14:39 UTC
fd2f10546c88a8accc199703b67ea31b427705b1
5 files changed
+36
-23
.buildkite/release-pipeline.yaml
+4
-12
@@ -1,24 +1,22 @@
1
steps:
2
# aarch64 + CUDA builds. PyTorch 2.8 aarch64 + CUDA wheel is only available on CUDA 12.9
3
- label: "Build arm64 wheel - CUDA 12.9"
4
+ depends_on: ~
5
id: build-wheel-arm64-cuda-12-9
6
agents:
7
queue: arm64_cpu_queue_postmerge
8
commands:
9
# #NOTE: torch_cuda_arch_list is derived from upstream PyTorch build files here:
10
# https://github.com/pytorch/pytorch/blob/main/.ci/aarch64_linux/aarch64_ci_build.sh#L7
10
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list='8.7 9.0 10.0+PTX 12.0' --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
11
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg VLLM_MAIN_CUDA_VERSION=12.9 --build-arg torch_cuda_arch_list='8.7 9.0 10.0+PTX 12.0' --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
12
- "mkdir artifacts"
13
- "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
14
- "bash .buildkite/scripts/upload-wheels.sh"
15
env:
16
DOCKER_BUILDKIT: "1"
17
17
- - block: "Build CUDA 12.8 wheel"
18
- key: block-build-cu128-wheel
19
-
18
- label: "Build wheel - CUDA 12.8"
21
- depends_on: block-build-cu128-wheel
19
+ depends_on: ~
20
id: build-wheel-cuda-12-8
21
agents:
22
queue: cpu_queue_postmerge
@@ -30,12 +28,8 @@ steps:
28
env:
29
DOCKER_BUILDKIT: "1"
30
33
- - block: "Build CUDA 12.6 wheel"
34
- key: block-build-cu126-wheel
35
- depends_on: ~
36
-
31
- label: "Build wheel - CUDA 12.6"
38
- depends_on: block-build-cu126-wheel
32
+ depends_on: ~
33
id: build-wheel-cuda-12-6
34
agents:
35
queue: cpu_queue_postmerge
@@ -102,8 +96,6 @@ steps:
96
depends_on:
97
- create-multi-arch-manifest
98
- build-wheel-cuda-12-8
105
- - build-wheel-cuda-12-6
106
- - build-wheel-cuda-12-9
99
id: annotate-release-workflow
100
agents:
101
queue: cpu_queue_postmerge
.buildkite/scripts/annotate-release.sh
+22
-7
@@ -14,18 +14,33 @@ buildkite-agent annotate --style 'info' --context 'release-workflow' << EOF
14
To download the wheel:
15
\`\`\`
16
aws s3 cp s3://vllm-wheels/${RELEASE_VERSION}/vllm-${RELEASE_VERSION}-cp38-abi3-manylinux1_x86_64.whl .
17
+aws s3 cp s3://vllm-wheels/${RELEASE_VERSION}/vllm-${RELEASE_VERSION}-cp38-abi3-manylinux2014_aarch64.whl .
18
+
19
aws s3 cp s3://vllm-wheels/${RELEASE_VERSION}+cu126/vllm-${RELEASE_VERSION}+cu126-cp38-abi3-manylinux1_x86_64.whl .
18
-aws s3 cp s3://vllm-wheels/${RELEASE_VERSION}+cu118/vllm-${RELEASE_VERSION}+cu118-cp38-abi3-manylinux1_x86_64.whl .
20
+aws s3 cp s3://vllm-wheels/${RELEASE_VERSION}+cu129/vllm-${RELEASE_VERSION}+cu129-cp38-abi3-manylinux1_x86_64.whl .
21
\`\`\`
22
23
To download and upload the image:
24
25
\`\`\`
24
-docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}
25
-docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT} vllm/vllm-openai
26
-docker tag vllm/vllm-openai vllm/vllm-openai:latest
27
-docker tag vllm/vllm-openai vllm/vllm-openai:v${RELEASE_VERSION}
28
-docker push vllm/vllm-openai:latest
29
-docker push vllm/vllm-openai:v${RELEASE_VERSION}
26
+docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-x86_64
27
+docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-aarch64
28
+
29
+docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-x86_64 vllm/vllm-openai:x86_64
30
+docker tag vllm/vllm-openai:x86_64 vllm/vllm-openai:latest-x86_64
31
+docker tag vllm/vllm-openai:x86_64 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64
32
+docker push vllm/vllm-openai:latest-x86_64
33
+docker push vllm/vllm-openai:v${RELEASE_VERSION}-x86_64
34
+
35
+docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-aarch64 vllm/vllm-openai:aarch64
36
+docker tag vllm/vllm-openai:aarch64 vllm/vllm-openai:latest-aarch64
37
+docker tag vllm/vllm-openai:aarch64 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64
38
+docker push vllm/vllm-openai:latest-aarch64
39
+docker push vllm/vllm-openai:v${RELEASE_VERSION}-aarch64
40
+
41
+docker manifest create vllm/vllm-openai:latest vllm/vllm-openai:latest-x86_64 vllm/vllm-openai:latest-aarch64 --amend
42
+docker manifest create vllm/vllm-openai:v${RELEASE_VERSION} vllm/vllm-openai:v${RELEASE_VERSION}-x86_64 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64 --amend
43
+docker manifest push vllm/vllm-openai:latest
44
+docker manifest push vllm/vllm-openai:v${RELEASE_VERSION}
45
\`\`\`
46
EOF
\ No newline at end of file
docker/Dockerfile
+2
@@ -196,6 +196,7 @@ ARG SCCACHE_S3_NO_CREDENTIALS=0
196
197
# Flag to control whether to use pre-built vLLM wheels
198
ARG VLLM_USE_PRECOMPILED=""
199
+ARG VLLM_MAIN_CUDA_VERSION=""
200
201
# if USE_SCCACHE is set, use sccache to speed up compilation
202
RUN --mount=type=cache,target=/root/.cache/uv \
@@ -213,6 +214,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
214
&& export SCCACHE_IDLE_TIMEOUT=0 \
215
&& export CMAKE_BUILD_TYPE=Release \
216
&& export VLLM_USE_PRECOMPILED="${VLLM_USE_PRECOMPILED}" \
217
+ && export VLLM_MAIN_CUDA_VERSION="${VLLM_MAIN_CUDA_VERSION}" \
218
&& export VLLM_DOCKER_BUILD_CONTEXT=1 \
219
&& sccache --show-stats \
220
&& python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38 \
setup.py
+2
-4
@@ -56,8 +56,6 @@ elif (sys.platform.startswith("linux") and torch.version.cuda is None
56
# fallback to cpu
57
VLLM_TARGET_DEVICE = "cpu"
58
59
-MAIN_CUDA_VERSION = "12.8"
60
-
59
60
def is_sccache_available() -> bool:
61
return which("sccache") is not None and \
@@ -507,7 +505,7 @@ def get_vllm_version() -> str:
505
version += f"{sep}precompiled"
506
else:
507
cuda_version = str(get_nvcc_cuda_version())
510
- if cuda_version != MAIN_CUDA_VERSION:
508
+ if cuda_version != envs.VLLM_MAIN_CUDA_VERSION:
509
cuda_version_str = cuda_version.replace(".", "")[:3]
510
# skip this for source tarball, required for pypi
511
if "sdist" not in sys.argv:
@@ -515,7 +513,7 @@ def get_vllm_version() -> str:
513
elif _is_hip():
514
# Get the Rocm Version
515
rocm_version = get_rocm_version() or torch.version.hip
518
- if rocm_version and rocm_version != MAIN_CUDA_VERSION:
516
+ if rocm_version and rocm_version != envs.VLLM_MAIN_CUDA_VERSION:
517
version += f"{sep}rocm{rocm_version.replace('.', '')[:3]}"
518
elif _is_tpu():
519
version += f"{sep}tpu"
vllm/envs.py
+6
@@ -70,6 +70,7 @@ if TYPE_CHECKING:
70
VLLM_VIDEO_LOADER_BACKEND: str = "opencv"
71
VLLM_MM_INPUT_CACHE_GIB: int = 4
72
VLLM_TARGET_DEVICE: str = "cuda"
73
+ VLLM_MAIN_CUDA_VERSION: str = "12.8"
74
MAX_JOBS: Optional[str] = None
75
NVCC_THREADS: Optional[str] = None
76
VLLM_USE_PRECOMPILED: bool = False
@@ -249,6 +250,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
250
"VLLM_TARGET_DEVICE":
251
lambda: os.getenv("VLLM_TARGET_DEVICE", "cuda").lower(),
252
253
+ # Main CUDA version of vLLM, supporting [12.6, 12.8, 12.9],
254
+ # 12.8 is the default. This follows PyTorch but can be overridden.
255
+ "VLLM_MAIN_CUDA_VERSION":
256
+ lambda: os.getenv("VLLM_MAIN_CUDA_VERSION", "").lower() or "12.8",
257
+
258
# Maximum number of compilation jobs to run in parallel.
259
# By default this is the number of CPUs
260
"MAX_JOBS":