A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Update release pipeline post PyTorch 2.8.0 update (#24073)
Signed-off-by: Huy Do <huydhn@gmail.com> Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: Huy Do <huydhn@gmail.com>
youkaichao committed
Sep 3, 2025 at 10:09 UTC
42dc59dbaceb2b9aa1477e9b3e0c33b379678468
3 files changed
+29
-27
.buildkite/release-pipeline.yaml
+16
-16
@@ -1,21 +1,24 @@
1
steps:
2
- # aarch64 + CUDA builds
3
- - label: "Build arm64 wheel - CUDA 12.8"
4
- id: build-wheel-arm64-cuda-12-8
2
+ # aarch64 + CUDA builds. PyTorch 2.8 aarch64 + CUDA wheel is only available on CUDA 12.9
3
+ - label: "Build arm64 wheel - CUDA 12.9"
4
+ id: build-wheel-arm64-cuda-12-9
5
agents:
6
queue: arm64_cpu_queue_postmerge
7
commands:
8
# #NOTE: torch_cuda_arch_list is derived from upstream PyTorch build files here:
9
# https://github.com/pytorch/pytorch/blob/main/.ci/aarch64_linux/aarch64_ci_build.sh#L7
10
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.8.1 --build-arg torch_cuda_arch_list='8.7 9.0 10.0+PTX 12.0' --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
10
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list='8.7 9.0 10.0+PTX 12.0' --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
11
- "mkdir artifacts"
12
- "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
13
- "bash .buildkite/scripts/upload-wheels.sh"
14
env:
15
DOCKER_BUILDKIT: "1"
16
17
- # x86 + CUDA builds
17
+ - block: "Build CUDA 12.8 wheel"
18
+ key: block-build-cu128-wheel
19
+
20
- label: "Build wheel - CUDA 12.8"
21
+ depends_on: block-build-cu128-wheel
22
id: build-wheel-cuda-12-8
23
agents:
24
queue: cpu_queue_postmerge
@@ -44,18 +47,14 @@ steps:
47
env:
48
DOCKER_BUILDKIT: "1"
49
47
- # Note(simon): We can always build CUDA 11.8 wheel to ensure the build is working.
48
- # However, this block can be uncommented to save some compute hours.
49
- # - block: "Build CUDA 11.8 wheel"
50
- # key: block-build-cu118-wheel
51
-
52
- - label: "Build wheel - CUDA 11.8"
53
- # depends_on: block-build-cu118-wheel
54
- id: build-wheel-cuda-11-8
50
+ # x86 + CUDA builds
51
+ - label: "Build wheel - CUDA 12.9"
52
+ depends_on: ~
53
+ id: build-wheel-cuda-12-9
54
agents:
55
queue: cpu_queue_postmerge
56
commands:
58
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=11.8.0 --build-arg torch_cuda_arch_list='7.0 7.5 8.0 8.9 9.0+PTX' --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
57
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list='7.0 7.5 8.0 8.9 9.0+PTX' --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
58
- "mkdir artifacts"
59
- "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
60
- "bash .buildkite/scripts/upload-wheels.sh"
@@ -75,6 +74,7 @@ steps:
74
- "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT"
75
- "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT"
76
77
+ # PyTorch 2.8 aarch64 + CUDA wheel is only available on CUDA 12.9
78
- label: "Build release image (arm64)"
79
depends_on: ~
80
id: build-release-image-arm64
@@ -82,7 +82,7 @@ steps:
82
queue: arm64_cpu_queue_postmerge
83
commands:
84
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
85
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.8.1 --build-arg torch_cuda_arch_list='8.7 9.0 10.0+PTX 12.0' --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) --target vllm-openai --progress plain -f docker/Dockerfile ."
85
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list='8.7 9.0 10.0+PTX 12.0' --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) --target vllm-openai --progress plain -f docker/Dockerfile ."
86
- "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)"
87
88
# Add job to create multi-arch manifest
@@ -103,7 +103,7 @@ steps:
103
- create-multi-arch-manifest
104
- build-wheel-cuda-12-8
105
- build-wheel-cuda-12-6
106
- - build-wheel-cuda-11-8
106
+ - build-wheel-cuda-12-9
107
id: annotate-release-workflow
108
agents:
109
queue: cpu_queue_postmerge
.buildkite/scripts/upload-wheels.sh
+12
-10
@@ -58,14 +58,15 @@ python3 .buildkite/generate_index.py --wheel "$normal_wheel"
58
aws s3 cp "$wheel" "s3://vllm-wheels/$BUILDKITE_COMMIT/"
59
aws s3 cp "$normal_wheel" "s3://vllm-wheels/$BUILDKITE_COMMIT/"
60
61
-if [[ $normal_wheel == *"cu118"* ]]; then
62
- # if $normal_wheel matches cu118, do not upload the index.html
63
- echo "Skipping index files for cu118 wheels"
64
-elif [[ $normal_wheel == *"cu126"* ]]; then
61
+if [[ $normal_wheel == *"cu126"* ]]; then
62
# if $normal_wheel matches cu126, do not upload the index.html
63
echo "Skipping index files for cu126 wheels"
64
+elif [[ $normal_wheel == *"cu128"* ]]; then
65
+ # if $normal_wheel matches cu128, do not upload the index.html
66
+ echo "Skipping index files for cu128 wheels"
67
else
68
- # only upload index.html for cu128 wheels (default wheels)
68
+ # only upload index.html for cu129 wheels (default wheels) as it
69
+ # is available on both x86 and arm64
70
aws s3 cp index.html "s3://vllm-wheels/$BUILDKITE_COMMIT/vllm/index.html"
71
aws s3 cp "s3://vllm-wheels/nightly/index.html" "s3://vllm-wheels/$BUILDKITE_COMMIT/index.html"
72
fi
@@ -74,14 +75,15 @@ fi
75
aws s3 cp "$wheel" "s3://vllm-wheels/nightly/"
76
aws s3 cp "$normal_wheel" "s3://vllm-wheels/nightly/"
77
77
-if [[ $normal_wheel == *"cu118"* ]]; then
78
- # if $normal_wheel matches cu118, do not upload the index.html
79
- echo "Skipping index files for cu118 wheels"
80
-elif [[ $normal_wheel == *"cu126"* ]]; then
78
+if [[ $normal_wheel == *"cu126"* ]]; then
79
# if $normal_wheel matches cu126, do not upload the index.html
80
echo "Skipping index files for cu126 wheels"
81
+elif [[ $normal_wheel == *"cu128"* ]]; then
82
+ # if $normal_wheel matches cu128, do not upload the index.html
83
+ echo "Skipping index files for cu128 wheels"
84
else
84
- # only upload index.html for cu128 wheels (default wheels)
85
+ # only upload index.html for cu129 wheels (default wheels) as it
86
+ # is available on both x86 and arm64
87
aws s3 cp index.html "s3://vllm-wheels/nightly/vllm/index.html"
88
fi
89
tools/install_deepgemm.sh
+1
-1
@@ -105,4 +105,4 @@ fi
105
106
popd
107
108
-echo "✅ DeepGEMM installation completed successfully"
\ No newline at end of file
108
+echo "✅ DeepGEMM installation completed successfully"