A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 22:15 UTC
Use CUDA 12.4 as default for release and nightly wheels (#12098)
Michael Goin committed
Feb 26, 2025 at 22:06 UTC
ca377cf1b99373f51b881bdd6d48d862cb1394d1
4 files changed
+25
-9
.buildkite/release-pipeline.yaml
+12
-1
@@ -1,4 +1,15 @@
1
steps:
2
+ - label: "Build wheel - CUDA 12.4"
3
+ agents:
4
+ queue: cpu_queue_postmerge
5
+ commands:
6
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.4.0 --tag vllm-ci:build-image --target build --progress plain ."
7
+ - "mkdir artifacts"
8
+ - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
9
+ - "bash .buildkite/upload-wheels.sh"
10
+ env:
11
+ DOCKER_BUILDKIT: "1"
12
+
13
- label: "Build wheel - CUDA 12.1"
14
agents:
15
queue: cpu_queue_postmerge
@@ -37,7 +48,7 @@ steps:
48
queue: cpu_queue_postmerge
49
commands:
50
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
40
- - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.1.0 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT --target vllm-openai --progress plain ."
51
+ - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.4.0 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT --target vllm-openai --progress plain ."
52
- "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT"
53
54
- label: "Build and publish TPU release image"
.buildkite/upload-wheels.sh
+8
-2
@@ -50,8 +50,11 @@ aws s3 cp "$normal_wheel" "s3://vllm-wheels/$BUILDKITE_COMMIT/"
50
if [[ $normal_wheel == *"cu118"* ]]; then
51
# if $normal_wheel matches cu118, do not upload the index.html
52
echo "Skipping index files for cu118 wheels"
53
+elif [[ $normal_wheel == *"cu121"* ]]; then
54
+ # if $normal_wheel matches cu121, do not upload the index.html
55
+ echo "Skipping index files for cu121 wheels"
56
else
54
- # only upload index.html for cu12 wheels (default wheels)
57
+ # only upload index.html for cu124 wheels (default wheels)
58
aws s3 cp index.html "s3://vllm-wheels/$BUILDKITE_COMMIT/vllm/index.html"
59
aws s3 cp "s3://vllm-wheels/nightly/index.html" "s3://vllm-wheels/$BUILDKITE_COMMIT/index.html"
60
fi
@@ -63,8 +66,11 @@ aws s3 cp "$normal_wheel" "s3://vllm-wheels/nightly/"
66
if [[ $normal_wheel == *"cu118"* ]]; then
67
# if $normal_wheel matches cu118, do not upload the index.html
68
echo "Skipping index files for cu118 wheels"
69
+elif [[ $normal_wheel == *"cu121"* ]]; then
70
+ # if $normal_wheel matches cu121, do not upload the index.html
71
+ echo "Skipping index files for cu121 wheels"
72
else
67
- # only upload index.html for cu12 wheels (default wheels)
73
+ # only upload index.html for cu124 wheels (default wheels)
74
aws s3 cp index.html "s3://vllm-wheels/nightly/vllm/index.html"
75
fi
76
docs/source/getting_started/installation/gpu/cuda.inc.md
+2
-2
@@ -23,12 +23,12 @@ Therefore, it is recommended to install vLLM with a **fresh new** environment. I
23
You can install vLLM using either `pip` or `uv pip`:
24
25
```console
26
-# Install vLLM with CUDA 12.1.
26
+# Install vLLM with CUDA 12.4.
27
pip install vllm # If you are using pip.
28
uv pip install vllm # If you are using uv.
29
```
30
31
-As of now, vLLM's binaries are compiled with CUDA 12.1 and public PyTorch release versions by default. We also provide vLLM binaries compiled with CUDA 11.8 and public PyTorch release versions:
31
+As of now, vLLM's binaries are compiled with CUDA 12.4 and public PyTorch release versions by default. We also provide vLLM binaries compiled with CUDA 12.1, 11.8, and public PyTorch release versions:
32
33
```console
34
# Install vLLM with CUDA 11.8.
setup.py
+3
-4
@@ -54,7 +54,7 @@ elif (sys.platform.startswith("linux") and torch.version.cuda is None
54
# fallback to cpu
55
VLLM_TARGET_DEVICE = "cpu"
56
57
-MAIN_CUDA_VERSION = "12.1"
57
+MAIN_CUDA_VERSION = "12.4"
58
59
60
def is_sccache_available() -> bool:
@@ -571,9 +571,8 @@ def get_requirements() -> List[str]:
571
cuda_major, cuda_minor = torch.version.cuda.split(".")
572
modified_requirements = []
573
for req in requirements:
574
- if ("vllm-flash-attn" in req
575
- and not (cuda_major == "12" and cuda_minor == "1")):
576
- # vllm-flash-attn is built only for CUDA 12.1.
574
+ if ("vllm-flash-attn" in req and cuda_major != "12"):
575
+ # vllm-flash-attn is built only for CUDA 12.x.
576
# Skip for other versions.
577
continue
578
modified_requirements.append(req)