@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / becb7de40b

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

Update PyTorch to 2.9.0+cu129 (#24994)

Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com>

Huy Do committed Oct 21, 2025 at 14:20 UTC becb7de40b293c840a930c008a648a6177666da9

16 files changed +68 -67

.buildkite/test-pipeline.yaml

+7 -2

@@ -172,6 +172,8 @@ steps:

172 - tests/v1/engine/test_engine_core_client.py

173 - tests/distributed/test_symm_mem_allreduce.py

174 commands:

175 + # https://github.com/NVIDIA/nccl/issues/1838

176 + - export NCCL_CUMEM_HOST_ENABLE=0

177 # test with torchrun tp=2 and external_dp=2

178 - torchrun --nproc-per-node=4 distributed/test_torchrun_example.py

179 # test with torchrun tp=2 and pp=2

@@ -349,7 +351,8 @@ steps:

351 - python3 offline_inference/basic/embed.py

352 - python3 offline_inference/basic/score.py

353 - python3 offline_inference/spec_decode.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048

352 - - python3 offline_inference/spec_decode.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048

354 + # https://github.com/vllm-project/vllm/pull/26682 uses slightly more memory in PyTorch 2.9+ causing this test to OOM in 1xL4 GPU

355 + - python3 offline_inference/spec_decode.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536

356

357 - label: Platform Tests (CUDA) # 4min

358 timeout_in_minutes: 15

@@ -534,7 +537,7 @@ steps:

537 # https://github.com/pytorch/ao/issues/2919, we'll have to skip new torchao tests for now

538 # we can only upgrade after this is resolved

539 # TODO(jerryzh168): resolve the above comment

537 - - uv pip install --system torchao==0.13.0

540 + - uv pip install --system torchao==0.13.0 --index-url https://download.pytorch.org/whl/cu129

541 - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py

542

543 - label: LM Eval Small Models # 53min

@@ -975,6 +978,8 @@ steps:

978 - tests/v1/shutdown

979 - tests/v1/worker/test_worker_memory_snapshot.py

980 commands:

981 + # https://github.com/NVIDIA/nccl/issues/1838

982 + - export NCCL_CUMEM_HOST_ENABLE=0

983 - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py

984 - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py

985 - DP_SIZE=2 pytest -v -s v1/entrypoints/openai/test_multi_api_servers.py

.pre-commit-config.yaml

+1 -1

@@ -38,7 +38,7 @@ repos:

38 rev: 0.9.1

39 hooks:

40 - id: pip-compile

41 - args: [requirements/test.in, -o, requirements/test.txt, --index-strategy, unsafe-best-match, --torch-backend, cu128, --python-platform, x86_64-manylinux_2_28]

41 + args: [requirements/test.in, -o, requirements/test.txt, --index-strategy, unsafe-best-match, --torch-backend, cu129, --python-platform, x86_64-manylinux_2_28]

42 files: ^requirements/test\.(in|txt)$

43 - repo: local

44 hooks:

CMakeLists.txt

+2 -2

@@ -49,8 +49,8 @@ set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx942;gfx950;gfx1030;gfx1100;gfx1

49 # requirements.txt files and should be kept consistent. The ROCm torch

50 # versions are derived from docker/Dockerfile.rocm

51 #

52 -set(TORCH_SUPPORTED_VERSION_CUDA "2.8.0")

53 -set(TORCH_SUPPORTED_VERSION_ROCM "2.8.0")

52 +set(TORCH_SUPPORTED_VERSION_CUDA "2.9.0")

53 +set(TORCH_SUPPORTED_VERSION_ROCM "2.9.0")

55 #

56 # Try to find python package with an executable that exactly matches

docker/Dockerfile

+12 -2

@@ -5,7 +5,7 @@

5 # docs/contributing/dockerfile/dockerfile.md and

6 # docs/assets/contributing/dockerfile-stages-dependency.png

8 -ARG CUDA_VERSION=12.8.1

8 +ARG CUDA_VERSION=12.9.1

9 ARG PYTHON_VERSION=3.12

11 # By parameterizing the base images, we allow third-party to use their own

@@ -275,6 +275,7 @@ WORKDIR /vllm-workspace

275 ENV DEBIAN_FRONTEND=noninteractive

276 ARG TARGETPLATFORM

277

278 +# TODO (huydhn): There is no prebuilt gdrcopy package on 12.9 at the moment

279 ARG GDRCOPY_CUDA_VERSION=12.8

280 # Keep in line with FINAL_BASE_IMAGE

281 ARG GDRCOPY_OS_VERSION=Ubuntu22_04

@@ -360,6 +361,13 @@ RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist

361 && uv pip install --system dist/*.whl --verbose \

362 --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')

363

364 +# TODO (huydhn): Remove this once xformers is released for 2.9.0

365 +RUN --mount=type=cache,target=/root/.cache/uv bash - <<'BASH'

366 + . /etc/environment

367 + export TORCH_CUDA_ARCH_LIST='7.5 8.0+PTX 9.0a'

368 + uv pip install --system --no-build-isolation "git+https://github.com/facebookresearch/xformers@v0.0.32.post2"

369 +BASH

370 +

371 # Install FlashInfer pre-compiled kernel cache and binaries

372 # https://docs.flashinfer.ai/installation.html

373 RUN --mount=type=cache,target=/root/.cache/uv \

@@ -426,6 +434,7 @@ ARG PYTHON_VERSION

434

435 ARG PIP_INDEX_URL UV_INDEX_URL

436 ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL

437 +ARG PYTORCH_CUDA_INDEX_BASE_URL

438

439 # This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out

440 # Reference: https://github.com/astral-sh/uv/pull/1694

@@ -438,7 +447,8 @@ ENV UV_LINK_MODE=copy

447 RUN --mount=type=cache,target=/root/.cache/uv \

448 CUDA_MAJOR="${CUDA_VERSION%%.*}"; \

449 if [ "$CUDA_MAJOR" -ge 12 ]; then \

441 - uv pip install --system -r requirements/dev.txt; \

450 + uv pip install --system -r requirements/dev.txt \

451 + --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \

452 fi

453

454 # install development dependencies (for testing)

docker/Dockerfile.cpu

@@ -199,9 +199,13 @@ FROM base AS vllm-test-deps

199

200 WORKDIR /workspace/vllm

201

202 +# TODO: Update to 2.9.0 when there is a new build for intel_extension_for_pytorch for that version

203 RUN --mount=type=bind,src=requirements/test.in,target=requirements/test.in \

204 cp requirements/test.in requirements/cpu-test.in && \

205 sed -i '/mamba_ssm/d' requirements/cpu-test.in && \

206 + sed -i 's/^torch==.*/torch==2.8.0/g' requirements/cpu-test.in && \

207 + sed -i 's/torchaudio.*/torchaudio/g' requirements/cpu-test.in && \

208 + sed -i 's/torchvision.*/torchvision/g' requirements/cpu-test.in && \

209 uv pip compile requirements/cpu-test.in -o requirements/cpu-test.txt --index-strategy unsafe-best-match --torch-backend cpu

210

211 RUN --mount=type=cache,target=/root/.cache/uv \

docs/assets/contributing/dockerfile-stages-dependency.png

Binary files a/docs/assets/contributing/dockerfile-stages-dependency.png and b/docs/assets/contributing/dockerfile-stages-dependency.png differ

docs/contributing/ci/update_pytorch_version.md

+4 -22

@@ -87,7 +87,7 @@ is ineffective.

88 While ongoing efforts like <https://github.com/vllm-project/vllm/issues/17419>

89 address the long build time at its source, the current workaround is to set `VLLM_CI_BRANCH`

90 -to a custom branch provided by @khluu (`VLLM_CI_BRANCH=khluu/use_postmerge_q`)

90 +to a custom branch provided by @khluu (`VLLM_CI_BRANCH=khluu/long_build`)

91 when manually triggering a build on Buildkite. This branch accomplishes two things:

93 1. Increase the timeout limit to 10 hours so that the build doesn't time out.

@@ -100,35 +100,17 @@ to warm it up so that future builds are faster.

100

101 ## Update dependencies

102

103 -Several vLLM dependencies, such as FlashInfer, also depend on PyTorch and need

103 +Several vLLM dependencies like xFormers depend on PyTorch and need

104 to be updated accordingly. Rather than waiting for all of them to publish new

105 releases (which would take too much time), they can be built from

106 source to unblock the update process.

107

108 -### FlashInfer

109 -

110 -Here is how to build and install it from source with `torch2.7.0+cu128` in vLLM [Dockerfile](https://github.com/vllm-project/vllm/blob/27bebcd89792d5c4b08af7a65095759526f2f9e1/docker/Dockerfile#L259-L271):

111 -

112 -```bash

113 -export TORCH_CUDA_ARCH_LIST='7.5 8.0 8.9 9.0 10.0+PTX'

114 -export FLASHINFER_ENABLE_SM90=1

115 -uv pip install --system \

116 - --no-build-isolation "git+https://github.com/flashinfer-ai/flashinfer@v0.2.6.post1"

117 -```

118 -

119 -One caveat is that building FlashInfer from source adds approximately 30

120 -minutes to the vLLM build time. Therefore, it's preferable to cache the wheel in a

121 -public location for immediate installation, such as [this FlashInfer wheel link](https://download.pytorch.org/whl/cu128/flashinfer/flashinfer_python-0.2.6.post1%2Bcu128torch2.7-cp39-abi3-linux_x86_64.whl). For future releases, contact the PyTorch release

122 -team if you want to get the package published there.

123 -

108 ### xFormers

109

126 -Similar to FlashInfer, here is how to build and install xFormers from source:

127 -

110 ```bash

129 -export TORCH_CUDA_ARCH_LIST='7.0 7.5 8.0 8.9 9.0 10.0+PTX'

111 +export TORCH_CUDA_ARCH_LIST='7.5 8.0+PTX 9.0a'

112 MAX_JOBS=16 uv pip install --system \

131 - --no-build-isolation "git+https://github.com/facebookresearch/xformers@v0.0.30"

113 + --no-build-isolation "git+https://github.com/facebookresearch/xformers@v0.0.32.post2"

114 ```

115

116 ## Update all the different vLLM platforms

pyproject.toml

+1 -1

@@ -6,7 +6,7 @@ requires = [

6 "packaging>=24.2",

7 "setuptools>=77.0.3,<80.0.0",

8 "setuptools-scm>=8.0",

9 - "torch == 2.8.0",

9 + "torch == 2.9.0",

10 "wheel",

11 "jinja2",

12 ]

requirements/build.txt

+1 -1

@@ -4,7 +4,7 @@ ninja

4 packaging>=24.2

5 setuptools>=77.0.3,<80.0.0

6 setuptools-scm>=8

7 -torch==2.8.0

7 +torch==2.9.0

8 wheel

9 jinja2>=3.1.6

10 regex

requirements/cuda.txt

+5 -5

@@ -5,11 +5,11 @@ numba == 0.61.2 # Required for N-gram speculative decoding

6 # Dependencies for NVIDIA GPUs

7 ray[cgraph]>=2.48.0 # Ray Compiled Graph, required for pipeline parallelism in V1.

8 -torch==2.8.0

9 -torchaudio==2.8.0

8 +torch==2.9.0

9 +torchaudio==2.9.0

10 # These must be updated alongside torch

11 -torchvision==0.23.0 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version

11 +torchvision==0.24.0 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version

12 # https://github.com/facebookresearch/xformers/releases/tag/v0.0.32.post1

13 -xformers==0.0.32.post1; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch >= 2.8

13 +# xformers==0.0.32.post1; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch >= 2.8

14 # FlashInfer should be updated together with the Dockerfile

15 -flashinfer-python==0.4.1

\ No newline at end of file

15 +flashinfer-python==0.4.1

requirements/rocm-build.txt

+5 -5

@@ -1,12 +1,12 @@

1 # Common dependencies

2 -r common.txt

4 ---extra-index-url https://download.pytorch.org/whl/rocm6.3

5 -torch==2.8.0

6 -torchvision==0.23.0

7 -torchaudio==2.8.0

4 +--extra-index-url https://download.pytorch.org/whl/rocm6.4

5 +torch==2.9.0

6 +torchvision==0.24.0

7 +torchaudio==2.9.0

9 -triton==3.3.0

9 +triton==3.5.0

10 cmake>=3.26.1,<4

11 packaging>=24.2

12 setuptools>=77.0.3,<80.0.0

requirements/test.in

+4 -4

@@ -24,9 +24,9 @@ soundfile # required for audio tests

24 jiwer # required for audio tests

25 tblib # for pickling test exceptions

26 timm >=1.0.17 # required for internvl and gemma3n-mm test

27 -torch==2.8.0

28 -torchaudio==2.8.0

29 -torchvision==0.23.0

27 +torch==2.9.0

28 +torchaudio==2.9.0

29 +torchvision==0.24.0

30 transformers_stream_generator # required for qwen-vl test

31 matplotlib # required for qwen-vl test

32 mistral_common[image,audio] >= 1.8.5 # required for voxtral test

@@ -55,4 +55,4 @@ fastsafetensors>=0.1.10

55 pydantic>=2.12 # 2.11 leads to error on python 3.13

56 decord==0.6.0

57 terratorch @ git+https://github.com/IBM/terratorch.git@1.1.rc3 # required for PrithviMAE test

58 -gpt-oss >= 0.0.7; python_version > '3.11'

\ No newline at end of file

58 +gpt-oss >= 0.0.7; python_version > '3.11'

requirements/test.txt

+19 -18

@@ -1,5 +1,5 @@

1 # This file was autogenerated by uv via the following command:

2 -# uv pip compile requirements/test.in -o requirements/test.txt --index-strategy unsafe-best-match --torch-backend cu128 --python-platform x86_64-manylinux_2_28

2 +# uv pip compile requirements/test.in -o requirements/test.txt --index-strategy unsafe-best-match --torch-backend cu129 --python-platform x86_64-manylinux_2_28

3 absl-py==2.1.0

4 # via rouge-score

5 accelerate==1.0.1

@@ -573,42 +573,44 @@ numpy==1.26.4

573 # tritonclient

574 # vocos

575 # xarray

576 -nvidia-cublas-cu12==12.8.4.1

576 +nvidia-cublas-cu12==12.9.1.4

577 # via

578 # nvidia-cudnn-cu12

579 # nvidia-cusolver-cu12

580 # torch

581 -nvidia-cuda-cupti-cu12==12.8.90

581 +nvidia-cuda-cupti-cu12==12.9.79

582 # via torch

583 -nvidia-cuda-nvrtc-cu12==12.8.93

583 +nvidia-cuda-nvrtc-cu12==12.9.86

584 # via torch

585 -nvidia-cuda-runtime-cu12==12.8.90

585 +nvidia-cuda-runtime-cu12==12.9.79

586 # via torch

587 nvidia-cudnn-cu12==9.10.2.21

588 # via torch

589 -nvidia-cufft-cu12==11.3.3.83

589 +nvidia-cufft-cu12==11.4.1.4

590 # via torch

591 -nvidia-cufile-cu12==1.13.1.3

591 +nvidia-cufile-cu12==1.14.1.1

592 # via torch

593 -nvidia-curand-cu12==10.3.9.90

593 +nvidia-curand-cu12==10.3.10.19

594 # via torch

595 -nvidia-cusolver-cu12==11.7.3.90

595 +nvidia-cusolver-cu12==11.7.5.82

596 # via torch

597 -nvidia-cusparse-cu12==12.5.8.93

597 +nvidia-cusparse-cu12==12.5.10.65

598 # via

599 # nvidia-cusolver-cu12

600 # torch

601 nvidia-cusparselt-cu12==0.7.1

602 # via torch

603 -nvidia-nccl-cu12==2.27.3

603 +nvidia-nccl-cu12==2.27.5

604 # via torch

605 -nvidia-nvjitlink-cu12==12.8.93

605 +nvidia-nvjitlink-cu12==12.9.86

606 # via

607 # nvidia-cufft-cu12

608 # nvidia-cusolver-cu12

609 # nvidia-cusparse-cu12

610 # torch

611 -nvidia-nvtx-cu12==12.8.90

611 +nvidia-nvshmem-cu12==3.3.20

612 + # via torch

613 +nvidia-nvtx-cu12==12.9.79

614 # via torch

615 omegaconf==2.3.0

616 # via

@@ -1017,7 +1019,6 @@ setuptools==77.0.3

1019 # lightning-utilities

1020 # pytablewriter

1021 # torch

1020 - # triton

1022 shapely==2.1.1

1023 # via

1024 # geopandas

@@ -1122,7 +1123,7 @@ tomli==2.2.1

1123 # via schemathesis

1124 tomli-w==1.2.0

1125 # via schemathesis

1125 -torch==2.8.0+cu128

1126 +torch==2.9.0+cu129

1127 # via

1128 # -r requirements/test.in

1129 # accelerate

@@ -1151,7 +1152,7 @@ torch==2.8.0+cu128

1152 # torchvision

1153 # vector-quantize-pytorch

1154 # vocos

1154 -torchaudio==2.8.0+cu128

1155 +torchaudio==2.9.0+cu129

1156 # via

1157 # -r requirements/test.in

1158 # encodec

@@ -1164,7 +1165,7 @@ torchmetrics==1.7.4

1165 # pytorch-lightning

1166 # terratorch

1167 # torchgeo

1167 -torchvision==0.23.0+cu128

1168 +torchvision==0.24.0+cu129

1169 # via

1170 # -r requirements/test.in

1171 # lightly

@@ -1205,7 +1206,7 @@ transformers==4.56.2

1206 # transformers-stream-generator

1207 transformers-stream-generator==0.0.5

1208 # via -r requirements/test.in

1208 -triton==3.4.0

1209 +triton==3.5.0

1210 # via torch

1211 tritonclient==2.51.0

1212 # via

tests/lora/test_chatglm3_tp.py

+1 -1

@@ -109,7 +109,7 @@ def test_chatglm3_lora_tp4_fully_sharded_loras(chatglm3_lora_files):

109 tensor_parallel_size=4,

110 trust_remote_code=True,

111 fully_sharded_loras=True,

112 - gpu_memory_utilization=0.85,

112 + gpu_memory_utilization=0.8,

113 compilation_config=vllm.config.CompilationConfig( # Avoid OOM

114 cudagraph_specialize_lora=False,

115 ),

tests/v1/e2e/test_async_sched_and_preempt.py

@@ -3,6 +3,7 @@

3 from typing import Any

5 import pytest

6 +import torch._dynamo.config as dynamo_config

8 from vllm import SamplingParams

@@ -12,6 +13,7 @@ from ...models.utils import check_outputs_equal

13 MODEL = "Qwen/Qwen3-0.6B"

16 +@dynamo_config.patch(cache_size_limit=16)

17 def test_preempt_and_async_scheduling_e2e(monkeypatch: pytest.MonkeyPatch):

18 """Test consistency of combos of async scheduling, preemption,

19 uni/multiproc executor, and various sampling parameters."""

tools/install_gdrcopy.sh

-3

@@ -7,18 +7,15 @@ set -euo pipefail

7 # Requires: curl, apt-get, root privileges

8 if [[ $(id -u) -ne 0 ]]; then

9 echo "Must be run as root" >&2

10 -

10 exit 1

11 fi

12 if [[ $# -ne 3 ]]; then

13 echo "Usage: $0 <GDRCOPY_OS_VERSION> <GDRCOPY_CUDA_VERSION> <uuarch(x64|aarch64)>" >&2

14 exit 1

15 fi

17 -

16 OS_VER="$1"

17 CUDA_VER="$2"

18 UUARCH_RAW="$3"

21 -

19 # Normalize/validate arch

20 case "${UUARCH_RAW,,}" in

21 aarch64|arm64)