A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Update PyTorch to 2.8.0 (#20358)
Signed-off-by: Huy Do <huydhn@gmail.com> Co-authored-by: Michael Goin <mgoin64@gmail.com>
Huy Do committed
Aug 29, 2025 at 03:57 UTC
67c14906aaa480d4fee2606f31c784ae21f8a633
12 files changed
+50
-45
.buildkite/test-pipeline.yaml
+2
-2
@@ -462,8 +462,8 @@ steps:
462
- tests/quantization
463
commands:
464
# temporary install here since we need nightly, will move to requirements/test.in
465
- # after torchao 0.12 release
466
- - pip install --pre torchao --index-url https://download.pytorch.org/whl/nightly/cu126
465
+ # after torchao 0.12 release, and pin a working version of torchao nightly here
466
+ - pip install --pre torchao==0.13.0.dev20250814 --index-url https://download.pytorch.org/whl/nightly/cu128
467
- VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization
468
469
- label: LM Eval Small Models # 53min
CMakeLists.txt
+2
-2
@@ -45,8 +45,8 @@ set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx942;gfx950;gfx1030;gfx1100;gfx1
45
# requirements.txt files and should be kept consistent. The ROCm torch
46
# versions are derived from docker/Dockerfile.rocm
47
#
48
-set(TORCH_SUPPORTED_VERSION_CUDA "2.7.1")
49
-set(TORCH_SUPPORTED_VERSION_ROCM "2.7.0")
48
+set(TORCH_SUPPORTED_VERSION_CUDA "2.8.0")
49
+set(TORCH_SUPPORTED_VERSION_ROCM "2.8.0")
50
51
#
52
# Try to find python package with an executable that exactly matches
pyproject.toml
+1
-1
@@ -6,7 +6,7 @@ requires = [
6
"packaging>=24.2",
7
"setuptools>=77.0.3,<80.0.0",
8
"setuptools-scm>=8.0",
9
- "torch == 2.7.1",
9
+ "torch == 2.8.0",
10
"wheel",
11
"jinja2",
12
]
requirements/build.txt
+2
-1
@@ -4,7 +4,8 @@ ninja
4
packaging>=24.2
5
setuptools>=77.0.3,<80.0.0
6
setuptools-scm>=8
7
-torch==2.7.1
7
+torch==2.8.0
8
wheel
9
jinja2>=3.1.6
10
regex
11
+build
requirements/cpu.txt
+4
-5
@@ -9,17 +9,16 @@ packaging>=24.2
9
setuptools>=77.0.3,<80.0.0
10
--extra-index-url https://download.pytorch.org/whl/cpu
11
torch==2.6.0+cpu; platform_machine == "x86_64" # torch>2.6.0+cpu has performance regression on x86 platform, see https://github.com/pytorch/pytorch/pull/151218
12
-torch==2.7.0; platform_system == "Darwin"
13
-torch==2.7.0; platform_machine == "ppc64le"
14
-torch==2.6.0; platform_machine == "aarch64" # for arm64 CPUs, torch 2.7.0 has a issue: https://github.com/vllm-project/vllm/issues/17960
12
+torch==2.8.0; platform_system == "Darwin"
13
+torch==2.8.0; platform_machine == "ppc64le" or platform_machine == "aarch64"
14
15
# required for the image processor of minicpm-o-2_6, this must be updated alongside torch
16
torchaudio; platform_machine != "ppc64le" and platform_machine != "s390x"
18
-torchaudio==2.7.0; platform_machine == "ppc64le"
17
+torchaudio==2.8.0; platform_machine == "ppc64le"
18
19
# required for the image processor of phi3v, this must be updated alongside torch
20
torchvision; platform_machine != "ppc64le" and platform_machine != "s390x"
22
-torchvision==0.22.0; platform_machine == "ppc64le"
21
+torchvision==0.23.0; platform_machine == "ppc64le"
22
datasets # for benchmark scripts
23
24
# Intel Extension for PyTorch, only for x86_64 CPUs
requirements/cuda.txt
+5
-5
@@ -6,9 +6,9 @@ numba == 0.61.2; python_version > '3.9'
6
7
# Dependencies for NVIDIA GPUs
8
ray[cgraph]>=2.48.0 # Ray Compiled Graph, required for pipeline parallelism in V1.
9
-torch==2.7.1
10
-torchaudio==2.7.1
9
+torch==2.8.0
10
+torchaudio==2.8.0
11
# These must be updated alongside torch
12
-torchvision==0.22.1 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
13
-# https://github.com/facebookresearch/xformers/releases/tag/v0.0.31
14
-xformers==0.0.31; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch >= 2.7
\ No newline at end of file
12
+torchvision==0.23.0 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
13
+# https://github.com/facebookresearch/xformers/releases/tag/v0.0.32.post1
14
+xformers==0.0.32.post1; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch >= 2.8
requirements/rocm-build.txt
+4
-4
@@ -1,10 +1,10 @@
1
# Common dependencies
2
-r common.txt
3
4
---extra-index-url https://download.pytorch.org/whl/rocm6.2.4
5
-torch==2.7.0
6
-torchvision==0.22.0
7
-torchaudio==2.7.0
4
+--extra-index-url https://download.pytorch.org/whl/rocm6.3
5
+torch==2.8.0
6
+torchvision==0.23.0
7
+torchaudio==2.8.0
8
9
triton==3.3.0
10
cmake>=3.26.1,<4
requirements/test.in
+3
-3
@@ -22,9 +22,9 @@ sentence-transformers # required for embedding tests
22
soundfile # required for audio tests
23
jiwer # required for audio tests
24
timm >=1.0.17 # required for internvl and gemma3n-mm test
25
-torch==2.7.1
26
-torchaudio==2.7.1
27
-torchvision==0.22.1
25
+torch==2.8.0
26
+torchaudio==2.8.0
27
+torchvision==0.23.0
28
transformers_stream_generator # required for qwen-vl test
29
matplotlib # required for qwen-vl test
30
mistral_common[image,audio] >= 1.8.2 # required for voxtral test
requirements/test.txt
+18
-18
@@ -541,42 +541,42 @@ numpy==1.26.4
541
# tritonclient
542
# vocos
543
# xarray
544
-nvidia-cublas-cu12==12.8.3.14
544
+nvidia-cublas-cu12==12.8.4.1
545
# via
546
# nvidia-cudnn-cu12
547
# nvidia-cusolver-cu12
548
# torch
549
-nvidia-cuda-cupti-cu12==12.8.57
549
+nvidia-cuda-cupti-cu12==12.8.90
550
# via torch
551
-nvidia-cuda-nvrtc-cu12==12.8.61
551
+nvidia-cuda-nvrtc-cu12==12.8.93
552
# via torch
553
-nvidia-cuda-runtime-cu12==12.8.57
553
+nvidia-cuda-runtime-cu12==12.8.90
554
# via torch
555
-nvidia-cudnn-cu12==9.7.1.26
555
+nvidia-cudnn-cu12==9.10.2.21
556
# via torch
557
-nvidia-cufft-cu12==11.3.3.41
557
+nvidia-cufft-cu12==11.3.3.83
558
# via torch
559
-nvidia-cufile-cu12==1.13.0.11
559
+nvidia-cufile-cu12==1.13.1.3
560
# via torch
561
-nvidia-curand-cu12==10.3.9.55
561
+nvidia-curand-cu12==10.3.9.90
562
# via torch
563
-nvidia-cusolver-cu12==11.7.2.55
563
+nvidia-cusolver-cu12==11.7.3.90
564
# via torch
565
-nvidia-cusparse-cu12==12.5.7.53
565
+nvidia-cusparse-cu12==12.5.8.93
566
# via
567
# nvidia-cusolver-cu12
568
# torch
569
-nvidia-cusparselt-cu12==0.6.3
569
+nvidia-cusparselt-cu12==0.7.1
570
# via torch
571
-nvidia-nccl-cu12==2.26.2
571
+nvidia-nccl-cu12==2.27.3
572
# via torch
573
-nvidia-nvjitlink-cu12==12.8.61
573
+nvidia-nvjitlink-cu12==12.8.93
574
# via
575
# nvidia-cufft-cu12
576
# nvidia-cusolver-cu12
577
# nvidia-cusparse-cu12
578
# torch
579
-nvidia-nvtx-cu12==12.8.55
579
+nvidia-nvtx-cu12==12.8.90
580
# via torch
581
omegaconf==2.3.0
582
# via
@@ -1069,7 +1069,7 @@ tomli==2.2.1
1069
# via schemathesis
1070
tomli-w==1.2.0
1071
# via schemathesis
1072
-torch==2.7.1+cu128
1072
+torch==2.8.0+cu128
1073
# via
1074
# -r requirements/test.in
1075
# accelerate
@@ -1098,7 +1098,7 @@ torch==2.7.1+cu128
1098
# torchvision
1099
# vector-quantize-pytorch
1100
# vocos
1101
-torchaudio==2.7.1+cu128
1101
+torchaudio==2.8.0+cu128
1102
# via
1103
# -r requirements/test.in
1104
# encodec
@@ -1111,7 +1111,7 @@ torchmetrics==1.7.4
1111
# pytorch-lightning
1112
# terratorch
1113
# torchgeo
1114
-torchvision==0.22.1+cu128
1114
+torchvision==0.23.0+cu128
1115
# via
1116
# -r requirements/test.in
1117
# lightly
@@ -1152,7 +1152,7 @@ transformers==4.55.2
1152
# transformers-stream-generator
1153
transformers-stream-generator==0.0.5
1154
# via -r requirements/test.in
1155
-triton==3.3.1
1155
+triton==3.4.0
1156
# via torch
1157
tritonclient==2.51.0
1158
# via
tests/distributed/test_sequence_parallel.py
+1
-1
@@ -292,7 +292,7 @@ SP_TEST_MODELS = [
292
# TODO support other models
293
# [LANGUAGE GENERATION]
294
"meta-llama/Llama-3.2-1B-Instruct",
295
- "RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8"
295
+ "RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8",
296
]
297
298
tests/lora/test_chatglm3_tp.py
+5
-1
@@ -87,6 +87,9 @@ def test_chatglm3_lora_tp4(chatglm3_lora_files):
87
@multi_gpu_test(num_gpus=4)
88
@create_new_process_for_each_test()
89
def test_chatglm3_lora_tp4_fully_sharded_loras(chatglm3_lora_files):
90
+ # https://github.com/NVIDIA/nccl/issues/1790, set a lower value for
91
+ # gpu_memory_utilization here because NCCL >= 2.26.3 seems to use
92
+ # more GPU memory causing vLLM to OOM
93
llm = vllm.LLM(MODEL_PATH,
94
max_model_len=1024,
95
enable_lora=True,
@@ -95,7 +98,8 @@ def test_chatglm3_lora_tp4_fully_sharded_loras(chatglm3_lora_files):
98
tensor_parallel_size=4,
99
trust_remote_code=True,
100
fully_sharded_loras=True,
98
- enable_chunked_prefill=True)
101
+ enable_chunked_prefill=True,
102
+ gpu_memory_utilization=0.85)
103
output1 = do_sample(llm, chatglm3_lora_files, lora_id=1)
104
for i in range(len(EXPECTED_LORA_OUTPUT)):
105
assert output1[i] == EXPECTED_LORA_OUTPUT[i]
vllm/v1/attention/backends/flex_attention.py
+3
-2
@@ -789,6 +789,7 @@ def get_kernel_options(query, block_m, block_n,
789
device_props = torch.cuda.get_device_properties()
790
max_shared_memory = device_props.shared_memory_per_block_optin
791
if max_shared_memory < 144 * 1024:
792
- kernel_options["BLOCK_M"] = 32
793
- kernel_options["BLOCK_N"] = 32
792
+ kernel_options["BLOCK_M"] = kernel_options["BLOCK_M"] // 2
793
+ kernel_options["BLOCK_N"] = kernel_options["BLOCK_N"] // 2
794
+
795
return kernel_options