A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
Update to torch==2.6.0 (#12721)
Signed-off-by: mgoin <michael@neuralmagic.com> Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: luka <luka@neuralmagic.com> Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Michael Goin committed
Mar 14, 2025 at 16:58 UTC
14f301b541ffe95187ff90938c7f94f642bc5bfa
9 files changed
+43
-23
CMakeLists.txt
+2
-2
@@ -46,8 +46,8 @@ set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx942;gfx1030;gfx1100;gfx1101")
46
# requirements.txt files and should be kept consistent. The ROCm torch
47
# versions are derived from Dockerfile.rocm
48
#
49
-set(TORCH_SUPPORTED_VERSION_CUDA "2.5.1")
50
-set(TORCH_SUPPORTED_VERSION_ROCM "2.5.1")
49
+set(TORCH_SUPPORTED_VERSION_CUDA "2.6.0")
50
+set(TORCH_SUPPORTED_VERSION_ROCM "2.6.0")
51
52
#
53
# Try to find python package with an executable that exactly matches
Dockerfile
+1
-1
@@ -222,7 +222,7 @@ RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist
222
223
RUN --mount=type=cache,target=/root/.cache/uv \
224
if [ "$TARGETPLATFORM" != "linux/arm64" ]; then \
225
- uv pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.2.1.post1/flashinfer_python-0.2.1.post1+cu124torch2.5-cp38-abi3-linux_x86_64.whl ; \
225
+ uv pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.2.1.post2/flashinfer_python-0.2.1.post2+cu124torch2.6-cp38-abi3-linux_x86_64.whl ; \
226
fi
227
COPY examples examples
228
pyproject.toml
+1
-1
@@ -6,7 +6,7 @@ requires = [
6
"packaging",
7
"setuptools>=61",
8
"setuptools-scm>=8.0",
9
- "torch == 2.5.1",
9
+ "torch == 2.6.0",
10
"wheel",
11
"jinja2",
12
]
requirements/build.txt
+1
-1
@@ -4,6 +4,6 @@ ninja
4
packaging
5
setuptools>=61
6
setuptools-scm>=8
7
-torch==2.5.1
7
+torch==2.6.0
8
wheel
9
jinja2
requirements/cuda.txt
+5
-5
@@ -4,9 +4,9 @@
4
numba == 0.60.0 # v0.61 doesn't support Python 3.9. Required for N-gram speculative decoding
5
6
# Dependencies for NVIDIA GPUs
7
-ray[cgraph] >= 2.43.0 # Ray Compiled Graph, required for pipeline parallelism in V1.
8
-torch == 2.5.1
9
-torchaudio==2.5.1
7
+ray[cgraph]>=2.43.0 # Ray Compiled Graph, required for pipeline parallelism in V1.
8
+torch==2.6.0
9
+torchaudio==2.6.0
10
# These must be updated alongside torch
11
-torchvision == 0.20.1 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
12
-xformers == 0.0.28.post3; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch 2.5.1
11
+torchvision==0.21.0 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
12
+xformers==0.0.29.post2; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch 2.6.0
requirements/test.in
+4
-3
@@ -21,8 +21,9 @@ sentence-transformers # required for embedding tests
21
soundfile # required for audio tests
22
jiwer # required for audio tests
23
timm # required for internvl test
24
-torch==2.5.1
25
-torchaudio==2.5.1
24
+torch==2.6.0
25
+torchaudio==2.6.0
26
+torchvision==0.21.0
27
transformers_stream_generator # required for qwen-vl test
28
matplotlib # required for qwen-vl test
29
mistral_common[opencv] >= 1.5.0 # required for pixtral test
@@ -30,7 +31,7 @@ datamodel_code_generator # required for minicpm3 test
31
lm-eval[api]==0.4.4 # required for model evaluation test
32
transformers==4.48.2
33
# quantization
33
-bitsandbytes>=0.45.0
34
+bitsandbytes>=0.45.3
35
buildkite-test-collector==0.1.9
36
37
genai_perf==0.0.8
requirements/test.txt
+10
-8
@@ -33,7 +33,7 @@ audioread==3.0.1
33
# via librosa
34
awscli==1.35.23
35
# via -r requirements/test.in
36
-bitsandbytes==0.45.0
36
+bitsandbytes==0.45.3
37
# via -r requirements/test.in
38
black==24.10.0
39
# via datamodel-code-generator
@@ -127,7 +127,6 @@ filelock==3.16.1
127
# ray
128
# torch
129
# transformers
130
- # triton
130
fonttools==4.54.1
131
# via matplotlib
132
frozendict==2.4.6
@@ -320,6 +319,8 @@ nvidia-cusparse-cu12==12.3.1.170
319
# via
320
# nvidia-cusolver-cu12
321
# torch
322
+nvidia-cusparselt-cu12==0.6.2
323
+ # via torch
324
nvidia-nccl-cu12==2.21.5
325
# via torch
326
nvidia-nvjitlink-cu12==12.4.127
@@ -591,7 +592,7 @@ timm==1.0.11
592
# via -r requirements/test.in
593
tokenizers==0.21.0
594
# via transformers
594
-torch==2.5.1
595
+torch==2.6.0
596
# via
597
# -r requirements/test.in
598
# accelerate
@@ -607,13 +608,15 @@ torch==2.5.1
608
# torchvision
609
# vector-quantize-pytorch
610
# vocos
610
-torchaudio==2.5.1
611
+torchaudio==2.6.0
612
# via
613
# -r requirements/test.in
614
# encodec
615
# vocos
615
-torchvision==0.20.1
616
- # via timm
616
+torchvision==0.21.0
617
+ # via
618
+ # -r requirements/test.in
619
+ # timm
620
tqdm==4.66.6
621
# via
622
# datasets
@@ -638,7 +641,7 @@ transformers==4.48.2
641
# transformers-stream-generator
642
transformers-stream-generator==0.0.5
643
# via -r requirements/test.in
641
-triton==3.1.0
644
+triton==3.2.0
645
# via torch
646
tritonclient==2.51.0
647
# via
@@ -651,7 +654,6 @@ typepy==1.3.2
654
# tabledata
655
typing-extensions==4.12.2
656
# via
654
- # bitsandbytes
657
# huggingface-hub
658
# librosa
659
# mistral-common
tests/compile/backend.py
+4
-2
@@ -6,6 +6,7 @@ from typing import Callable, Union
6
from torch import fx
7
8
from vllm.compilation.inductor_pass import InductorPass
9
+from vllm.config import get_current_vllm_config
10
11
12
class TestBackend:
@@ -17,13 +18,14 @@ class TestBackend:
18
Inductor config can be modified directly by editing the inductor_config
19
property. This can be helpful for adding passes like the
20
'pre_grad_custom_pass' and the 'post_grad_custom_pre_pass'.
21
+ Inductor config is default-initialized from VllmConfig.CompilationConfig.
22
"""
23
24
def __init__(self, *passes: Union[InductorPass, Callable[[fx.Graph],
25
None]]):
26
self.custom_passes = list(passes)
25
- from torch._inductor import config
26
- self.inductor_config = config.shallow_copy_dict()
27
+ compile_config = get_current_vllm_config().compilation_config
28
+ self.inductor_config = compile_config.inductor_compile_config
29
self.inductor_config['force_disable_caches'] = True
30
self.inductor_config['post_grad_custom_post_pass'] = self.post_pass
31
vllm/config.py
+15
@@ -52,6 +52,8 @@ if TYPE_CHECKING:
52
else:
53
QuantizationConfig = None
54
55
+from packaging.version import Version
56
+
57
logger = init_logger(__name__)
58
59
# This value is chosen to have a balance between ITL and TTFT. Note it is
@@ -3126,6 +3128,19 @@ class CompilationConfig(BaseModel):
3128
count_all = self.custom_ops.count("all")
3129
assert count_none + count_all <= 1, "Can only specify 'none' or 'all'"
3130
3131
+ # TODO(zou3519/luka): There are 2 issues with auto-functionalization V2:
3132
+ # 1. A bug in PyTorch, fixed in 2.7:
3133
+ # https://github.com/pytorch/pytorch/issues/147924
3134
+ # 2. Custom passes (fusion) rely on auto-functionalization V1 and don't
3135
+ # work with V2. Addressing this will take extra engineering effort
3136
+ # and it is not yet a priority. RFC here:
3137
+ # https://github.com/vllm-project/vllm/issues/14703
3138
+
3139
+ if Version(torch.__version__) >= Version("2.6"):
3140
+ KEY = 'enable_auto_functionalized_v2'
3141
+ if KEY not in self.inductor_compile_config:
3142
+ self.inductor_compile_config[KEY] = False
3143
+
3144
if self.splitting_ops is None:
3145
if envs.VLLM_USE_V1:
3146
# v1 must split the graph on attention ops