A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[torch.compile] add warning for unsupported models (#10622)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 24, 2024 at 23:27 UTC
65813781a2e2e76d18741601afe66b870a90a717
3 files changed
+18
vllm/compilation/counter.py
+1
@@ -5,6 +5,7 @@ from contextlib import contextmanager
5
6
@dataclasses.dataclass
7
class CompilationCounter:
8
+ num_models_seen: int = 0
9
num_graphs_seen: int = 0
10
# including the splitting ops
11
num_piecewise_graphs_seen: int = 0
vllm/compilation/decorators.py
+2
@@ -3,6 +3,7 @@ from typing import Dict, List, Optional, Union
3
4
import torch
5
6
+from vllm.compilation.counter import compilation_counter
7
from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher
8
from vllm.config import CompilationLevel, VllmConfig
9
from vllm.logger import init_logger
@@ -130,6 +131,7 @@ def _support_torch_compile(cls: type,
131
] or not supports_dynamo()
132
if self.do_not_compile:
133
return
134
+ compilation_counter.num_models_seen += 1
135
TorchCompileWrapperWithCustomDispatcher.__init__(
136
self, compilation_level=vllm_config.compilation_config.level)
137
vllm/plugins/__init__.py
+15
@@ -80,6 +80,9 @@ def set_current_vllm_config(vllm_config: "VllmConfig"):
80
"""
81
global _current_vllm_config
82
old_vllm_config = _current_vllm_config
83
+ from vllm.compilation.counter import compilation_counter
84
+ from vllm.config import CompilationLevel
85
+ num_models_seen = compilation_counter.num_models_seen
86
try:
87
_current_vllm_config = vllm_config
88
yield
@@ -88,6 +91,18 @@ def set_current_vllm_config(vllm_config: "VllmConfig"):
91
vllm_config.compilation_config.enabled_custom_ops)
92
logger.debug("disabled custom ops: %s",
93
vllm_config.compilation_config.disabled_custom_ops)
94
+ if vllm_config.compilation_config.level == CompilationLevel.PIECEWISE \
95
+ and compilation_counter.num_models_seen == num_models_seen:
96
+ # If the model supports compilation,
97
+ # compilation_counter.num_models_seen should be increased
98
+ # by at least 1.
99
+ # If it is not increased, it means the model does not support
100
+ # compilation (does not have @support_torch_compile decorator).
101
+ logger.warning(
102
+ "`torch.compile` is turned on, but the model %s"
103
+ " does not support it. Please open an issue on GitHub"
104
+ "if you want it to be supported.",
105
+ vllm_config.model_config.model)
106
_current_vllm_config = old_vllm_config
107
108