@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 65813781a2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[torch.compile] add warning for unsupported models (#10622)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 24, 2024 at 23:27 UTC 65813781a2e2e76d18741601afe66b870a90a717
3 files changed +18
vllm/compilation/counter.py
+1
@@ -5,6 +5,7 @@ from contextlib import contextmanager
5
6 @dataclasses.dataclass
7 class CompilationCounter:
8 + num_models_seen: int = 0
9 num_graphs_seen: int = 0
10 # including the splitting ops
11 num_piecewise_graphs_seen: int = 0
vllm/compilation/decorators.py
+2
@@ -3,6 +3,7 @@ from typing import Dict, List, Optional, Union
3
4 import torch
5
6 +from vllm.compilation.counter import compilation_counter
7 from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher
8 from vllm.config import CompilationLevel, VllmConfig
9 from vllm.logger import init_logger
@@ -130,6 +131,7 @@ def _support_torch_compile(cls: type,
131 ] or not supports_dynamo()
132 if self.do_not_compile:
133 return
134 + compilation_counter.num_models_seen += 1
135 TorchCompileWrapperWithCustomDispatcher.__init__(
136 self, compilation_level=vllm_config.compilation_config.level)
137
vllm/plugins/__init__.py
+15
@@ -80,6 +80,9 @@ def set_current_vllm_config(vllm_config: "VllmConfig"):
80 """
81 global _current_vllm_config
82 old_vllm_config = _current_vllm_config
83 + from vllm.compilation.counter import compilation_counter
84 + from vllm.config import CompilationLevel
85 + num_models_seen = compilation_counter.num_models_seen
86 try:
87 _current_vllm_config = vllm_config
88 yield
@@ -88,6 +91,18 @@ def set_current_vllm_config(vllm_config: "VllmConfig"):
91 vllm_config.compilation_config.enabled_custom_ops)
92 logger.debug("disabled custom ops: %s",
93 vllm_config.compilation_config.disabled_custom_ops)
94 + if vllm_config.compilation_config.level == CompilationLevel.PIECEWISE \
95 + and compilation_counter.num_models_seen == num_models_seen:
96 + # If the model supports compilation,
97 + # compilation_counter.num_models_seen should be increased
98 + # by at least 1.
99 + # If it is not increased, it means the model does not support
100 + # compilation (does not have @support_torch_compile decorator).
101 + logger.warning(
102 + "`torch.compile` is turned on, but the model %s"
103 + " does not support it. Please open an issue on GitHub"
104 + "if you want it to be supported.",
105 + vllm_config.model_config.model)
106 _current_vllm_config = old_vllm_config
107
108