A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[torch.compile] add dynamo time tracking (#11005)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Dec 8, 2024 at 23:09 UTC
d1c2e15eb31ef12e688ce0cb71895f88eaf4cd4f
3 files changed
+16
-5
vllm/compilation/backends.py
+6
@@ -265,7 +265,13 @@ class VllmBackend:
265
266
def __call__(self, graph: fx.GraphModule, example_inputs) -> Callable:
267
268
+ # when dynamo calls the backend, it means the bytecode
269
+ # transform and analysis are done
270
compilation_counter.num_graphs_seen += 1
271
+ from .monitor import torch_compile_start_time
272
+ dynamo_time = time.time() - torch_compile_start_time
273
+ logger.info("Dynamo bytecode transform time: %.2f s", dynamo_time)
274
+ self.compilation_configs.compilation_time += dynamo_time
275
276
# we control the compilation process, each instance can only be
277
# called once
vllm/compilation/decorators.py
+3
-3
@@ -145,6 +145,7 @@ def _support_torch_compile(
145
146
def __init__(self, *, vllm_config: VllmConfig, prefix: str = '', **kwargs):
147
old_init(self, vllm_config=vllm_config, prefix=prefix, **kwargs)
148
+ self.vllm_config = vllm_config
149
# for CompilationLevel.DYNAMO_AS_IS , the upper level model runner
150
# will handle the compilation, so we don't need to do anything here.
151
self.do_not_compile = \
@@ -157,9 +158,6 @@ def _support_torch_compile(
158
TorchCompileWrapperWithCustomDispatcher.__init__(
159
self, compilation_level=vllm_config.compilation_config.level)
160
160
- if vllm_config.compilation_config.level == CompilationLevel.PIECEWISE:
161
- start_monitoring_torch_compile(vllm_config.compilation_config)
162
-
161
cls.__init__ = __init__
162
163
def __call__(self, *args, **kwargs):
@@ -186,6 +184,8 @@ def _support_torch_compile(
184
raise ValueError(
185
"Unsupported dynamic dimensions"
186
f" {dims} for argument {k} with type {type(arg)}.")
187
+ # here, it is the starting point of the `torch.compile` process
188
+ start_monitoring_torch_compile(self.vllm_config.compilation_config)
189
190
# if we don't use custom dispatcher, we can directly call the
191
# compiled function and let torch.compile handle the dispatching,
vllm/compilation/monitor.py
+7
-2
@@ -1,14 +1,19 @@
1
+import time
2
+
3
from vllm.config import CompilationConfig, CompilationLevel
4
from vllm.logger import init_logger
5
6
logger = init_logger(__name__)
7
8
+torch_compile_start_time: float = 0.0
9
+
10
11
def start_monitoring_torch_compile(compilation_config: CompilationConfig):
8
- pass
12
+ global torch_compile_start_time
13
+ torch_compile_start_time = time.time()
14
15
16
def end_monitoring_torch_compile(compilation_config: CompilationConfig):
17
if compilation_config.level == CompilationLevel.PIECEWISE:
13
- logger.info("graph compilation takes %.2f s in total",
18
+ logger.info("torch.compile takes %.2f s in total",
19
compilation_config.compilation_time)