@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d1c2e15eb3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[torch.compile] add dynamo time tracking (#11005)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Dec 8, 2024 at 23:09 UTC d1c2e15eb31ef12e688ce0cb71895f88eaf4cd4f
3 files changed +16 -5
vllm/compilation/backends.py
+6
@@ -265,7 +265,13 @@ class VllmBackend:
265
266 def __call__(self, graph: fx.GraphModule, example_inputs) -> Callable:
267
268 + # when dynamo calls the backend, it means the bytecode
269 + # transform and analysis are done
270 compilation_counter.num_graphs_seen += 1
271 + from .monitor import torch_compile_start_time
272 + dynamo_time = time.time() - torch_compile_start_time
273 + logger.info("Dynamo bytecode transform time: %.2f s", dynamo_time)
274 + self.compilation_configs.compilation_time += dynamo_time
275
276 # we control the compilation process, each instance can only be
277 # called once
vllm/compilation/decorators.py
+3 -3
@@ -145,6 +145,7 @@ def _support_torch_compile(
145
146 def __init__(self, *, vllm_config: VllmConfig, prefix: str = '', **kwargs):
147 old_init(self, vllm_config=vllm_config, prefix=prefix, **kwargs)
148 + self.vllm_config = vllm_config
149 # for CompilationLevel.DYNAMO_AS_IS , the upper level model runner
150 # will handle the compilation, so we don't need to do anything here.
151 self.do_not_compile = \
@@ -157,9 +158,6 @@ def _support_torch_compile(
158 TorchCompileWrapperWithCustomDispatcher.__init__(
159 self, compilation_level=vllm_config.compilation_config.level)
160
160 - if vllm_config.compilation_config.level == CompilationLevel.PIECEWISE:
161 - start_monitoring_torch_compile(vllm_config.compilation_config)
162 -
161 cls.__init__ = __init__
162
163 def __call__(self, *args, **kwargs):
@@ -186,6 +184,8 @@ def _support_torch_compile(
184 raise ValueError(
185 "Unsupported dynamic dimensions"
186 f" {dims} for argument {k} with type {type(arg)}.")
187 + # here, it is the starting point of the `torch.compile` process
188 + start_monitoring_torch_compile(self.vllm_config.compilation_config)
189
190 # if we don't use custom dispatcher, we can directly call the
191 # compiled function and let torch.compile handle the dispatching,
vllm/compilation/monitor.py
+7 -2
@@ -1,14 +1,19 @@
1 +import time
2 +
3 from vllm.config import CompilationConfig, CompilationLevel
4 from vllm.logger import init_logger
5
6 logger = init_logger(__name__)
7
8 +torch_compile_start_time: float = 0.0
9 +
10
11 def start_monitoring_torch_compile(compilation_config: CompilationConfig):
8 - pass
12 + global torch_compile_start_time
13 + torch_compile_start_time = time.time()
14
15
16 def end_monitoring_torch_compile(compilation_config: CompilationConfig):
17 if compilation_config.level == CompilationLevel.PIECEWISE:
13 - logger.info("graph compilation takes %.2f s in total",
18 + logger.info("torch.compile takes %.2f s in total",
19 compilation_config.compilation_time)