@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b031a455a9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[torch.compile] add logging for compilation time (#10941)

Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

youkaichao committed Dec 6, 2024 at 02:07 UTC b031a455a9fa9d57952281dac2a1146d6440790f
6 files changed +75 -10
vllm/compilation/backends.py
+46 -10
@@ -1,5 +1,6 @@
1 import copy
2 import dataclasses
3 +import time
4 from contextlib import ExitStack
5 from typing import Any, Callable, Dict, List, Optional, Sequence, Set, Tuple
6 from unittest.mock import patch
@@ -14,6 +15,7 @@ from vllm.utils import weak_ref_tensors
15
16 from .counter import compilation_counter
17 from .inductor_pass import InductorPass
18 +from .monitor import end_monitoring_torch_compile
19 from .pass_manager import PostGradPassManager
20
21 logger = init_logger(__name__)
@@ -22,20 +24,21 @@ logger = init_logger(__name__)
24 def wrap_inductor(graph,
25 example_inputs,
26 additional_inductor_config,
25 - do_logging=False,
27 + compilation_config: CompilationConfig,
28 + graph_index: int = 0,
29 + num_graphs: int = 1,
30 runtime_shape: Optional[int] = None,
31 use_inductor: bool = True):
32 + if graph_index == 0:
33 + # before compiling the first graph, record the start time
34 + global compilation_start_time
35 + compilation_start_time = time.time()
36 +
37 if not use_inductor:
38 return graph
39
40 compilation_counter.num_inductor_compilations += 1
41
33 - if do_logging:
34 - if runtime_shape is None:
35 - logger.info("Compiling a graph for general shape")
36 - else:
37 - logger.info("Compiling a graph for shape %s", runtime_shape)
38 -
42 from torch._inductor import config
43 current_config = config.shallow_copy_dict()
44 from torch._inductor.compile_fx import compile_fx
@@ -52,7 +55,23 @@ def wrap_inductor(graph,
55 # inductor can inplace modify the graph, so we need to copy it
56 # see https://github.com/pytorch/pytorch/issues/138980
57 graph = copy.deepcopy(graph)
55 - return compile_fx(graph, example_inputs, config_patches=current_config)
58 + compiled_graph = compile_fx(graph,
59 + example_inputs,
60 + config_patches=current_config)
61 +
62 + # after compiling the last graph, record the end time
63 + if graph_index == num_graphs - 1:
64 + now = time.time()
65 + elapsed = now - compilation_start_time
66 + compilation_config.compilation_time += elapsed
67 + if runtime_shape is None:
68 + logger.info("Compiling a graph for general shape takes %.2f s",
69 + elapsed)
70 + else:
71 + logger.info("Compiling a graph for shape %s takes %.2f s",
72 + runtime_shape, elapsed)
73 +
74 + return compiled_graph
75
76
77 @dataclasses.dataclass
@@ -114,6 +133,8 @@ def split_graph(graph: fx.GraphModule,
133 # we share the global graph pool among all the backends
134 global_graph_pool = None
135
136 +compilation_start_time = 0.0
137 +
138
139 class PiecewiseCompileInterpreter(torch.fx.Interpreter):
140 """Code adapted from `torch.fx.passes.shape_prop.ShapeProp`.
@@ -157,12 +178,15 @@ class PiecewiseCompileInterpreter(torch.fx.Interpreter):
178 sym_shape_indices = [
179 i for i, x in enumerate(args) if isinstance(x, torch.SymInt)
180 ]
181 + global compilation_start_time
182 compiled_graph_for_general_shape = wrap_inductor(
183 submod,
184 args,
185 self.compilation_configs.inductor_compile_config,
186 + self.compilation_configs,
187 + graph_index=index,
188 + num_graphs=len(self.compile_submod_names),
189 runtime_shape=None,
165 - do_logging=index == 0,
190 use_inductor=self.compilation_configs.use_inductor)
191
192 self.module.__dict__[target] = PiecewiseBackend(
@@ -379,6 +403,8 @@ class PiecewiseBackend:
403 # the entries for different shapes that we need to either
404 # compile or capture cudagraph
405 self.concrete_size_entries: Dict[int, ConcreteSizeEntry] = {}
406 + self.to_be_compiled_sizes: Set[int] = self.compile_sizes.union(
407 + self.capture_sizes)
408 for shape in self.compile_sizes.union(self.capture_sizes):
409 self.concrete_size_entries[shape] = ConcreteSizeEntry(
410 runtime_shape=shape,
@@ -389,6 +415,9 @@ class PiecewiseBackend:
415 def __call__(self, *args) -> Any:
416 if not self.first_run_finished:
417 self.first_run_finished = True
418 + # no specific sizes to compile
419 + if self.is_last_graph and not self.to_be_compiled_sizes:
420 + end_monitoring_torch_compile(self.compilation_configs)
421 return self.compiled_graph_for_general_shape(*args)
422
423 runtime_shape = args[self.sym_shape_indices[0]]
@@ -403,15 +432,22 @@ class PiecewiseBackend:
432
433 if entry.need_to_compile and not entry.compiled:
434 entry.compiled = True
435 + self.to_be_compiled_sizes.remove(runtime_shape)
436 # args are real arguments
437 entry.runnable = wrap_inductor(
438 self.graph,
439 args,
440 self.compilation_configs.inductor_compile_config,
441 + self.compilation_configs,
442 + graph_index=self.piecewise_compile_index,
443 + num_graphs=self.total_piecewise_compiles,
444 runtime_shape=runtime_shape,
412 - do_logging=self.is_first_graph,
445 use_inductor=self.compilation_configs.use_inductor)
446
447 + # finished compilations for all required shapes
448 + if self.is_last_graph and not self.to_be_compiled_sizes:
449 + end_monitoring_torch_compile(self.compilation_configs)
450 +
451 if not entry.use_cudagraph:
452 return entry.runnable(*args)
453
vllm/compilation/decorators.py
+5
@@ -11,6 +11,8 @@ from vllm.logger import init_logger
11 from vllm.sequence import IntermediateTensors
12 from vllm.utils import supports_dynamo
13
14 +from .monitor import start_monitoring_torch_compile
15 +
16 logger = init_logger(__name__)
17
18 _T = TypeVar("_T", bound=type[nn.Module])
@@ -155,6 +157,9 @@ def _support_torch_compile(
157 TorchCompileWrapperWithCustomDispatcher.__init__(
158 self, compilation_level=vllm_config.compilation_config.level)
159
160 + if vllm_config.compilation_config.level == CompilationLevel.PIECEWISE:
161 + start_monitoring_torch_compile(vllm_config.compilation_config)
162 +
163 cls.__init__ = __init__
164
165 def __call__(self, *args, **kwargs):
vllm/compilation/monitor.py new
+14
@@ -0,0 +1,14 @@
1 +from vllm.config import CompilationConfig, CompilationLevel
2 +from vllm.logger import init_logger
3 +
4 +logger = init_logger(__name__)
5 +
6 +
7 +def start_monitoring_torch_compile(compilation_config: CompilationConfig):
8 + pass
9 +
10 +
11 +def end_monitoring_torch_compile(compilation_config: CompilationConfig):
12 + if compilation_config.level == CompilationLevel.PIECEWISE:
13 + logger.info("graph compilation takes %.2f s in total",
14 + compilation_config.compilation_time)
vllm/config.py
+2
@@ -2281,6 +2281,7 @@ class CompilationConfig(BaseModel):
2281 # keep track of enabled and disabled custom ops
2282 enabled_custom_ops: Counter[str] = PrivateAttr
2283 disabled_custom_ops: Counter[str] = PrivateAttr
2284 + compilation_time: float = PrivateAttr
2285
2286 # Per-model forward context
2287 # Mainly used to store attention cls
@@ -2319,6 +2320,7 @@ class CompilationConfig(BaseModel):
2320 self.enabled_custom_ops = Counter()
2321 self.disabled_custom_ops = Counter()
2322 self.static_forward_context = {}
2323 + self.compilation_time = 0.0
2324
2325 def init_backend(self) -> Union[str, Callable]:
2326 if self.level == CompilationLevel.NO_COMPILATION:
vllm/engine/llm_engine.py
+4
@@ -473,6 +473,7 @@ class LLMEngine:
473 The workers will determine the number of blocks in both the GPU cache
474 and the swap CPU cache.
475 """
476 + start = time.time()
477 num_gpu_blocks, num_cpu_blocks = (
478 self.model_executor.determine_num_available_blocks())
479
@@ -488,6 +489,9 @@ class LLMEngine:
489 self.cache_config.num_cpu_blocks = num_cpu_blocks
490
491 self.model_executor.initialize_cache(num_gpu_blocks, num_cpu_blocks)
492 + elapsed = time.time() - start
493 + logger.info(("init engine (profile, create kv cache, "
494 + "warmup model) took %.2f seconds"), elapsed)
495
496 @classmethod
497 def _get_executor_cls(cls,
vllm/v1/engine/core.py
+4
@@ -67,6 +67,7 @@ class EngineCore:
67
68 def _initialize_kv_caches(self,
69 cache_config: CacheConfig) -> Tuple[int, int]:
70 + start = time.time()
71 num_gpu_blocks, _ = self.model_executor.determine_num_available_blocks(
72 )
73
@@ -80,6 +81,9 @@ class EngineCore:
81
82 num_cpu_blocks = 0
83 self.model_executor.initialize_cache(num_gpu_blocks)
84 + elapsed = time.time() - start
85 + logger.info(("init engine (profile, create kv cache, "
86 + "warmup model) took %.2f seconds"), elapsed)
87 return num_gpu_blocks, num_cpu_blocks
88
89 def add_request(self, request: EngineCoreRequest):