A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[torch.compile] add logging for compilation time (#10941)
Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
youkaichao committed
Dec 6, 2024 at 02:07 UTC
b031a455a9fa9d57952281dac2a1146d6440790f
6 files changed
+75
-10
vllm/compilation/backends.py
+46
-10
@@ -1,5 +1,6 @@
1
import copy
2
import dataclasses
3
+import time
4
from contextlib import ExitStack
5
from typing import Any, Callable, Dict, List, Optional, Sequence, Set, Tuple
6
from unittest.mock import patch
@@ -14,6 +15,7 @@ from vllm.utils import weak_ref_tensors
15
16
from .counter import compilation_counter
17
from .inductor_pass import InductorPass
18
+from .monitor import end_monitoring_torch_compile
19
from .pass_manager import PostGradPassManager
20
21
logger = init_logger(__name__)
@@ -22,20 +24,21 @@ logger = init_logger(__name__)
24
def wrap_inductor(graph,
25
example_inputs,
26
additional_inductor_config,
25
- do_logging=False,
27
+ compilation_config: CompilationConfig,
28
+ graph_index: int = 0,
29
+ num_graphs: int = 1,
30
runtime_shape: Optional[int] = None,
31
use_inductor: bool = True):
32
+ if graph_index == 0:
33
+ # before compiling the first graph, record the start time
34
+ global compilation_start_time
35
+ compilation_start_time = time.time()
36
+
37
if not use_inductor:
38
return graph
39
40
compilation_counter.num_inductor_compilations += 1
41
33
- if do_logging:
34
- if runtime_shape is None:
35
- logger.info("Compiling a graph for general shape")
36
- else:
37
- logger.info("Compiling a graph for shape %s", runtime_shape)
38
-
42
from torch._inductor import config
43
current_config = config.shallow_copy_dict()
44
from torch._inductor.compile_fx import compile_fx
@@ -52,7 +55,23 @@ def wrap_inductor(graph,
55
# inductor can inplace modify the graph, so we need to copy it
56
# see https://github.com/pytorch/pytorch/issues/138980
57
graph = copy.deepcopy(graph)
55
- return compile_fx(graph, example_inputs, config_patches=current_config)
58
+ compiled_graph = compile_fx(graph,
59
+ example_inputs,
60
+ config_patches=current_config)
61
+
62
+ # after compiling the last graph, record the end time
63
+ if graph_index == num_graphs - 1:
64
+ now = time.time()
65
+ elapsed = now - compilation_start_time
66
+ compilation_config.compilation_time += elapsed
67
+ if runtime_shape is None:
68
+ logger.info("Compiling a graph for general shape takes %.2f s",
69
+ elapsed)
70
+ else:
71
+ logger.info("Compiling a graph for shape %s takes %.2f s",
72
+ runtime_shape, elapsed)
73
+
74
+ return compiled_graph
75
76
77
@dataclasses.dataclass
@@ -114,6 +133,8 @@ def split_graph(graph: fx.GraphModule,
133
# we share the global graph pool among all the backends
134
global_graph_pool = None
135
136
+compilation_start_time = 0.0
137
+
138
139
class PiecewiseCompileInterpreter(torch.fx.Interpreter):
140
"""Code adapted from `torch.fx.passes.shape_prop.ShapeProp`.
@@ -157,12 +178,15 @@ class PiecewiseCompileInterpreter(torch.fx.Interpreter):
178
sym_shape_indices = [
179
i for i, x in enumerate(args) if isinstance(x, torch.SymInt)
180
]
181
+ global compilation_start_time
182
compiled_graph_for_general_shape = wrap_inductor(
183
submod,
184
args,
185
self.compilation_configs.inductor_compile_config,
186
+ self.compilation_configs,
187
+ graph_index=index,
188
+ num_graphs=len(self.compile_submod_names),
189
runtime_shape=None,
165
- do_logging=index == 0,
190
use_inductor=self.compilation_configs.use_inductor)
191
192
self.module.__dict__[target] = PiecewiseBackend(
@@ -379,6 +403,8 @@ class PiecewiseBackend:
403
# the entries for different shapes that we need to either
404
# compile or capture cudagraph
405
self.concrete_size_entries: Dict[int, ConcreteSizeEntry] = {}
406
+ self.to_be_compiled_sizes: Set[int] = self.compile_sizes.union(
407
+ self.capture_sizes)
408
for shape in self.compile_sizes.union(self.capture_sizes):
409
self.concrete_size_entries[shape] = ConcreteSizeEntry(
410
runtime_shape=shape,
@@ -389,6 +415,9 @@ class PiecewiseBackend:
415
def __call__(self, *args) -> Any:
416
if not self.first_run_finished:
417
self.first_run_finished = True
418
+ # no specific sizes to compile
419
+ if self.is_last_graph and not self.to_be_compiled_sizes:
420
+ end_monitoring_torch_compile(self.compilation_configs)
421
return self.compiled_graph_for_general_shape(*args)
422
423
runtime_shape = args[self.sym_shape_indices[0]]
@@ -403,15 +432,22 @@ class PiecewiseBackend:
432
433
if entry.need_to_compile and not entry.compiled:
434
entry.compiled = True
435
+ self.to_be_compiled_sizes.remove(runtime_shape)
436
# args are real arguments
437
entry.runnable = wrap_inductor(
438
self.graph,
439
args,
440
self.compilation_configs.inductor_compile_config,
441
+ self.compilation_configs,
442
+ graph_index=self.piecewise_compile_index,
443
+ num_graphs=self.total_piecewise_compiles,
444
runtime_shape=runtime_shape,
412
- do_logging=self.is_first_graph,
445
use_inductor=self.compilation_configs.use_inductor)
446
447
+ # finished compilations for all required shapes
448
+ if self.is_last_graph and not self.to_be_compiled_sizes:
449
+ end_monitoring_torch_compile(self.compilation_configs)
450
+
451
if not entry.use_cudagraph:
452
return entry.runnable(*args)
453
vllm/compilation/decorators.py
+5
@@ -11,6 +11,8 @@ from vllm.logger import init_logger
11
from vllm.sequence import IntermediateTensors
12
from vllm.utils import supports_dynamo
13
14
+from .monitor import start_monitoring_torch_compile
15
+
16
logger = init_logger(__name__)
17
18
_T = TypeVar("_T", bound=type[nn.Module])
@@ -155,6 +157,9 @@ def _support_torch_compile(
157
TorchCompileWrapperWithCustomDispatcher.__init__(
158
self, compilation_level=vllm_config.compilation_config.level)
159
160
+ if vllm_config.compilation_config.level == CompilationLevel.PIECEWISE:
161
+ start_monitoring_torch_compile(vllm_config.compilation_config)
162
+
163
cls.__init__ = __init__
164
165
def __call__(self, *args, **kwargs):
vllm/compilation/monitor.py
new
+14
@@ -0,0 +1,14 @@
1
+from vllm.config import CompilationConfig, CompilationLevel
2
+from vllm.logger import init_logger
3
+
4
+logger = init_logger(__name__)
5
+
6
+
7
+def start_monitoring_torch_compile(compilation_config: CompilationConfig):
8
+ pass
9
+
10
+
11
+def end_monitoring_torch_compile(compilation_config: CompilationConfig):
12
+ if compilation_config.level == CompilationLevel.PIECEWISE:
13
+ logger.info("graph compilation takes %.2f s in total",
14
+ compilation_config.compilation_time)
vllm/config.py
+2
@@ -2281,6 +2281,7 @@ class CompilationConfig(BaseModel):
2281
# keep track of enabled and disabled custom ops
2282
enabled_custom_ops: Counter[str] = PrivateAttr
2283
disabled_custom_ops: Counter[str] = PrivateAttr
2284
+ compilation_time: float = PrivateAttr
2285
2286
# Per-model forward context
2287
# Mainly used to store attention cls
@@ -2319,6 +2320,7 @@ class CompilationConfig(BaseModel):
2320
self.enabled_custom_ops = Counter()
2321
self.disabled_custom_ops = Counter()
2322
self.static_forward_context = {}
2323
+ self.compilation_time = 0.0
2324
2325
def init_backend(self) -> Union[str, Callable]:
2326
if self.level == CompilationLevel.NO_COMPILATION:
vllm/engine/llm_engine.py
+4
@@ -473,6 +473,7 @@ class LLMEngine:
473
The workers will determine the number of blocks in both the GPU cache
474
and the swap CPU cache.
475
"""
476
+ start = time.time()
477
num_gpu_blocks, num_cpu_blocks = (
478
self.model_executor.determine_num_available_blocks())
479
@@ -488,6 +489,9 @@ class LLMEngine:
489
self.cache_config.num_cpu_blocks = num_cpu_blocks
490
491
self.model_executor.initialize_cache(num_gpu_blocks, num_cpu_blocks)
492
+ elapsed = time.time() - start
493
+ logger.info(("init engine (profile, create kv cache, "
494
+ "warmup model) took %.2f seconds"), elapsed)
495
496
@classmethod
497
def _get_executor_cls(cls,
vllm/v1/engine/core.py
+4
@@ -67,6 +67,7 @@ class EngineCore:
67
68
def _initialize_kv_caches(self,
69
cache_config: CacheConfig) -> Tuple[int, int]:
70
+ start = time.time()
71
num_gpu_blocks, _ = self.model_executor.determine_num_available_blocks(
72
)
73
@@ -80,6 +81,9 @@ class EngineCore:
81
82
num_cpu_blocks = 0
83
self.model_executor.initialize_cache(num_gpu_blocks)
84
+ elapsed = time.time() - start
85
+ logger.info(("init engine (profile, create kv cache, "
86
+ "warmup model) took %.2f seconds"), elapsed)
87
return num_gpu_blocks, num_cpu_blocks
88
89
def add_request(self, request: EngineCoreRequest):