@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 91642db952
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[torch.compile] use depyf to dump torch.compile internals (#10972)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Dec 11, 2024 at 10:43 UTC 91642db952458fbb6ae7c2d167757dc86b105991
7 files changed +66 -42
requirements-common.txt
+1
@@ -33,3 +33,4 @@ six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that need
33 setuptools>=74.1.1; python_version > '3.11' # Setuptools is used by triton, we need to ensure a modern version is installed for 3.12+ so that it does not try to import distutils, which was removed in 3.12
34 einops # Required for Qwen2-VL.
35 compressed-tensors == 0.8.0 # required for compressed-tensors
36 +depyf==0.18.0 # required for profiling and debugging torch.compile
vllm/compilation/backends.py
+36 -33
@@ -9,7 +9,7 @@ import torch
9 import torch.fx as fx
10
11 import vllm.envs as envs
12 -from vllm.config import CompilationConfig
12 +from vllm.config import CompilationConfig, VllmConfig
13 from vllm.logger import init_logger
14 from vllm.utils import weak_ref_tensors
15
@@ -149,14 +149,15 @@ class PiecewiseCompileInterpreter(torch.fx.Interpreter):
149 """
150
151 def __init__(self, module: torch.fx.GraphModule,
152 - compile_submod_names: List[str],
153 - compilation_configs: CompilationConfig, graph_pool):
152 + compile_submod_names: List[str], vllm_config: VllmConfig,
153 + graph_pool):
154 super().__init__(module)
155 from torch._guards import detect_fake_mode
156 self.fake_mode = detect_fake_mode()
157 self.compile_submod_names = compile_submod_names
158 - self.compilation_configs = compilation_configs
158 + self.compilation_config = vllm_config.compilation_config
159 self.graph_pool = graph_pool
160 + self.vllm_config = vllm_config
161
162 def run(self, *args):
163 fake_args = [
@@ -182,15 +183,15 @@ class PiecewiseCompileInterpreter(torch.fx.Interpreter):
183 compiled_graph_for_general_shape = wrap_inductor(
184 submod,
185 args,
185 - self.compilation_configs.inductor_compile_config,
186 - self.compilation_configs,
186 + self.compilation_config.inductor_compile_config,
187 + self.compilation_config,
188 graph_index=index,
189 num_graphs=len(self.compile_submod_names),
190 runtime_shape=None,
190 - use_inductor=self.compilation_configs.use_inductor)
191 + use_inductor=self.compilation_config.use_inductor)
192
193 self.module.__dict__[target] = PiecewiseBackend(
193 - submod, self.compilation_configs, self.graph_pool, index,
194 + submod, self.vllm_config, self.graph_pool, index,
195 len(self.compile_submod_names), sym_shape_indices,
196 compiled_graph_for_general_shape)
197
@@ -211,7 +212,8 @@ class VllmBackend:
212 which handles the post-grad passes.
213 """
214
214 - compilation_configs: CompilationConfig
215 + vllm_config: VllmConfig
216 + compilation_config: CompilationConfig
217 graph_pool: Any
218 _called: bool = False
219 # the graph we compiled
@@ -227,7 +229,7 @@ class VllmBackend:
229
230 def __init__(
231 self,
230 - compilation_configs: CompilationConfig,
232 + vllm_config: VllmConfig,
233 ):
234 global global_graph_pool
235 if global_graph_pool is None:
@@ -244,13 +246,14 @@ class VllmBackend:
246 self.sym_tensor_indices = []
247 self.input_buffers = []
248
247 - self.compilation_configs = compilation_configs
249 + self.vllm_config = vllm_config
250 + self.compilation_config = vllm_config.compilation_config
251
252 # `torch.compile` is JIT compiled, so we don't need to
253 # do anything here
254
255 def configure_post_pass(self):
253 - config = self.compilation_configs
256 + config = self.compilation_config
257 self.post_grad_pass_manager.configure(config.pass_config)
258
259 # Post-grad custom passes are run using the post_grad_custom_post_pass
@@ -271,7 +274,7 @@ class VllmBackend:
274 from .monitor import torch_compile_start_time
275 dynamo_time = time.time() - torch_compile_start_time
276 logger.info("Dynamo bytecode transform time: %.2f s", dynamo_time)
274 - self.compilation_configs.compilation_time += dynamo_time
277 + self.compilation_config.compilation_time += dynamo_time
278
279 # we control the compilation process, each instance can only be
280 # called once
@@ -281,7 +284,7 @@ class VllmBackend:
284 self.configure_post_pass()
285
286 self.split_gm, self.piecewise_graphs = split_graph(
284 - graph, self.compilation_configs.splitting_ops)
287 + graph, self.compilation_config.splitting_ops)
288
289 from torch._dynamo.utils import lazy_format_graph_code
290 logger.debug("%s", lazy_format_graph_code("before split", self.graph))
@@ -298,13 +301,13 @@ class VllmBackend:
301 # propagate the split graph to the piecewise backend,
302 # compile submodules with symbolic shapes
303 PiecewiseCompileInterpreter(self.split_gm, submod_names_to_compile,
301 - self.compilation_configs,
304 + self.vllm_config,
305 self.graph_pool).run(*example_inputs)
306
307 self._called = True
308
306 - if not self.compilation_configs.use_cudagraph or \
307 - not self.compilation_configs.cudagraph_copy_inputs:
309 + if not self.compilation_config.use_cudagraph or \
310 + not self.compilation_config.cudagraph_copy_inputs:
311 return self.split_gm
312
313 # if we need to copy input buffers for cudagraph
@@ -364,10 +367,9 @@ class ConcreteSizeEntry:
367
368 class PiecewiseBackend:
369
367 - def __init__(self, graph: fx.GraphModule,
368 - compilation_configs: CompilationConfig, graph_pool: Any,
369 - piecewise_compile_index: int, total_piecewise_compiles: int,
370 - sym_shape_indices: List[int],
370 + def __init__(self, graph: fx.GraphModule, vllm_config: VllmConfig,
371 + graph_pool: Any, piecewise_compile_index: int,
372 + total_piecewise_compiles: int, sym_shape_indices: List[int],
373 compiled_graph_for_general_shape: Callable):
374 """
375 The backend for piecewise compilation.
@@ -375,7 +377,7 @@ class PiecewiseBackend:
377
378 We will compile `self.graph` once for the general shape,
379 and then compile for different shapes specified in
378 - `compilation_configs.compile_sizes`.
380 + `compilation_config.compile_sizes`.
381
382 Independently, we will capture cudagraph for different shapes.
383
@@ -383,7 +385,8 @@ class PiecewiseBackend:
385 compile it first, and then capture cudagraph.
386 """
387 self.graph = graph
386 - self.compilation_configs = compilation_configs
388 + self.vllm_config = vllm_config
389 + self.compilation_config = vllm_config.compilation_config
390 self.graph_pool = graph_pool
391 self.piecewise_compile_index = piecewise_compile_index
392 self.total_piecewise_compiles = total_piecewise_compiles
@@ -393,10 +396,10 @@ class PiecewiseBackend:
396 piecewise_compile_index == total_piecewise_compiles - 1)
397
398 self.compile_sizes: Set[int] = set(
396 - self.compilation_configs.compile_sizes)
399 + self.compilation_config.compile_sizes)
400 self.capture_sizes: Set[int] = set(
398 - self.compilation_configs.capture_sizes
399 - ) if self.compilation_configs.use_cudagraph else set()
401 + self.compilation_config.capture_sizes
402 + ) if self.compilation_config.use_cudagraph else set()
403
404 self.first_run_finished = False
405
@@ -423,7 +426,7 @@ class PiecewiseBackend:
426 self.first_run_finished = True
427 # no specific sizes to compile
428 if self.is_last_graph and not self.to_be_compiled_sizes:
426 - end_monitoring_torch_compile(self.compilation_configs)
429 + end_monitoring_torch_compile(self.vllm_config)
430 return self.compiled_graph_for_general_shape(*args)
431
432 runtime_shape = args[self.sym_shape_indices[0]]
@@ -443,28 +446,28 @@ class PiecewiseBackend:
446 entry.runnable = wrap_inductor(
447 self.graph,
448 args,
446 - self.compilation_configs.inductor_compile_config,
447 - self.compilation_configs,
449 + self.compilation_config.inductor_compile_config,
450 + self.compilation_config,
451 graph_index=self.piecewise_compile_index,
452 num_graphs=self.total_piecewise_compiles,
453 runtime_shape=runtime_shape,
451 - use_inductor=self.compilation_configs.use_inductor)
454 + use_inductor=self.compilation_config.use_inductor)
455
456 # finished compilations for all required shapes
457 if self.is_last_graph and not self.to_be_compiled_sizes:
455 - end_monitoring_torch_compile(self.compilation_configs)
458 + end_monitoring_torch_compile(self.vllm_config)
459
460 if not entry.use_cudagraph:
461 return entry.runnable(*args)
462
463 if entry.cudagraph is None:
461 - if entry.num_finished_warmup < self.compilation_configs.cudagraph_num_of_warmups: # noqa
464 + if entry.num_finished_warmup < self.compilation_config.cudagraph_num_of_warmups: # noqa
465 entry.num_finished_warmup += 1
466 if self.is_first_graph:
467 logger.debug(
468 "Warming up %s/%s for shape %s",
469 entry.num_finished_warmup,
467 - self.compilation_configs.cudagraph_num_of_warmups,
470 + self.compilation_config.cudagraph_num_of_warmups,
471 runtime_shape)
472 return entry.runnable(*args)
473
vllm/compilation/decorators.py
+1 -1
@@ -185,7 +185,7 @@ def _support_torch_compile(
185 "Unsupported dynamic dimensions"
186 f" {dims} for argument {k} with type {type(arg)}.")
187 # here, it is the starting point of the `torch.compile` process
188 - start_monitoring_torch_compile(self.vllm_config.compilation_config)
188 + start_monitoring_torch_compile(self.vllm_config)
189
190 # if we don't use custom dispatcher, we can directly call the
191 # compiled function and let torch.compile handle the dispatching,
vllm/compilation/monitor.py
+20 -3
@@ -1,19 +1,36 @@
1 +import os
2 import time
3
3 -from vllm.config import CompilationConfig, CompilationLevel
4 +from vllm.config import CompilationConfig, CompilationLevel, VllmConfig
5 from vllm.logger import init_logger
6
7 logger = init_logger(__name__)
8
9 +context_manager = None
10 torch_compile_start_time: float = 0.0
11
12
11 -def start_monitoring_torch_compile(compilation_config: CompilationConfig):
13 +def start_monitoring_torch_compile(vllm_config: VllmConfig):
14 global torch_compile_start_time
15 torch_compile_start_time = time.time()
16
17 + compilation_config: CompilationConfig = vllm_config.compilation_config
18 + if compilation_config.level == CompilationLevel.PIECEWISE and \
19 + compilation_config.debug_dump_path:
20 + import depyf
21 + path = os.path.join(compilation_config.debug_dump_path,
22 + f"rank_{vllm_config.parallel_config.rank}")
23 + global context_manager
24 + context_manager = depyf.prepare_debug(path)
25 + context_manager.__enter__()
26
16 -def end_monitoring_torch_compile(compilation_config: CompilationConfig):
27 +
28 +def end_monitoring_torch_compile(vllm_config: VllmConfig):
29 + compilation_config: CompilationConfig = vllm_config.compilation_config
30 if compilation_config.level == CompilationLevel.PIECEWISE:
31 logger.info("torch.compile takes %.2f s in total",
32 compilation_config.compilation_time)
33 + global context_manager
34 + if context_manager is not None:
35 + context_manager.__exit__(None, None, None)
36 + context_manager = None
vllm/compilation/wrapper.py
+2 -2
@@ -32,8 +32,8 @@ class TorchCompileWrapperWithCustomDispatcher:
32 # default compilation settings
33 # compiling the forward method
34
35 - backend = get_current_vllm_config(
36 - ).compilation_config.init_backend()
35 + vllm_config = get_current_vllm_config()
36 + backend = vllm_config.compilation_config.init_backend(vllm_config)
37
38 compiled_callable = torch.compile(
39 self.forward,
vllm/config.py
+4 -2
@@ -2222,6 +2222,7 @@ class CompilationConfig(BaseModel):
2222 - 1: dynamo as is.
2223 - 2: dynamo once.
2224 - 3: piecewise compilation.
2225 + - debug_dump_path: the path to dump the debug information.
2226 - backend: the backend for compilation. It needs to be a string.
2227 - "" (empty string): use the default backend.
2228 - "eager"/"openxla"/...: use the specified backend registered in PyTorch.
@@ -2289,6 +2290,7 @@ class CompilationConfig(BaseModel):
2290 certain small batchsizes, where inductor is good at optimizing.
2291 """ # noqa
2292 level: int = 0
2293 + debug_dump_path: str = ""
2294 backend: str = ""
2295 custom_ops: List[str] = Field(default_factory=list)
2296 splitting_ops: List[str] = Field(default_factory=lambda: [
@@ -2394,7 +2396,7 @@ class CompilationConfig(BaseModel):
2396 self.static_forward_context = {}
2397 self.compilation_time = 0.0
2398
2397 - def init_backend(self) -> Union[str, Callable]:
2399 + def init_backend(self, vllm_config: "VllmConfig") -> Union[str, Callable]:
2400 if self.level == CompilationLevel.NO_COMPILATION:
2401 raise ValueError("No compilation level is set.")
2402
@@ -2413,7 +2415,7 @@ class CompilationConfig(BaseModel):
2415 # merge with the config use_inductor
2416 assert self.level == CompilationLevel.PIECEWISE
2417 from vllm.compilation.backends import VllmBackend
2416 - return VllmBackend(self)
2418 + return VllmBackend(vllm_config)
2419
2420 def init_with_cudagraph_sizes(self, sizes_to_specialize: List[int]):
2421 """To complete the initialization of config,
vllm/worker/model_runner.py
+2 -1
@@ -1162,7 +1162,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1162
1163 if self.vllm_config.compilation_config.level ==\
1164 CompilationLevel.DYNAMO_AS_IS and supports_dynamo():
1165 - backend = self.vllm_config.compilation_config.init_backend()
1165 + backend = self.vllm_config.compilation_config.init_backend(
1166 + self.vllm_config)
1167 self.model = torch.compile(
1168 self.model,
1169 fullgraph=envs.VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE,