A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[torch.compile] use depyf to dump torch.compile internals (#10972)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Dec 11, 2024 at 10:43 UTC
91642db952458fbb6ae7c2d167757dc86b105991
7 files changed
+66
-42
requirements-common.txt
+1
@@ -33,3 +33,4 @@ six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that need
33
setuptools>=74.1.1; python_version > '3.11' # Setuptools is used by triton, we need to ensure a modern version is installed for 3.12+ so that it does not try to import distutils, which was removed in 3.12
34
einops # Required for Qwen2-VL.
35
compressed-tensors == 0.8.0 # required for compressed-tensors
36
+depyf==0.18.0 # required for profiling and debugging torch.compile
vllm/compilation/backends.py
+36
-33
@@ -9,7 +9,7 @@ import torch
9
import torch.fx as fx
10
11
import vllm.envs as envs
12
-from vllm.config import CompilationConfig
12
+from vllm.config import CompilationConfig, VllmConfig
13
from vllm.logger import init_logger
14
from vllm.utils import weak_ref_tensors
15
@@ -149,14 +149,15 @@ class PiecewiseCompileInterpreter(torch.fx.Interpreter):
149
"""
150
151
def __init__(self, module: torch.fx.GraphModule,
152
- compile_submod_names: List[str],
153
- compilation_configs: CompilationConfig, graph_pool):
152
+ compile_submod_names: List[str], vllm_config: VllmConfig,
153
+ graph_pool):
154
super().__init__(module)
155
from torch._guards import detect_fake_mode
156
self.fake_mode = detect_fake_mode()
157
self.compile_submod_names = compile_submod_names
158
- self.compilation_configs = compilation_configs
158
+ self.compilation_config = vllm_config.compilation_config
159
self.graph_pool = graph_pool
160
+ self.vllm_config = vllm_config
161
162
def run(self, *args):
163
fake_args = [
@@ -182,15 +183,15 @@ class PiecewiseCompileInterpreter(torch.fx.Interpreter):
183
compiled_graph_for_general_shape = wrap_inductor(
184
submod,
185
args,
185
- self.compilation_configs.inductor_compile_config,
186
- self.compilation_configs,
186
+ self.compilation_config.inductor_compile_config,
187
+ self.compilation_config,
188
graph_index=index,
189
num_graphs=len(self.compile_submod_names),
190
runtime_shape=None,
190
- use_inductor=self.compilation_configs.use_inductor)
191
+ use_inductor=self.compilation_config.use_inductor)
192
193
self.module.__dict__[target] = PiecewiseBackend(
193
- submod, self.compilation_configs, self.graph_pool, index,
194
+ submod, self.vllm_config, self.graph_pool, index,
195
len(self.compile_submod_names), sym_shape_indices,
196
compiled_graph_for_general_shape)
197
@@ -211,7 +212,8 @@ class VllmBackend:
212
which handles the post-grad passes.
213
"""
214
214
- compilation_configs: CompilationConfig
215
+ vllm_config: VllmConfig
216
+ compilation_config: CompilationConfig
217
graph_pool: Any
218
_called: bool = False
219
# the graph we compiled
@@ -227,7 +229,7 @@ class VllmBackend:
229
230
def __init__(
231
self,
230
- compilation_configs: CompilationConfig,
232
+ vllm_config: VllmConfig,
233
):
234
global global_graph_pool
235
if global_graph_pool is None:
@@ -244,13 +246,14 @@ class VllmBackend:
246
self.sym_tensor_indices = []
247
self.input_buffers = []
248
247
- self.compilation_configs = compilation_configs
249
+ self.vllm_config = vllm_config
250
+ self.compilation_config = vllm_config.compilation_config
251
252
# `torch.compile` is JIT compiled, so we don't need to
253
# do anything here
254
255
def configure_post_pass(self):
253
- config = self.compilation_configs
256
+ config = self.compilation_config
257
self.post_grad_pass_manager.configure(config.pass_config)
258
259
# Post-grad custom passes are run using the post_grad_custom_post_pass
@@ -271,7 +274,7 @@ class VllmBackend:
274
from .monitor import torch_compile_start_time
275
dynamo_time = time.time() - torch_compile_start_time
276
logger.info("Dynamo bytecode transform time: %.2f s", dynamo_time)
274
- self.compilation_configs.compilation_time += dynamo_time
277
+ self.compilation_config.compilation_time += dynamo_time
278
279
# we control the compilation process, each instance can only be
280
# called once
@@ -281,7 +284,7 @@ class VllmBackend:
284
self.configure_post_pass()
285
286
self.split_gm, self.piecewise_graphs = split_graph(
284
- graph, self.compilation_configs.splitting_ops)
287
+ graph, self.compilation_config.splitting_ops)
288
289
from torch._dynamo.utils import lazy_format_graph_code
290
logger.debug("%s", lazy_format_graph_code("before split", self.graph))
@@ -298,13 +301,13 @@ class VllmBackend:
301
# propagate the split graph to the piecewise backend,
302
# compile submodules with symbolic shapes
303
PiecewiseCompileInterpreter(self.split_gm, submod_names_to_compile,
301
- self.compilation_configs,
304
+ self.vllm_config,
305
self.graph_pool).run(*example_inputs)
306
307
self._called = True
308
306
- if not self.compilation_configs.use_cudagraph or \
307
- not self.compilation_configs.cudagraph_copy_inputs:
309
+ if not self.compilation_config.use_cudagraph or \
310
+ not self.compilation_config.cudagraph_copy_inputs:
311
return self.split_gm
312
313
# if we need to copy input buffers for cudagraph
@@ -364,10 +367,9 @@ class ConcreteSizeEntry:
367
368
class PiecewiseBackend:
369
367
- def __init__(self, graph: fx.GraphModule,
368
- compilation_configs: CompilationConfig, graph_pool: Any,
369
- piecewise_compile_index: int, total_piecewise_compiles: int,
370
- sym_shape_indices: List[int],
370
+ def __init__(self, graph: fx.GraphModule, vllm_config: VllmConfig,
371
+ graph_pool: Any, piecewise_compile_index: int,
372
+ total_piecewise_compiles: int, sym_shape_indices: List[int],
373
compiled_graph_for_general_shape: Callable):
374
"""
375
The backend for piecewise compilation.
@@ -375,7 +377,7 @@ class PiecewiseBackend:
377
378
We will compile `self.graph` once for the general shape,
379
and then compile for different shapes specified in
378
- `compilation_configs.compile_sizes`.
380
+ `compilation_config.compile_sizes`.
381
382
Independently, we will capture cudagraph for different shapes.
383
@@ -383,7 +385,8 @@ class PiecewiseBackend:
385
compile it first, and then capture cudagraph.
386
"""
387
self.graph = graph
386
- self.compilation_configs = compilation_configs
388
+ self.vllm_config = vllm_config
389
+ self.compilation_config = vllm_config.compilation_config
390
self.graph_pool = graph_pool
391
self.piecewise_compile_index = piecewise_compile_index
392
self.total_piecewise_compiles = total_piecewise_compiles
@@ -393,10 +396,10 @@ class PiecewiseBackend:
396
piecewise_compile_index == total_piecewise_compiles - 1)
397
398
self.compile_sizes: Set[int] = set(
396
- self.compilation_configs.compile_sizes)
399
+ self.compilation_config.compile_sizes)
400
self.capture_sizes: Set[int] = set(
398
- self.compilation_configs.capture_sizes
399
- ) if self.compilation_configs.use_cudagraph else set()
401
+ self.compilation_config.capture_sizes
402
+ ) if self.compilation_config.use_cudagraph else set()
403
404
self.first_run_finished = False
405
@@ -423,7 +426,7 @@ class PiecewiseBackend:
426
self.first_run_finished = True
427
# no specific sizes to compile
428
if self.is_last_graph and not self.to_be_compiled_sizes:
426
- end_monitoring_torch_compile(self.compilation_configs)
429
+ end_monitoring_torch_compile(self.vllm_config)
430
return self.compiled_graph_for_general_shape(*args)
431
432
runtime_shape = args[self.sym_shape_indices[0]]
@@ -443,28 +446,28 @@ class PiecewiseBackend:
446
entry.runnable = wrap_inductor(
447
self.graph,
448
args,
446
- self.compilation_configs.inductor_compile_config,
447
- self.compilation_configs,
449
+ self.compilation_config.inductor_compile_config,
450
+ self.compilation_config,
451
graph_index=self.piecewise_compile_index,
452
num_graphs=self.total_piecewise_compiles,
453
runtime_shape=runtime_shape,
451
- use_inductor=self.compilation_configs.use_inductor)
454
+ use_inductor=self.compilation_config.use_inductor)
455
456
# finished compilations for all required shapes
457
if self.is_last_graph and not self.to_be_compiled_sizes:
455
- end_monitoring_torch_compile(self.compilation_configs)
458
+ end_monitoring_torch_compile(self.vllm_config)
459
460
if not entry.use_cudagraph:
461
return entry.runnable(*args)
462
463
if entry.cudagraph is None:
461
- if entry.num_finished_warmup < self.compilation_configs.cudagraph_num_of_warmups: # noqa
464
+ if entry.num_finished_warmup < self.compilation_config.cudagraph_num_of_warmups: # noqa
465
entry.num_finished_warmup += 1
466
if self.is_first_graph:
467
logger.debug(
468
"Warming up %s/%s for shape %s",
469
entry.num_finished_warmup,
467
- self.compilation_configs.cudagraph_num_of_warmups,
470
+ self.compilation_config.cudagraph_num_of_warmups,
471
runtime_shape)
472
return entry.runnable(*args)
473
vllm/compilation/decorators.py
+1
-1
@@ -185,7 +185,7 @@ def _support_torch_compile(
185
"Unsupported dynamic dimensions"
186
f" {dims} for argument {k} with type {type(arg)}.")
187
# here, it is the starting point of the `torch.compile` process
188
- start_monitoring_torch_compile(self.vllm_config.compilation_config)
188
+ start_monitoring_torch_compile(self.vllm_config)
189
190
# if we don't use custom dispatcher, we can directly call the
191
# compiled function and let torch.compile handle the dispatching,
vllm/compilation/monitor.py
+20
-3
@@ -1,19 +1,36 @@
1
+import os
2
import time
3
3
-from vllm.config import CompilationConfig, CompilationLevel
4
+from vllm.config import CompilationConfig, CompilationLevel, VllmConfig
5
from vllm.logger import init_logger
6
7
logger = init_logger(__name__)
8
9
+context_manager = None
10
torch_compile_start_time: float = 0.0
11
12
11
-def start_monitoring_torch_compile(compilation_config: CompilationConfig):
13
+def start_monitoring_torch_compile(vllm_config: VllmConfig):
14
global torch_compile_start_time
15
torch_compile_start_time = time.time()
16
17
+ compilation_config: CompilationConfig = vllm_config.compilation_config
18
+ if compilation_config.level == CompilationLevel.PIECEWISE and \
19
+ compilation_config.debug_dump_path:
20
+ import depyf
21
+ path = os.path.join(compilation_config.debug_dump_path,
22
+ f"rank_{vllm_config.parallel_config.rank}")
23
+ global context_manager
24
+ context_manager = depyf.prepare_debug(path)
25
+ context_manager.__enter__()
26
16
-def end_monitoring_torch_compile(compilation_config: CompilationConfig):
27
+
28
+def end_monitoring_torch_compile(vllm_config: VllmConfig):
29
+ compilation_config: CompilationConfig = vllm_config.compilation_config
30
if compilation_config.level == CompilationLevel.PIECEWISE:
31
logger.info("torch.compile takes %.2f s in total",
32
compilation_config.compilation_time)
33
+ global context_manager
34
+ if context_manager is not None:
35
+ context_manager.__exit__(None, None, None)
36
+ context_manager = None
vllm/compilation/wrapper.py
+2
-2
@@ -32,8 +32,8 @@ class TorchCompileWrapperWithCustomDispatcher:
32
# default compilation settings
33
# compiling the forward method
34
35
- backend = get_current_vllm_config(
36
- ).compilation_config.init_backend()
35
+ vllm_config = get_current_vllm_config()
36
+ backend = vllm_config.compilation_config.init_backend(vllm_config)
37
38
compiled_callable = torch.compile(
39
self.forward,
vllm/config.py
+4
-2
@@ -2222,6 +2222,7 @@ class CompilationConfig(BaseModel):
2222
- 1: dynamo as is.
2223
- 2: dynamo once.
2224
- 3: piecewise compilation.
2225
+ - debug_dump_path: the path to dump the debug information.
2226
- backend: the backend for compilation. It needs to be a string.
2227
- "" (empty string): use the default backend.
2228
- "eager"/"openxla"/...: use the specified backend registered in PyTorch.
@@ -2289,6 +2290,7 @@ class CompilationConfig(BaseModel):
2290
certain small batchsizes, where inductor is good at optimizing.
2291
""" # noqa
2292
level: int = 0
2293
+ debug_dump_path: str = ""
2294
backend: str = ""
2295
custom_ops: List[str] = Field(default_factory=list)
2296
splitting_ops: List[str] = Field(default_factory=lambda: [
@@ -2394,7 +2396,7 @@ class CompilationConfig(BaseModel):
2396
self.static_forward_context = {}
2397
self.compilation_time = 0.0
2398
2397
- def init_backend(self) -> Union[str, Callable]:
2399
+ def init_backend(self, vllm_config: "VllmConfig") -> Union[str, Callable]:
2400
if self.level == CompilationLevel.NO_COMPILATION:
2401
raise ValueError("No compilation level is set.")
2402
@@ -2413,7 +2415,7 @@ class CompilationConfig(BaseModel):
2415
# merge with the config use_inductor
2416
assert self.level == CompilationLevel.PIECEWISE
2417
from vllm.compilation.backends import VllmBackend
2416
- return VllmBackend(self)
2418
+ return VllmBackend(vllm_config)
2419
2420
def init_with_cudagraph_sizes(self, sizes_to_specialize: List[int]):
2421
"""To complete the initialization of config,
vllm/worker/model_runner.py
+2
-1
@@ -1162,7 +1162,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1162
1163
if self.vllm_config.compilation_config.level ==\
1164
CompilationLevel.DYNAMO_AS_IS and supports_dynamo():
1165
- backend = self.vllm_config.compilation_config.init_backend()
1165
+ backend = self.vllm_config.compilation_config.init_backend(
1166
+ self.vllm_config)
1167
self.model = torch.compile(
1168
self.model,
1169
fullgraph=envs.VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE,