A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[torch.compile] use interpreter with stable api from pytorch (#9889)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 1, 2024 at 11:50 UTC
aff1fd81881bf29f82ad6ba55b301828764cd120
1 file changed
+89
-76
vllm/compilation/backends.py
+89
-76
@@ -243,6 +243,65 @@ def split_graph(graph: fx.GraphModule,
243
return split_gm, outputs
244
245
246
+# we share the global graph pool among all the backends
247
+global_graph_pool = None
248
+
249
+
250
+class PiecewiseCompileInterpreter(torch.fx.Interpreter):
251
+ """Code adapted from `torch.fx.passes.shape_prop.ShapeProp`.
252
+ It runs the given graph with fake inputs, and compile some
253
+ submodules specified by `compile_submod_names` with the given
254
+ compilation configs.
255
+ """
256
+
257
+ def __init__(self, module: torch.fx.GraphModule,
258
+ compile_submod_names: List[str],
259
+ compilation_configs: CompilationConfig, graph_pool):
260
+ super().__init__(module)
261
+ from torch._guards import detect_fake_mode
262
+ self.fake_mode = detect_fake_mode()
263
+ self.compile_submod_names = compile_submod_names
264
+ self.compilation_configs = compilation_configs
265
+ self.graph_pool = graph_pool
266
+ self.have_seen_first_graph = False
267
+
268
+ def run(self, *args):
269
+ fake_args = [
270
+ self.fake_mode.from_tensor(t) if isinstance(t, torch.Tensor) else t
271
+ for t in args
272
+ ]
273
+ return super().run(*fake_args)
274
+
275
+ def call_module(self, target: torch.fx.node.Target,
276
+ args: Tuple[torch.fx.node.Argument,
277
+ ...], kwargs: Dict[str, Any]) -> Any:
278
+ assert isinstance(target, str)
279
+ output = super().call_module(target, args, kwargs)
280
+
281
+ if target in self.compile_submod_names:
282
+ submod = self.fetch_attr(target)
283
+ sym_shape_indices = [
284
+ i for i, x in enumerate(args) if isinstance(x, torch.SymInt)
285
+ ]
286
+ compiled_graph_for_general_shape = wrap_inductor(
287
+ submod,
288
+ args,
289
+ self.compilation_configs.inductor_compile_config,
290
+ runtime_shape=None,
291
+ do_logging=not self.have_seen_first_graph,
292
+ use_inductor=self.compilation_configs.use_inductor)
293
+
294
+ self.module.__dict__[target] = PiecewiseBackend(
295
+ submod, self.compilation_configs, self.graph_pool,
296
+ not self.have_seen_first_graph, sym_shape_indices,
297
+ compiled_graph_for_general_shape)
298
+
299
+ self.have_seen_first_graph = True
300
+ compilation_counter.num_piecewise_capturable_graphs_seen += 1
301
+
302
+ return output
303
+
304
+
305
class VllmBackend:
306
"""The compilation backend for `torch.compile` with VLLM.
307
It is used for compilation level of `CompilationLevel.PIECEWISE`,
@@ -263,8 +322,14 @@ class VllmBackend:
322
returned_callable: Callable
323
324
def __init__(self, ):
266
- # every instance of VllmBackend has its own graph pool
267
- self.graph_pool = torch.cuda.graph_pool_handle()
325
+ global global_graph_pool
326
+ if global_graph_pool is None:
327
+ global_graph_pool = torch.cuda.graph_pool_handle()
328
+
329
+ # TODO: in the future, if we want to use multiple
330
+ # streams, it might not be safe to share a global pool.
331
+ # only investigate this when we use multiple streams
332
+ self.graph_pool = global_graph_pool
333
334
# `torch.compile` is JIT compiled, so we don't need to
335
# do anything here
@@ -286,55 +351,26 @@ class VllmBackend:
351
self.split_gm, self.piecewise_graphs = split_graph(
352
graph, self.compilation_configs.non_cudagraph_ops)
353
289
- returned_callable: Callable # type: ignore
354
+ from torch._dynamo.utils import lazy_format_graph_code
355
+ logger.debug("%s",
356
+ lazy_format_graph_code("stiching module", self.split_gm))
357
291
- if len(self.piecewise_graphs) == 0:
292
- compilation_counter.num_piecewise_graphs_seen += 1
293
- compilation_counter.num_piecewise_capturable_graphs_seen += 1
294
- returned_callable = PiecewiseBackend(graph,
295
- self.compilation_configs,
296
- self.graph_pool,
297
- is_first_graph=True)
298
- else:
299
- from torch._dynamo.utils import lazy_format_graph_code
300
- logger.debug(
301
- "%s", lazy_format_graph_code("stiching module", self.split_gm))
302
-
303
- is_first_graph = True
304
-
305
- for item in self.piecewise_graphs:
306
- compilation_counter.num_piecewise_graphs_seen += 1
307
- compilation_counter.num_piecewise_capturable_graphs_seen += not item.is_splitting_graph # noqa
308
- if not item.is_splitting_graph:
309
- # cannot setattr to a module, so we need to set
310
- # the attribute in the __dict__
311
- self.split_gm.__dict__[
312
- item.submod_name] = PiecewiseBackend(
313
- item.graph, self.compilation_configs,
314
- self.graph_pool, is_first_graph)
315
- is_first_graph = False
316
- returned_callable = self.split_gm
317
-
318
- self.returned_callable = returned_callable
319
- # trigger the first compilation
320
- # code borrowed from https://github.com/pytorch/pytorch/blob/4e3e08b71171fa34172b2362ff668553fac75f27/torch/_dynamo/backends/distributed.py#L206 # noqa
321
- # to turn the inputs into fake tensors
322
- import torch._guards
323
- from torch._guards import detect_fake_mode
324
- fake_mode = detect_fake_mode(example_inputs)
325
- fake_args = []
326
- for arg in example_inputs:
327
- if isinstance(arg, torch.Tensor) and not isinstance(
328
- arg, torch._subclasses.FakeTensor):
329
- fake_args.append(
330
- torch._dynamo.utils.to_fake_tensor(arg, fake_mode))
331
- else:
332
- fake_args.append(arg)
333
- self.returned_callable(*fake_args)
358
+ compilation_counter.num_piecewise_graphs_seen += len(
359
+ self.piecewise_graphs)
360
+ submod_names_to_compile = [
361
+ item.submod_name for item in self.piecewise_graphs
362
+ if not item.is_splitting_graph
363
+ ]
364
+
365
+ # propagate the split graph to the piecewise backend,
366
+ # compile submodules with symbolic shapes
367
+ PiecewiseCompileInterpreter(self.split_gm, submod_names_to_compile,
368
+ self.compilation_configs,
369
+ self.graph_pool).run(*example_inputs)
370
371
self._called = True
372
337
- return self.returned_callable
373
+ return self.split_gm
374
375
376
@dataclasses.dataclass
@@ -352,11 +388,10 @@ class ConcreteSizeEntry:
388
389
class PiecewiseBackend:
390
355
- def __init__(self,
356
- graph: fx.GraphModule,
357
- compilation_configs: CompilationConfig,
358
- graph_pool: Any,
359
- is_first_graph: bool = False):
391
+ def __init__(self, graph: fx.GraphModule,
392
+ compilation_configs: CompilationConfig, graph_pool: Any,
393
+ is_first_graph: bool, sym_shape_indices: List[int],
394
+ compiled_graph_for_general_shape: Callable):
395
"""
396
The backend for piecewise compilation.
397
It mainly handles the compilation and cudagraph capturing.
@@ -381,12 +416,11 @@ class PiecewiseBackend:
416
self.compilation_configs.capture_sizes
417
) if self.compilation_configs.use_cudagraph else set()
418
384
- self.compile_finished = False
419
self.first_run_finished = False
420
387
- self.compiled_graph_for_general_shape: Callable = None # type: ignore
421
+ self.compiled_graph_for_general_shape = compiled_graph_for_general_shape # noqa
422
389
- self.sym_shape_indices: List[int] = []
423
+ self.sym_shape_indices = sym_shape_indices
424
425
# the entries for different shapes that we need to either
426
# compile or capture cudagraph
@@ -399,27 +433,6 @@ class PiecewiseBackend:
433
)
434
435
def __call__(self, *args) -> Any:
402
-
403
- if not self.compile_finished:
404
- self.compile_finished = True
405
-
406
- # this is the first compilation, we will compile a graph with
407
- # dynamic shape, as the caller will mark first dimension as dynamic
408
-
409
- self.sym_shape_indices = [
410
- i for i, x in enumerate(args) if isinstance(x, torch.SymInt)
411
- ]
412
-
413
- self.compiled_graph_for_general_shape = wrap_inductor(
414
- self.graph,
415
- args,
416
- self.compilation_configs.inductor_compile_config,
417
- runtime_shape=None,
418
- do_logging=self.is_first_graph,
419
- use_inductor=self.compilation_configs.use_inductor)
420
-
421
- return self.graph(*args)
422
-
436
if not self.first_run_finished:
437
self.first_run_finished = True
438
return self.compiled_graph_for_general_shape(*args)