@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / aff1fd8188
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[torch.compile] use interpreter with stable api from pytorch (#9889)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 1, 2024 at 11:50 UTC aff1fd81881bf29f82ad6ba55b301828764cd120
1 file changed +89 -76
vllm/compilation/backends.py
+89 -76
@@ -243,6 +243,65 @@ def split_graph(graph: fx.GraphModule,
243 return split_gm, outputs
244
245
246 +# we share the global graph pool among all the backends
247 +global_graph_pool = None
248 +
249 +
250 +class PiecewiseCompileInterpreter(torch.fx.Interpreter):
251 + """Code adapted from `torch.fx.passes.shape_prop.ShapeProp`.
252 + It runs the given graph with fake inputs, and compile some
253 + submodules specified by `compile_submod_names` with the given
254 + compilation configs.
255 + """
256 +
257 + def __init__(self, module: torch.fx.GraphModule,
258 + compile_submod_names: List[str],
259 + compilation_configs: CompilationConfig, graph_pool):
260 + super().__init__(module)
261 + from torch._guards import detect_fake_mode
262 + self.fake_mode = detect_fake_mode()
263 + self.compile_submod_names = compile_submod_names
264 + self.compilation_configs = compilation_configs
265 + self.graph_pool = graph_pool
266 + self.have_seen_first_graph = False
267 +
268 + def run(self, *args):
269 + fake_args = [
270 + self.fake_mode.from_tensor(t) if isinstance(t, torch.Tensor) else t
271 + for t in args
272 + ]
273 + return super().run(*fake_args)
274 +
275 + def call_module(self, target: torch.fx.node.Target,
276 + args: Tuple[torch.fx.node.Argument,
277 + ...], kwargs: Dict[str, Any]) -> Any:
278 + assert isinstance(target, str)
279 + output = super().call_module(target, args, kwargs)
280 +
281 + if target in self.compile_submod_names:
282 + submod = self.fetch_attr(target)
283 + sym_shape_indices = [
284 + i for i, x in enumerate(args) if isinstance(x, torch.SymInt)
285 + ]
286 + compiled_graph_for_general_shape = wrap_inductor(
287 + submod,
288 + args,
289 + self.compilation_configs.inductor_compile_config,
290 + runtime_shape=None,
291 + do_logging=not self.have_seen_first_graph,
292 + use_inductor=self.compilation_configs.use_inductor)
293 +
294 + self.module.__dict__[target] = PiecewiseBackend(
295 + submod, self.compilation_configs, self.graph_pool,
296 + not self.have_seen_first_graph, sym_shape_indices,
297 + compiled_graph_for_general_shape)
298 +
299 + self.have_seen_first_graph = True
300 + compilation_counter.num_piecewise_capturable_graphs_seen += 1
301 +
302 + return output
303 +
304 +
305 class VllmBackend:
306 """The compilation backend for `torch.compile` with VLLM.
307 It is used for compilation level of `CompilationLevel.PIECEWISE`,
@@ -263,8 +322,14 @@ class VllmBackend:
322 returned_callable: Callable
323
324 def __init__(self, ):
266 - # every instance of VllmBackend has its own graph pool
267 - self.graph_pool = torch.cuda.graph_pool_handle()
325 + global global_graph_pool
326 + if global_graph_pool is None:
327 + global_graph_pool = torch.cuda.graph_pool_handle()
328 +
329 + # TODO: in the future, if we want to use multiple
330 + # streams, it might not be safe to share a global pool.
331 + # only investigate this when we use multiple streams
332 + self.graph_pool = global_graph_pool
333
334 # `torch.compile` is JIT compiled, so we don't need to
335 # do anything here
@@ -286,55 +351,26 @@ class VllmBackend:
351 self.split_gm, self.piecewise_graphs = split_graph(
352 graph, self.compilation_configs.non_cudagraph_ops)
353
289 - returned_callable: Callable # type: ignore
354 + from torch._dynamo.utils import lazy_format_graph_code
355 + logger.debug("%s",
356 + lazy_format_graph_code("stiching module", self.split_gm))
357
291 - if len(self.piecewise_graphs) == 0:
292 - compilation_counter.num_piecewise_graphs_seen += 1
293 - compilation_counter.num_piecewise_capturable_graphs_seen += 1
294 - returned_callable = PiecewiseBackend(graph,
295 - self.compilation_configs,
296 - self.graph_pool,
297 - is_first_graph=True)
298 - else:
299 - from torch._dynamo.utils import lazy_format_graph_code
300 - logger.debug(
301 - "%s", lazy_format_graph_code("stiching module", self.split_gm))
302 -
303 - is_first_graph = True
304 -
305 - for item in self.piecewise_graphs:
306 - compilation_counter.num_piecewise_graphs_seen += 1
307 - compilation_counter.num_piecewise_capturable_graphs_seen += not item.is_splitting_graph # noqa
308 - if not item.is_splitting_graph:
309 - # cannot setattr to a module, so we need to set
310 - # the attribute in the __dict__
311 - self.split_gm.__dict__[
312 - item.submod_name] = PiecewiseBackend(
313 - item.graph, self.compilation_configs,
314 - self.graph_pool, is_first_graph)
315 - is_first_graph = False
316 - returned_callable = self.split_gm
317 -
318 - self.returned_callable = returned_callable
319 - # trigger the first compilation
320 - # code borrowed from https://github.com/pytorch/pytorch/blob/4e3e08b71171fa34172b2362ff668553fac75f27/torch/_dynamo/backends/distributed.py#L206 # noqa
321 - # to turn the inputs into fake tensors
322 - import torch._guards
323 - from torch._guards import detect_fake_mode
324 - fake_mode = detect_fake_mode(example_inputs)
325 - fake_args = []
326 - for arg in example_inputs:
327 - if isinstance(arg, torch.Tensor) and not isinstance(
328 - arg, torch._subclasses.FakeTensor):
329 - fake_args.append(
330 - torch._dynamo.utils.to_fake_tensor(arg, fake_mode))
331 - else:
332 - fake_args.append(arg)
333 - self.returned_callable(*fake_args)
358 + compilation_counter.num_piecewise_graphs_seen += len(
359 + self.piecewise_graphs)
360 + submod_names_to_compile = [
361 + item.submod_name for item in self.piecewise_graphs
362 + if not item.is_splitting_graph
363 + ]
364 +
365 + # propagate the split graph to the piecewise backend,
366 + # compile submodules with symbolic shapes
367 + PiecewiseCompileInterpreter(self.split_gm, submod_names_to_compile,
368 + self.compilation_configs,
369 + self.graph_pool).run(*example_inputs)
370
371 self._called = True
372
337 - return self.returned_callable
373 + return self.split_gm
374
375
376 @dataclasses.dataclass
@@ -352,11 +388,10 @@ class ConcreteSizeEntry:
388
389 class PiecewiseBackend:
390
355 - def __init__(self,
356 - graph: fx.GraphModule,
357 - compilation_configs: CompilationConfig,
358 - graph_pool: Any,
359 - is_first_graph: bool = False):
391 + def __init__(self, graph: fx.GraphModule,
392 + compilation_configs: CompilationConfig, graph_pool: Any,
393 + is_first_graph: bool, sym_shape_indices: List[int],
394 + compiled_graph_for_general_shape: Callable):
395 """
396 The backend for piecewise compilation.
397 It mainly handles the compilation and cudagraph capturing.
@@ -381,12 +416,11 @@ class PiecewiseBackend:
416 self.compilation_configs.capture_sizes
417 ) if self.compilation_configs.use_cudagraph else set()
418
384 - self.compile_finished = False
419 self.first_run_finished = False
420
387 - self.compiled_graph_for_general_shape: Callable = None # type: ignore
421 + self.compiled_graph_for_general_shape = compiled_graph_for_general_shape # noqa
422
389 - self.sym_shape_indices: List[int] = []
423 + self.sym_shape_indices = sym_shape_indices
424
425 # the entries for different shapes that we need to either
426 # compile or capture cudagraph
@@ -399,27 +433,6 @@ class PiecewiseBackend:
433 )
434
435 def __call__(self, *args) -> Any:
402 -
403 - if not self.compile_finished:
404 - self.compile_finished = True
405 -
406 - # this is the first compilation, we will compile a graph with
407 - # dynamic shape, as the caller will mark first dimension as dynamic
408 -
409 - self.sym_shape_indices = [
410 - i for i, x in enumerate(args) if isinstance(x, torch.SymInt)
411 - ]
412 -
413 - self.compiled_graph_for_general_shape = wrap_inductor(
414 - self.graph,
415 - args,
416 - self.compilation_configs.inductor_compile_config,
417 - runtime_shape=None,
418 - do_logging=self.is_first_graph,
419 - use_inductor=self.compilation_configs.use_inductor)
420 -
421 - return self.graph(*args)
422 -
436 if not self.first_run_finished:
437 self.first_run_finished = True
438 return self.compiled_graph_for_general_shape(*args)