@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c81081fece
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[torch.compile] transparent compilation with more logging (#12246)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Jan 21, 2025 at 19:32 UTC c81081fece240736e30e0f9a5ed82bb5b483c561
4 files changed +50 -7
vllm/compilation/backends.py
+25 -7
@@ -524,6 +524,7 @@ class VllmBackend:
524
525 def __call__(self, graph: fx.GraphModule, example_inputs) -> Callable:
526
527 + vllm_config = self.vllm_config
528 if not self.compilation_config.cache_dir:
529 # no provided cache dir, generate one based on the known factors
530 # that affects the compilation. if none of the factors change,
@@ -532,7 +533,6 @@ class VllmBackend:
533
534 # 1. factors come from the vllm_config (it mainly summarizes how the
535 # model is created)
535 - vllm_config = self.vllm_config
536 config_hash = vllm_config.compute_hash()
537
538 # 2. factors come from the code files that are traced by Dynamo (
@@ -556,20 +556,26 @@ class VllmBackend:
556 hash_key = hashlib.md5(
557 f"{config_hash}_{code_hash}".encode()).hexdigest()[:10]
558 cache_dir = os.path.join(
559 - envs.VLLM_CACHE_ROOT, "torch_compile_cache", hash_key,
560 - f"rank_{vllm_config.parallel_config.rank}")
561 - else:
562 - cache_dir = self.compilation_config.cache_dir
559 + envs.VLLM_CACHE_ROOT,
560 + "torch_compile_cache",
561 + hash_key,
562 + )
563 + self.compilation_config.cache_dir = cache_dir
564 +
565 + cache_dir = self.compilation_config.cache_dir
566 os.makedirs(cache_dir, exist_ok=True)
567 + local_cache_dir = os.path.join(
568 + cache_dir, f"rank_{vllm_config.parallel_config.rank}")
569 + self.compilation_config.local_cache_dir = local_cache_dir
570
571 disabled = envs.VLLM_DISABLE_COMPILE_CACHE
572 self.inductor_hash_cache: InductorHashCache = InductorHashCache(
567 - cache_dir, disabled=disabled)
573 + local_cache_dir, disabled=disabled)
574 if disabled:
575 logger.info("vLLM's torch.compile cache is disabled.")
576 else:
577 logger.info("Using cache directory: %s for vLLM's torch.compile",
572 - cache_dir)
578 + local_cache_dir)
579
580 # when dynamo calls the backend, it means the bytecode
581 # transform and analysis are done
@@ -609,6 +615,18 @@ class VllmBackend:
615 self.vllm_config, self.graph_pool,
616 self).run(*example_inputs)
617
618 + graph_path = os.path.join(local_cache_dir, "computation_graph.py")
619 + if not os.path.exists(graph_path):
620 + # code adapted from https://github.com/thuml/depyf/blob/dab831108a752d1facc00acdd6d4243891845c37/depyf/explain/patched_lazy_format_graph_code.py#L30 # noqa
621 + # use `print_readable` because it can include submodules
622 + src = "from __future__ import annotations\nimport torch\n" + \
623 + self.split_gm.print_readable(print_output=False)
624 + src = src.replace("<lambda>", "GraphModule")
625 + with open(graph_path, "w") as f:
626 + f.write(src)
627 +
628 + logger.debug("Computation graph saved to %s", graph_path)
629 +
630 self._called = True
631
632 if not self.compilation_config.use_cudagraph or \
vllm/compilation/decorators.py
+2
@@ -198,6 +198,8 @@ def _support_torch_compile(
198 f" {dims} for argument {k} with type {type(arg)}.")
199 # here, it is the starting point of the `torch.compile` process
200 start_monitoring_torch_compile(self.vllm_config)
201 + logger.debug("Start compiling function %s",
202 + self.original_code_object)
203
204 # if we don't use custom dispatcher, we can directly call the
205 # compiled function and let torch.compile handle the dispatching,
vllm/compilation/wrapper.py
+22
@@ -9,6 +9,9 @@ import torch
9
10 import vllm.envs as envs
11 from vllm.config import CompilationLevel, get_current_vllm_config
12 +from vllm.logger import init_logger
13 +
14 +logger = init_logger(__name__)
15
16
17 class TorchCompileWrapperWithCustomDispatcher:
@@ -82,6 +85,25 @@ class TorchCompileWrapperWithCustomDispatcher:
85 return
86
87 self.compiled_codes.append(new_code)
88 + local_cache_dir = self.vllm_config.compilation_config.local_cache_dir
89 + if isinstance(local_cache_dir, str):
90 + decompiled_file = os.path.join(local_cache_dir,
91 + "transformed_code.py")
92 + if not os.path.exists(decompiled_file):
93 + try:
94 + # usually the decompilation will succeed for most models,
95 + # as we guarantee a full-graph compilation in Dynamo.
96 + # but there's no 100% guarantee, since decompliation is
97 + # not a reversible process.
98 + import depyf
99 + src = depyf.decompile(new_code)
100 + with open(decompiled_file, "w") as f:
101 + f.write(src)
102 +
103 + logger.debug("Dynamo transformed code saved to %s",
104 + decompiled_file)
105 + except Exception:
106 + pass
107
108 if self.vllm_config.compilation_config.use_cudagraph and \
109 "update" in new_code.co_names:
vllm/config.py
+1
@@ -2785,6 +2785,7 @@ class CompilationConfig(BaseModel):
2785 compile_sizes: List[int] = PrivateAttr
2786 capture_sizes: List[int] = PrivateAttr
2787 max_capture_size: int = PrivateAttr
2788 + local_cache_dir: str = PrivateAttr # local cache dir for each rank
2789 # optimization:
2790 # Intuitively, bs_to_padded_graph_size should be Dict[int, int].
2791 # since we know all keys are in a range [0, max_capture_size],