A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[torch.compile] transparent compilation with more logging (#12246)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Jan 21, 2025 at 19:32 UTC
c81081fece240736e30e0f9a5ed82bb5b483c561
4 files changed
+50
-7
vllm/compilation/backends.py
+25
-7
@@ -524,6 +524,7 @@ class VllmBackend:
524
525
def __call__(self, graph: fx.GraphModule, example_inputs) -> Callable:
526
527
+ vllm_config = self.vllm_config
528
if not self.compilation_config.cache_dir:
529
# no provided cache dir, generate one based on the known factors
530
# that affects the compilation. if none of the factors change,
@@ -532,7 +533,6 @@ class VllmBackend:
533
534
# 1. factors come from the vllm_config (it mainly summarizes how the
535
# model is created)
535
- vllm_config = self.vllm_config
536
config_hash = vllm_config.compute_hash()
537
538
# 2. factors come from the code files that are traced by Dynamo (
@@ -556,20 +556,26 @@ class VllmBackend:
556
hash_key = hashlib.md5(
557
f"{config_hash}_{code_hash}".encode()).hexdigest()[:10]
558
cache_dir = os.path.join(
559
- envs.VLLM_CACHE_ROOT, "torch_compile_cache", hash_key,
560
- f"rank_{vllm_config.parallel_config.rank}")
561
- else:
562
- cache_dir = self.compilation_config.cache_dir
559
+ envs.VLLM_CACHE_ROOT,
560
+ "torch_compile_cache",
561
+ hash_key,
562
+ )
563
+ self.compilation_config.cache_dir = cache_dir
564
+
565
+ cache_dir = self.compilation_config.cache_dir
566
os.makedirs(cache_dir, exist_ok=True)
567
+ local_cache_dir = os.path.join(
568
+ cache_dir, f"rank_{vllm_config.parallel_config.rank}")
569
+ self.compilation_config.local_cache_dir = local_cache_dir
570
571
disabled = envs.VLLM_DISABLE_COMPILE_CACHE
572
self.inductor_hash_cache: InductorHashCache = InductorHashCache(
567
- cache_dir, disabled=disabled)
573
+ local_cache_dir, disabled=disabled)
574
if disabled:
575
logger.info("vLLM's torch.compile cache is disabled.")
576
else:
577
logger.info("Using cache directory: %s for vLLM's torch.compile",
572
- cache_dir)
578
+ local_cache_dir)
579
580
# when dynamo calls the backend, it means the bytecode
581
# transform and analysis are done
@@ -609,6 +615,18 @@ class VllmBackend:
615
self.vllm_config, self.graph_pool,
616
self).run(*example_inputs)
617
618
+ graph_path = os.path.join(local_cache_dir, "computation_graph.py")
619
+ if not os.path.exists(graph_path):
620
+ # code adapted from https://github.com/thuml/depyf/blob/dab831108a752d1facc00acdd6d4243891845c37/depyf/explain/patched_lazy_format_graph_code.py#L30 # noqa
621
+ # use `print_readable` because it can include submodules
622
+ src = "from __future__ import annotations\nimport torch\n" + \
623
+ self.split_gm.print_readable(print_output=False)
624
+ src = src.replace("<lambda>", "GraphModule")
625
+ with open(graph_path, "w") as f:
626
+ f.write(src)
627
+
628
+ logger.debug("Computation graph saved to %s", graph_path)
629
+
630
self._called = True
631
632
if not self.compilation_config.use_cudagraph or \
vllm/compilation/decorators.py
+2
@@ -198,6 +198,8 @@ def _support_torch_compile(
198
f" {dims} for argument {k} with type {type(arg)}.")
199
# here, it is the starting point of the `torch.compile` process
200
start_monitoring_torch_compile(self.vllm_config)
201
+ logger.debug("Start compiling function %s",
202
+ self.original_code_object)
203
204
# if we don't use custom dispatcher, we can directly call the
205
# compiled function and let torch.compile handle the dispatching,
vllm/compilation/wrapper.py
+22
@@ -9,6 +9,9 @@ import torch
9
10
import vllm.envs as envs
11
from vllm.config import CompilationLevel, get_current_vllm_config
12
+from vllm.logger import init_logger
13
+
14
+logger = init_logger(__name__)
15
16
17
class TorchCompileWrapperWithCustomDispatcher:
@@ -82,6 +85,25 @@ class TorchCompileWrapperWithCustomDispatcher:
85
return
86
87
self.compiled_codes.append(new_code)
88
+ local_cache_dir = self.vllm_config.compilation_config.local_cache_dir
89
+ if isinstance(local_cache_dir, str):
90
+ decompiled_file = os.path.join(local_cache_dir,
91
+ "transformed_code.py")
92
+ if not os.path.exists(decompiled_file):
93
+ try:
94
+ # usually the decompilation will succeed for most models,
95
+ # as we guarantee a full-graph compilation in Dynamo.
96
+ # but there's no 100% guarantee, since decompliation is
97
+ # not a reversible process.
98
+ import depyf
99
+ src = depyf.decompile(new_code)
100
+ with open(decompiled_file, "w") as f:
101
+ f.write(src)
102
+
103
+ logger.debug("Dynamo transformed code saved to %s",
104
+ decompiled_file)
105
+ except Exception:
106
+ pass
107
108
if self.vllm_config.compilation_config.use_cudagraph and \
109
"update" in new_code.co_names:
vllm/config.py
+1
@@ -2785,6 +2785,7 @@ class CompilationConfig(BaseModel):
2785
compile_sizes: List[int] = PrivateAttr
2786
capture_sizes: List[int] = PrivateAttr
2787
max_capture_size: int = PrivateAttr
2788
+ local_cache_dir: str = PrivateAttr # local cache dir for each rank
2789
# optimization:
2790
# Intuitively, bs_to_padded_graph_size should be Dict[int, int].
2791
# since we know all keys are in a range [0, max_capture_size],