@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f12141170a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[torch.compile] consider relevant code in compilation cache (#11614)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Jan 8, 2025 at 18:46 UTC f12141170a95ad866b3c55762623bc718994e1d7
4 files changed +99 -35
vllm/compilation/backends.py
+62 -8
@@ -145,6 +145,7 @@ def wrap_inductor(graph: fx.GraphModule,
145 example_inputs,
146 additional_inductor_config,
147 compilation_config: CompilationConfig,
148 + vllm_backend: "VllmBackend",
149 graph_index: int = 0,
150 num_graphs: int = 1,
151 runtime_shape: Optional[int] = None,
@@ -176,7 +177,7 @@ def wrap_inductor(graph: fx.GraphModule,
177 # see https://github.com/pytorch/pytorch/issues/138980
178 graph = copy.deepcopy(graph)
179
179 - cache_data = compilation_config.inductor_hash_cache
180 + cache_data = vllm_backend.inductor_hash_cache
181 if (runtime_shape, graph_index) in cache_data:
182 # we compiled this graph before
183 # so we can directly lookup the compiled graph via hash
@@ -196,7 +197,7 @@ def wrap_inductor(graph: fx.GraphModule,
197 hash_str, example_inputs, True, False)
198 assert inductor_compiled_graph is not None, (
199 "Inductor cache lookup failed. Please remove"
199 - f"the cache file {compilation_config.inductor_hash_cache.cache_file_path} and try again." # noqa
200 + f"the cache file {cache_data.cache_file_path} and try again." # noqa
201 )
202
203 # Inductor calling convention (function signature):
@@ -354,7 +355,7 @@ class PiecewiseCompileInterpreter(torch.fx.Interpreter):
355
356 def __init__(self, module: torch.fx.GraphModule,
357 compile_submod_names: List[str], vllm_config: VllmConfig,
357 - graph_pool):
358 + graph_pool, vllm_backend: "VllmBackend"):
359 super().__init__(module)
360 from torch._guards import detect_fake_mode
361 self.fake_mode = detect_fake_mode()
@@ -362,6 +363,7 @@ class PiecewiseCompileInterpreter(torch.fx.Interpreter):
363 self.compilation_config = vllm_config.compilation_config
364 self.graph_pool = graph_pool
365 self.vllm_config = vllm_config
366 + self.vllm_backend = vllm_backend
367
368 def run(self, *args):
369 fake_args = [
@@ -389,6 +391,7 @@ class PiecewiseCompileInterpreter(torch.fx.Interpreter):
391 args,
392 self.compilation_config.inductor_compile_config,
393 self.compilation_config,
394 + self.vllm_backend,
395 graph_index=index,
396 num_graphs=len(self.compile_submod_names),
397 runtime_shape=None,
@@ -397,7 +400,7 @@ class PiecewiseCompileInterpreter(torch.fx.Interpreter):
400 self.module.__dict__[target] = PiecewiseBackend(
401 submod, self.vllm_config, self.graph_pool, index,
402 len(self.compile_submod_names), sym_shape_indices,
400 - compiled_graph_for_general_shape)
403 + compiled_graph_for_general_shape, self.vllm_backend)
404
405 compilation_counter.num_piecewise_capturable_graphs_seen += 1
406
@@ -430,6 +433,7 @@ class VllmBackend:
433 post_grad_passes: Sequence[Callable]
434 sym_tensor_indices: List[int]
435 input_buffers: List[torch.Tensor]
436 + inductor_hash_cache: InductorHashCache
437
438 def __init__(
439 self,
@@ -472,6 +476,53 @@ class VllmBackend:
476
477 def __call__(self, graph: fx.GraphModule, example_inputs) -> Callable:
478
479 + if not self.compilation_config.cache_dir:
480 + # no provided cache dir, generate one based on the known factors
481 + # that affects the compilation. if none of the factors change,
482 + # the cache dir will be the same so that we can reuse the compiled
483 + # graph.
484 +
485 + # 1. factors come from the vllm_config (it mainly summarizes how the
486 + # model is created)
487 + vllm_config = self.vllm_config
488 + config_hash = vllm_config.compute_hash()
489 +
490 + # 2. factors come from the code files that are traced by Dynamo (
491 + # it mainly summarizes how the model is used in forward pass)
492 + forward_code_files = list(
493 + sorted(self.compilation_config.traced_files))
494 + self.compilation_config.traced_files.clear()
495 + logger.debug(
496 + "Traced files (to be considered for compilation cache):\n%s",
497 + "\n".join(forward_code_files))
498 + hash_content = []
499 + for filepath in forward_code_files:
500 + hash_content.append(filepath)
501 + with open(filepath) as f:
502 + hash_content.append(f.read())
503 + import hashlib
504 + code_hash = hashlib.md5(
505 + "\n".join(hash_content).encode()).hexdigest()
506 +
507 + # combine the two hashes to generate the cache dir
508 + hash_key = hashlib.md5(
509 + f"{config_hash}_{code_hash}".encode()).hexdigest()[:10]
510 + cache_dir = os.path.join(
511 + envs.VLLM_CACHE_ROOT, "torch_compile_cache", hash_key,
512 + f"rank_{vllm_config.parallel_config.rank}")
513 + else:
514 + cache_dir = self.compilation_config.cache_dir
515 + os.makedirs(cache_dir, exist_ok=True)
516 +
517 + disabled = envs.VLLM_DISABLE_COMPILE_CACHE
518 + self.inductor_hash_cache: InductorHashCache = InductorHashCache(
519 + cache_dir, disabled=disabled)
520 + if disabled:
521 + logger.info("vLLM's torch.compile cache is disabled.")
522 + else:
523 + logger.info("Using cache directory: %s for vLLM's torch.compile",
524 + cache_dir)
525 +
526 # when dynamo calls the backend, it means the bytecode
527 # transform and analysis are done
528 compilation_counter.num_graphs_seen += 1
@@ -507,8 +558,8 @@ class VllmBackend:
558 # propagate the split graph to the piecewise backend,
559 # compile submodules with symbolic shapes
560 PiecewiseCompileInterpreter(self.split_gm, submod_names_to_compile,
510 - self.vllm_config,
511 - self.graph_pool).run(*example_inputs)
561 + self.vllm_config, self.graph_pool,
562 + self).run(*example_inputs)
563
564 self._called = True
565
@@ -577,7 +628,8 @@ class PiecewiseBackend:
628 def __init__(self, graph: fx.GraphModule, vllm_config: VllmConfig,
629 graph_pool: Any, piecewise_compile_index: int,
630 total_piecewise_compiles: int, sym_shape_indices: List[int],
580 - compiled_graph_for_general_shape: Callable):
631 + compiled_graph_for_general_shape: Callable,
632 + vllm_backend: VllmBackend):
633 """
634 The backend for piecewise compilation.
635 It mainly handles the compilation and cudagraph capturing.
@@ -597,6 +649,7 @@ class PiecewiseBackend:
649 self.graph_pool = graph_pool
650 self.piecewise_compile_index = piecewise_compile_index
651 self.total_piecewise_compiles = total_piecewise_compiles
652 + self.vllm_backend = vllm_backend
653
654 self.is_first_graph = piecewise_compile_index == 0
655 self.is_last_graph = (
@@ -634,7 +687,7 @@ class PiecewiseBackend:
687 if self.is_last_graph and not self.to_be_compiled_sizes:
688 # no specific sizes to compile
689 # save the hash of the inductor graph for the next run
637 - self.compilation_config.inductor_hash_cache.save_to_file()
690 + self.vllm_backend.inductor_hash_cache.save_to_file()
691 end_monitoring_torch_compile(self.vllm_config)
692
693 def __call__(self, *args) -> Any:
@@ -662,6 +715,7 @@ class PiecewiseBackend:
715 args,
716 self.compilation_config.inductor_compile_config,
717 self.compilation_config,
718 + self.vllm_backend,
719 graph_index=self.piecewise_compile_index,
720 num_graphs=self.total_piecewise_compiles,
721 runtime_shape=runtime_shape,
vllm/compilation/decorators.py
+27 -1
@@ -1,8 +1,10 @@
1 import inspect
2 from typing import Callable, Dict, List, Optional, TypeVar, Union, overload
3 +from unittest.mock import patch
4
5 import torch
6 import torch.nn as nn
7 +from torch._dynamo.symbolic_convert import InliningInstructionTranslator
8
9 from vllm.compilation.counter import compilation_counter
10 from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispatcher
@@ -196,7 +198,31 @@ def _support_torch_compile(
198 # we need to control all the compilation of the model.
199 torch._dynamo.eval_frame.remove_from_cache(
200 self.original_code_object)
199 - return self.compiled_callable(*args, **kwargs)
201 +
202 + # collect all relevant files traced by Dynamo,
203 + # so that the compilation cache can trigger re-compilation
204 + # properly when any of these files change.
205 +
206 + # 1. the file containing the top-level forward function
207 + self.vllm_config.compilation_config.traced_files.add(
208 + self.original_code_object.co_filename)
209 +
210 + # 2. every time Dynamo sees a function call, it will inline
211 + # the function by calling InliningInstructionTranslator.inline_call
212 + # we hijack this function to know all the functions called
213 + # during Dynamo tracing, and their corresponding files
214 + inline_call = InliningInstructionTranslator.inline_call
215 +
216 + def patched_inline_call(parent, func, args, kwargs):
217 + code = func.get_code()
218 + self.vllm_config.compilation_config.traced_files.add(
219 + code.co_filename)
220 + return inline_call(parent, func, args, kwargs)
221 +
222 + with patch.object(InliningInstructionTranslator, 'inline_call',
223 + patched_inline_call):
224 + output = self.compiled_callable(*args, **kwargs)
225 + return output
226
227 # usually, capturing the model once is enough, and then we can
228 # dispatch to the compiled code directly, without going through
vllm/config.py
+3 -26
@@ -3,7 +3,6 @@ import copy
3 import enum
4 import hashlib
5 import json
6 -import os
6 import sys
7 import warnings
8 from contextlib import contextmanager
@@ -2778,9 +2777,8 @@ class CompilationConfig(BaseModel):
2777 # keep track of enabled and disabled custom ops
2778 enabled_custom_ops: Counter[str] = PrivateAttr
2779 disabled_custom_ops: Counter[str] = PrivateAttr
2780 + traced_files: Set[str] = PrivateAttr
2781 compilation_time: float = PrivateAttr
2782 - # should be InductorHashCache, but Pydantic does not support it
2783 - inductor_hash_cache: Any = PrivateAttr
2782
2783 # Per-model forward context
2784 # Mainly used to store attention cls
@@ -2818,6 +2816,7 @@ class CompilationConfig(BaseModel):
2816 "compilation_time",
2817 "bs_to_padded_graph_size",
2818 "pass_config",
2819 + "traced_files",
2820 }
2821 return self.model_dump_json(exclude=exclude, exclude_unset=True)
2822
@@ -2877,6 +2876,7 @@ class CompilationConfig(BaseModel):
2876
2877 self.enabled_custom_ops = Counter()
2878 self.disabled_custom_ops = Counter()
2879 + self.traced_files = set()
2880 self.static_forward_context = {}
2881 self.compilation_time = 0.0
2882
@@ -2899,29 +2899,6 @@ class CompilationConfig(BaseModel):
2899 # merge with the config use_inductor
2900 assert self.level == CompilationLevel.PIECEWISE
2901
2902 - if not self.cache_dir:
2903 - # no provided cache dir, generate one based on the known factors
2904 - # that affects the compilation. if none of the factors change,
2905 - # the cache dir will be the same so that we can reuse the compiled
2906 - # graph.
2907 - hash_key = vllm_config.compute_hash()
2908 - cache_dir = os.path.join(
2909 - envs.VLLM_CACHE_ROOT, "torch_compile_cache", hash_key,
2910 - f"rank_{vllm_config.parallel_config.rank}")
2911 - os.makedirs(cache_dir, exist_ok=True)
2912 - self.cache_dir = cache_dir
2913 -
2914 - disabled = envs.VLLM_DISABLE_COMPILE_CACHE
2915 - from vllm.compilation.backends import InductorHashCache
2916 - self.inductor_hash_cache: InductorHashCache = InductorHashCache(
2917 - self.cache_dir, disabled=disabled)
2918 - if disabled:
2919 - logger.info("vLLM's torch.compile cache is disabled.")
2920 - else:
2921 - logger.info(
2922 - "Using cache directory: %s for vLLM's torch.compile",
2923 - self.cache_dir)
2924 -
2902 from vllm.compilation.backends import VllmBackend
2903 return VllmBackend(vllm_config)
2904
vllm/sequence.py
+7
@@ -1108,6 +1108,13 @@ class IntermediateTensors:
1108
1109 tensors: Dict[str, torch.Tensor]
1110
1111 + def __init__(self, tensors):
1112 + # manually define this function, so that
1113 + # Dynamo knows `IntermediateTensors()` comes from this file.
1114 + # Otherwise, dataclass will generate this function by evaluating
1115 + # a string, and we will lose the information about the source file.
1116 + self.tensors = tensors
1117 +
1118 def __getitem__(self, key: Union[str, slice]):
1119 if isinstance(key, str):
1120 return self.tensors[key]