@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e66faf4809
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[torch.compile] store inductor compiled Python file (#12182)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Jan 19, 2025 at 16:27 UTC e66faf4809cebf0b2169887151f782fd99bf208f
2 files changed +60 -33
vllm/compilation/backends.py
+58 -22
@@ -25,23 +25,30 @@ from .pass_manager import PostGradPassManager
25 logger = init_logger(__name__)
26
27
28 +@dataclasses.dataclass
29 +class InductorArtifact:
30 + hash_str: str = ""
31 + file_path: str = ""
32 +
33 +
34 class InductorHashCache:
35 """
36 Disk format: a Python list of tuples, each tuple is
31 - (runtime_shape, graph_index, hash_str)
37 + (runtime_shape, graph_index, hash_str, file_path)
38 We use list of tuple for readability.
39
40 In-memory format: a defaultdict of dict, where the key is
41 runtime_shape, and the value is a dict of graph_index to hash_str.
42
37 - The data is essentially `Dict[Optional[int], Dict[int, str]]`,
43 + The data is essentially `Dict[Optional[int], Dict[int, InductorArtifact]]`,
44 we don't use json here because json doesn't support int as key.
45
46 TODO: better off-the-shelf solution to serialize the data?
47 """
48
49 def __init__(self, cache_dir: str, disabled: bool = False):
44 - self.cache: defaultdict = defaultdict(dict)
50 + self.cache: Dict[Optional[int],
51 + Dict[int, InductorArtifact]] = defaultdict(dict)
52 self.disabled = disabled
53 self.cache_dir = cache_dir
54 self.cache_file_path = os.path.join(cache_dir,
@@ -66,14 +73,25 @@ class InductorHashCache:
73 # because it is a safe way to parse Python literals.
74 # do not use eval(), it is unsafe.
75 list_data = ast.literal_eval(data)
69 - for runtime_shape, graph_index, hash_str in list_data:
70 - self.cache[runtime_shape][graph_index] = hash_str
76 + for item in list_data:
77 + runtime_shape = item[0]
78 + graph_index = item[1]
79 + hash_str = item[2]
80 + # for compatibility of old version,
81 + # where we don't have file_path.
82 + # NOTE: after running the new code, the file_path
83 + # will be updated.
84 + file_path = "" if len(item) == 3 else item[3]
85 + self.cache[runtime_shape][graph_index] = InductorArtifact(
86 + hash_str=hash_str, file_path=file_path)
87
88 def serialize(self) -> str:
89 data = []
74 - for runtime_shape, graph_index_to_hash_str in self.cache.items():
75 - for graph_index, hash_str in graph_index_to_hash_str.items():
76 - data.append((runtime_shape, graph_index, hash_str))
90 + for runtime_shape, value in self.cache.items():
91 + for graph_index, inductor_artifact in value.items():
92 + data.append(
93 + (runtime_shape, graph_index, inductor_artifact.hash_str,
94 + inductor_artifact.file_path))
95 printer = pprint.PrettyPrinter(indent=4)
96 return printer.pformat(data)
97
@@ -90,13 +108,14 @@ class InductorHashCache:
108 return runtime_shape in self.cache and graph_index in self.cache[
109 runtime_shape]
110
93 - def __getitem__(self, key: Tuple[Optional[int], int]) -> str:
111 + def __getitem__(self, key: Tuple[Optional[int], int]) -> InductorArtifact:
112 if self.disabled:
113 raise KeyError("cannot read from disabled cache")
114 runtime_shape, graph_index = key
115 return self.cache[runtime_shape][graph_index]
116
99 - def __setitem__(self, key: Tuple[Optional[int], int], value: str):
117 + def __setitem__(self, key: Tuple[Optional[int], int],
118 + value: InductorArtifact):
119 # setitem for disabled cache is fine, because we
120 # don't actually write to the disk
121 runtime_shape, graph_index = key
@@ -181,7 +200,8 @@ def wrap_inductor(graph: fx.GraphModule,
200 if (runtime_shape, graph_index) in cache_data:
201 # we compiled this graph before
202 # so we can directly lookup the compiled graph via hash
184 - hash_str = cache_data[(runtime_shape, graph_index)]
203 + inductor_artifact = cache_data[(runtime_shape, graph_index)]
204 + hash_str = inductor_artifact.hash_str
205 if graph_index == 0:
206 # adds some info logging for the first graph
207 logger.info(
@@ -199,6 +219,7 @@ def wrap_inductor(graph: fx.GraphModule,
219 "Inductor cache lookup failed. Please remove"
220 f"the cache file {cache_data.cache_file_path} and try again." # noqa
221 )
222 + inductor_artifact.file_path = inductor_compiled_graph.current_callable.__code__.co_filename # noqa
223
224 # Inductor calling convention (function signature):
225 # f(list) -> tuple
@@ -224,19 +245,20 @@ def wrap_inductor(graph: fx.GraphModule,
245 # the assumption is that we don't have nested Inductor compilation.
246 # compiled_fx_graph_hash will only be called once, and we can hook
247 # it to get the hash of the compiled graph directly.
227 - from torch._inductor.codecache import compiled_fx_graph_hash
248 +
249 + inductor_artifact = InductorArtifact()
250 + from torch._inductor.codecache import (FxGraphCache,
251 + compiled_fx_graph_hash)
252 + original_load = FxGraphCache.load
253 +
254 + def hijack_load(*args, **kwargs):
255 + inductor_compiled_graph = original_load(*args, **kwargs)
256 + inductor_artifact.file_path = inductor_compiled_graph.current_callable.__code__.co_filename # noqa
257 + return inductor_compiled_graph
258
259 def hijack_compiled_fx_graph_hash(*args, **kwargs):
260 out = compiled_fx_graph_hash(*args, **kwargs)
231 - # store the hash in the cache
232 - nonlocal cache_data
233 - cache_data[(runtime_shape, graph_index)] = out[0]
234 - if graph_index == 0:
235 - # adds some info logging for the first graph
236 - logger.info("Cache the graph of shape %s for later use",
237 - str(runtime_shape))
238 - logger.debug("store the %s-th graph for shape %s via hash %s",
239 - graph_index, str(runtime_shape), out[0])
261 + inductor_artifact.hash_str = out[0]
262 return out
263
264 def _check_can_cache(*args, **kwargs):
@@ -255,6 +277,11 @@ def wrap_inductor(graph: fx.GraphModule,
277 if not cache_data.disabled:
278 # compilation cache is enabled, patch several functions
279
280 + # hijack to get the compiled graph itself
281 + stack.enter_context(
282 + patch("torch._inductor.codecache.FxGraphCache.load",
283 + hijack_load))
284 +
285 # for hijacking the hash of the compiled graph
286 stack.enter_context(
287 patch("torch._inductor.codecache.compiled_fx_graph_hash",
@@ -275,7 +302,16 @@ def wrap_inductor(graph: fx.GraphModule,
302 compiled_graph = compile_fx(graph,
303 example_inputs,
304 config_patches=current_config)
278 -
305 + # store the inductor_artifact in the cache
306 + cache_data[(runtime_shape, graph_index)] = inductor_artifact
307 + if graph_index == 0:
308 + # adds some info logging for the first graph
309 + logger.info("Cache the graph of shape %s for later use",
310 + str(runtime_shape))
311 + logger.debug(
312 + "store the %s-th graph for shape %s via hash %s from file %s",
313 + graph_index, str(runtime_shape), inductor_artifact.hash_str,
314 + inductor_artifact.file_path)
315 # after compiling the last graph, record the end time
316 if graph_index == num_graphs - 1:
317 now = time.time()
vllm/config.py
+2 -11
@@ -2862,17 +2862,8 @@ class CompilationConfig(BaseModel):
2862 "vllm.unified_attention_with_output",
2863 ]
2864 else:
2865 - # v0 can use full graph compilation without splitting,
2866 - # splitting is optional.
2867 - # right now we still need it. kv cache shape
2868 - # will be included in the graph if we don't split
2869 - # the graph.
2870 - # TODO: hide kv cache in static forward context
2871 - # so that inductor does not see it.
2872 - self.splitting_ops = [
2873 - "vllm.unified_attention",
2874 - "vllm.unified_attention_with_output",
2875 - ]
2865 + # v0 uses full graph compilation
2866 + self.splitting_ops = []
2867
2868 for k, v in self.inductor_passes.items():
2869 if not isinstance(v, str):