A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[torch.compile] store inductor compiled Python file (#12182)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Jan 19, 2025 at 16:27 UTC
e66faf4809cebf0b2169887151f782fd99bf208f
2 files changed
+60
-33
vllm/compilation/backends.py
+58
-22
@@ -25,23 +25,30 @@ from .pass_manager import PostGradPassManager
25
logger = init_logger(__name__)
26
27
28
+@dataclasses.dataclass
29
+class InductorArtifact:
30
+ hash_str: str = ""
31
+ file_path: str = ""
32
+
33
+
34
class InductorHashCache:
35
"""
36
Disk format: a Python list of tuples, each tuple is
31
- (runtime_shape, graph_index, hash_str)
37
+ (runtime_shape, graph_index, hash_str, file_path)
38
We use list of tuple for readability.
39
40
In-memory format: a defaultdict of dict, where the key is
41
runtime_shape, and the value is a dict of graph_index to hash_str.
42
37
- The data is essentially `Dict[Optional[int], Dict[int, str]]`,
43
+ The data is essentially `Dict[Optional[int], Dict[int, InductorArtifact]]`,
44
we don't use json here because json doesn't support int as key.
45
46
TODO: better off-the-shelf solution to serialize the data?
47
"""
48
49
def __init__(self, cache_dir: str, disabled: bool = False):
44
- self.cache: defaultdict = defaultdict(dict)
50
+ self.cache: Dict[Optional[int],
51
+ Dict[int, InductorArtifact]] = defaultdict(dict)
52
self.disabled = disabled
53
self.cache_dir = cache_dir
54
self.cache_file_path = os.path.join(cache_dir,
@@ -66,14 +73,25 @@ class InductorHashCache:
73
# because it is a safe way to parse Python literals.
74
# do not use eval(), it is unsafe.
75
list_data = ast.literal_eval(data)
69
- for runtime_shape, graph_index, hash_str in list_data:
70
- self.cache[runtime_shape][graph_index] = hash_str
76
+ for item in list_data:
77
+ runtime_shape = item[0]
78
+ graph_index = item[1]
79
+ hash_str = item[2]
80
+ # for compatibility of old version,
81
+ # where we don't have file_path.
82
+ # NOTE: after running the new code, the file_path
83
+ # will be updated.
84
+ file_path = "" if len(item) == 3 else item[3]
85
+ self.cache[runtime_shape][graph_index] = InductorArtifact(
86
+ hash_str=hash_str, file_path=file_path)
87
88
def serialize(self) -> str:
89
data = []
74
- for runtime_shape, graph_index_to_hash_str in self.cache.items():
75
- for graph_index, hash_str in graph_index_to_hash_str.items():
76
- data.append((runtime_shape, graph_index, hash_str))
90
+ for runtime_shape, value in self.cache.items():
91
+ for graph_index, inductor_artifact in value.items():
92
+ data.append(
93
+ (runtime_shape, graph_index, inductor_artifact.hash_str,
94
+ inductor_artifact.file_path))
95
printer = pprint.PrettyPrinter(indent=4)
96
return printer.pformat(data)
97
@@ -90,13 +108,14 @@ class InductorHashCache:
108
return runtime_shape in self.cache and graph_index in self.cache[
109
runtime_shape]
110
93
- def __getitem__(self, key: Tuple[Optional[int], int]) -> str:
111
+ def __getitem__(self, key: Tuple[Optional[int], int]) -> InductorArtifact:
112
if self.disabled:
113
raise KeyError("cannot read from disabled cache")
114
runtime_shape, graph_index = key
115
return self.cache[runtime_shape][graph_index]
116
99
- def __setitem__(self, key: Tuple[Optional[int], int], value: str):
117
+ def __setitem__(self, key: Tuple[Optional[int], int],
118
+ value: InductorArtifact):
119
# setitem for disabled cache is fine, because we
120
# don't actually write to the disk
121
runtime_shape, graph_index = key
@@ -181,7 +200,8 @@ def wrap_inductor(graph: fx.GraphModule,
200
if (runtime_shape, graph_index) in cache_data:
201
# we compiled this graph before
202
# so we can directly lookup the compiled graph via hash
184
- hash_str = cache_data[(runtime_shape, graph_index)]
203
+ inductor_artifact = cache_data[(runtime_shape, graph_index)]
204
+ hash_str = inductor_artifact.hash_str
205
if graph_index == 0:
206
# adds some info logging for the first graph
207
logger.info(
@@ -199,6 +219,7 @@ def wrap_inductor(graph: fx.GraphModule,
219
"Inductor cache lookup failed. Please remove"
220
f"the cache file {cache_data.cache_file_path} and try again." # noqa
221
)
222
+ inductor_artifact.file_path = inductor_compiled_graph.current_callable.__code__.co_filename # noqa
223
224
# Inductor calling convention (function signature):
225
# f(list) -> tuple
@@ -224,19 +245,20 @@ def wrap_inductor(graph: fx.GraphModule,
245
# the assumption is that we don't have nested Inductor compilation.
246
# compiled_fx_graph_hash will only be called once, and we can hook
247
# it to get the hash of the compiled graph directly.
227
- from torch._inductor.codecache import compiled_fx_graph_hash
248
+
249
+ inductor_artifact = InductorArtifact()
250
+ from torch._inductor.codecache import (FxGraphCache,
251
+ compiled_fx_graph_hash)
252
+ original_load = FxGraphCache.load
253
+
254
+ def hijack_load(*args, **kwargs):
255
+ inductor_compiled_graph = original_load(*args, **kwargs)
256
+ inductor_artifact.file_path = inductor_compiled_graph.current_callable.__code__.co_filename # noqa
257
+ return inductor_compiled_graph
258
259
def hijack_compiled_fx_graph_hash(*args, **kwargs):
260
out = compiled_fx_graph_hash(*args, **kwargs)
231
- # store the hash in the cache
232
- nonlocal cache_data
233
- cache_data[(runtime_shape, graph_index)] = out[0]
234
- if graph_index == 0:
235
- # adds some info logging for the first graph
236
- logger.info("Cache the graph of shape %s for later use",
237
- str(runtime_shape))
238
- logger.debug("store the %s-th graph for shape %s via hash %s",
239
- graph_index, str(runtime_shape), out[0])
261
+ inductor_artifact.hash_str = out[0]
262
return out
263
264
def _check_can_cache(*args, **kwargs):
@@ -255,6 +277,11 @@ def wrap_inductor(graph: fx.GraphModule,
277
if not cache_data.disabled:
278
# compilation cache is enabled, patch several functions
279
280
+ # hijack to get the compiled graph itself
281
+ stack.enter_context(
282
+ patch("torch._inductor.codecache.FxGraphCache.load",
283
+ hijack_load))
284
+
285
# for hijacking the hash of the compiled graph
286
stack.enter_context(
287
patch("torch._inductor.codecache.compiled_fx_graph_hash",
@@ -275,7 +302,16 @@ def wrap_inductor(graph: fx.GraphModule,
302
compiled_graph = compile_fx(graph,
303
example_inputs,
304
config_patches=current_config)
278
-
305
+ # store the inductor_artifact in the cache
306
+ cache_data[(runtime_shape, graph_index)] = inductor_artifact
307
+ if graph_index == 0:
308
+ # adds some info logging for the first graph
309
+ logger.info("Cache the graph of shape %s for later use",
310
+ str(runtime_shape))
311
+ logger.debug(
312
+ "store the %s-th graph for shape %s via hash %s from file %s",
313
+ graph_index, str(runtime_shape), inductor_artifact.hash_str,
314
+ inductor_artifact.file_path)
315
# after compiling the last graph, record the end time
316
if graph_index == num_graphs - 1:
317
now = time.time()
vllm/config.py
+2
-11
@@ -2862,17 +2862,8 @@ class CompilationConfig(BaseModel):
2862
"vllm.unified_attention_with_output",
2863
]
2864
else:
2865
- # v0 can use full graph compilation without splitting,
2866
- # splitting is optional.
2867
- # right now we still need it. kv cache shape
2868
- # will be included in the graph if we don't split
2869
- # the graph.
2870
- # TODO: hide kv cache in static forward context
2871
- # so that inductor does not see it.
2872
- self.splitting_ops = [
2873
- "vllm.unified_attention",
2874
- "vllm.unified_attention_with_output",
2875
- ]
2865
+ # v0 uses full graph compilation
2866
+ self.splitting_ops = []
2867
2868
for k, v in self.inductor_passes.items():
2869
if not isinstance(v, str):