A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[torch.compile] PyTorch 2.6 and nightly compatibility (#12393)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Feb 7, 2025 at 01:09 UTC
09b95e36abbce747b52b9c3e7ae4cceaf40076ad
8 files changed
+493
-320
tests/compile/piecewise/test_simple.py
+1
-1
@@ -92,7 +92,7 @@ def test_simple_piecewise_compile():
92
num_graphs_seen=1, # one graph for the model
93
num_piecewise_graphs_seen=5, # 2 * num_layers + 1
94
num_piecewise_capturable_graphs_seen=3, # 1 + num_layers
95
- num_inductor_compilations=3, # num_piecewise_capturable_graphs_seen
95
+ num_backend_compilations=3, # num_piecewise_capturable_graphs_seen
96
num_cudagraph_caputured=
97
6, # num_cudagraph_sizes * num_piecewise_capturable_graphs_seen
98
):
tests/compile/piecewise/test_toy_llama.py
+3
-3
@@ -322,7 +322,7 @@ def test_toy_llama():
322
num_graphs_seen=0,
323
num_piecewise_graphs_seen=0,
324
num_piecewise_capturable_graphs_seen=0,
325
- num_inductor_compilations=0,
325
+ num_backend_compilations=0,
326
num_cudagraph_caputured=0,
327
):
328
outputs.append(run_model(llama_config, use_compile=False))
@@ -332,7 +332,7 @@ def test_toy_llama():
332
num_graphs_seen=1, # one graph for the model
333
num_piecewise_graphs_seen=1,
334
num_piecewise_capturable_graphs_seen=1,
335
- num_inductor_compilations=1, # num_piecewise_capturable_graphs_seen
335
+ num_backend_compilations=1, # num_piecewise_capturable_graphs_seen
336
num_cudagraph_caputured=
337
2, # num_cudagraph_sizes * num_piecewise_capturable_graphs_seen
338
):
@@ -345,7 +345,7 @@ def test_toy_llama():
345
1, # 2 * num_layers + 1
346
num_piecewise_capturable_graphs_seen=1 +
347
llama_config.num_layers, # 1 + num_layers
348
- num_inductor_compilations=1 +
348
+ num_backend_compilations=1 +
349
llama_config.num_layers, # num_piecewise_capturable_graphs_seen
350
num_cudagraph_caputured=2 *
351
(1 + llama_config.num_layers
vllm/compilation/backends.py
+133
-304
@@ -1,12 +1,10 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
import ast
4
-import copy
4
import dataclasses
5
import os
6
import pprint
7
import time
9
-from collections import defaultdict
8
from contextlib import ExitStack
9
from typing import Any, Callable, Dict, List, Optional, Sequence, Set, Tuple
10
from unittest.mock import patch
@@ -19,6 +17,7 @@ from vllm.config import CompilationConfig, VllmConfig
17
from vllm.logger import init_logger
18
from vllm.utils import weak_ref_tensors
19
20
+from .compiler_interface import EagerAdaptor, InductorAdaptor
21
from .counter import compilation_counter
22
from .inductor_pass import InductorPass
23
from .monitor import end_monitoring_torch_compile
@@ -27,306 +26,128 @@ from .pass_manager import PostGradPassManager
26
logger = init_logger(__name__)
27
28
30
-@dataclasses.dataclass
31
-class InductorArtifact:
32
- hash_str: str = ""
33
- file_path: str = ""
29
+class CompilerManager:
30
+ """
31
+ A manager to manage the compilation process, including
32
+ caching the compiled graph, loading the compiled graph,
33
+ and compiling the graph.
34
35
+ The cache is a dict mapping
36
+ `(runtime_shape, graph_index, backend_name)`
37
+ to `any_data` returned from the compiler.
38
36
-class InductorHashCache:
39
+ When serializing the cache, we save it to a Python file
40
+ for readability. We don't use json here because json doesn't
41
+ support int as key.
42
"""
38
- Disk format: a Python list of tuples, each tuple is
39
- (runtime_shape, graph_index, hash_str, file_path)
40
- We use list of tuple for readability.
43
42
- In-memory format: a defaultdict of dict, where the key is
43
- runtime_shape, and the value is a dict of graph_index to hash_str.
44
+ def __init__(self, use_inductor: bool):
45
+ self.cache: Dict[Tuple[Optional[int], int, str], Any] = dict()
46
+ cls = InductorAdaptor if use_inductor else EagerAdaptor
47
+ self.compiler = cls()
48
45
- The data is essentially `Dict[Optional[int], Dict[int, InductorArtifact]]`,
46
- we don't use json here because json doesn't support int as key.
47
-
48
- TODO: better off-the-shelf solution to serialize the data?
49
- """
49
+ def compute_hash(self, vllm_config: VllmConfig) -> str:
50
+ return self.compiler.compute_hash(vllm_config)
51
51
- def __init__(self, cache_dir: str, disabled: bool = False):
52
- self.cache: Dict[Optional[int],
53
- Dict[int, InductorArtifact]] = defaultdict(dict)
54
- self.disabled = disabled
52
+ def initialize_cache(self, cache_dir: str, disable_cache: bool = False):
53
+ self.disable_cache = disable_cache
54
self.cache_dir = cache_dir
56
- self.cache_file_path = os.path.join(cache_dir,
57
- "inductor_hash_cache.py")
58
- if disabled:
59
- return
60
- # set flags so that Inductor and Triton store their cache
61
- # in the cache_dir, then users only need to copy the cache_dir
62
- # to another machine to reuse the cache.
63
- inductor_cache = os.path.join(cache_dir, "inductor_cache")
64
- os.makedirs(inductor_cache, exist_ok=True)
65
- os.environ["TORCHINDUCTOR_CACHE_DIR"] = inductor_cache
66
- triton_cache = os.path.join(cache_dir, "triton_cache")
67
- os.makedirs(triton_cache, exist_ok=True)
68
- os.environ["TRITON_CACHE_DIR"] = triton_cache
69
- if os.path.exists(self.cache_file_path):
55
+ self.cache_file_path = os.path.join(cache_dir, "vllm_compile_cache.py")
56
+
57
+ if not disable_cache and os.path.exists(self.cache_file_path):
58
+ # load the cache from the file
59
with open(self.cache_file_path) as f:
71
- self.deserialize(f.read())
72
-
73
- def deserialize(self, data: str):
74
- # we use ast.literal_eval to parse the data
75
- # because it is a safe way to parse Python literals.
76
- # do not use eval(), it is unsafe.
77
- list_data = ast.literal_eval(data)
78
- for item in list_data:
79
- runtime_shape = item[0]
80
- graph_index = item[1]
81
- hash_str = item[2]
82
- # for compatibility of old version,
83
- # where we don't have file_path.
84
- # NOTE: after running the new code, the file_path
85
- # will be updated.
86
- file_path = "" if len(item) == 3 else item[3]
87
- self.cache[runtime_shape][graph_index] = InductorArtifact(
88
- hash_str=hash_str, file_path=file_path)
89
-
90
- def serialize(self) -> str:
91
- data = []
92
- for runtime_shape, value in self.cache.items():
93
- for graph_index, inductor_artifact in value.items():
94
- data.append(
95
- (runtime_shape, graph_index, inductor_artifact.hash_str,
96
- inductor_artifact.file_path))
97
- printer = pprint.PrettyPrinter(indent=4)
98
- return printer.pformat(data)
60
+ # we use ast.literal_eval to parse the data
61
+ # because it is a safe way to parse Python literals.
62
+ # do not use eval(), it is unsafe.
63
+ self.cache = ast.literal_eval(f.read())
64
+
65
+ self.compiler.initialize_cache(cache_dir=cache_dir,
66
+ disable_cache=disable_cache)
67
68
def save_to_file(self):
101
- if self.disabled:
69
+ if self.disable_cache:
70
return
71
with open(self.cache_file_path, "w") as f:
104
- f.write(self.serialize())
105
-
106
- def __contains__(self, key: Tuple[Optional[int], int]) -> bool:
107
- if self.disabled:
108
- return False
109
- runtime_shape, graph_index = key
110
- return runtime_shape in self.cache and graph_index in self.cache[
111
- runtime_shape]
112
-
113
- def __getitem__(self, key: Tuple[Optional[int], int]) -> InductorArtifact:
114
- if self.disabled:
115
- raise KeyError("cannot read from disabled cache")
116
- runtime_shape, graph_index = key
117
- return self.cache[runtime_shape][graph_index]
118
-
119
- def __setitem__(self, key: Tuple[Optional[int], int],
120
- value: InductorArtifact):
121
- # setitem for disabled cache is fine, because we
122
- # don't actually write to the disk
123
- runtime_shape, graph_index = key
124
- self.cache[runtime_shape][graph_index] = value
125
-
126
-
127
-class AlwaysHitShapeEnv:
128
- """
129
- Why do we need this class:
130
-
131
- For normal `torch.compile` usage, every compilation will have
132
- one Dynamo bytecode compilation and one Inductor compilation.
133
- The Inductor compilation happens under the context of the
134
- Dynamo bytecode compilation, and that context is used to
135
- determine the dynamic shape information, etc.
136
-
137
- For our use case, we only run Dynamo bytecode compilation once,
138
- and run Inductor compilation multiple times with different shapes
139
- plus a general shape. The compilation for specific shapes happens
140
- outside of the context of the Dynamo bytecode compilation. At that
141
- time, we don't have shape environment to provide to Inductor, and
142
- it will fail the Inductor code cache lookup.
143
-
144
- By providing a dummy shape environment that always hits, we can
145
- make the Inductor code cache lookup always hit, and we can
146
- compile the graph for different shapes as needed.
147
-
148
- The following dummy methods are obtained by trial-and-error
149
- until it works.
150
- """
151
-
152
- def __init__(self) -> None:
153
- self.guards: List[Any] = []
154
-
155
- def evaluate_guards_expression(self, *args, **kwargs):
156
- return True
157
-
158
- def get_pruned_guards(self, *args, **kwargs):
159
- return []
160
-
161
- def produce_guards_expression(self, *args, **kwargs):
162
- return ""
163
-
164
-
165
-def wrap_inductor(graph: fx.GraphModule,
166
- example_inputs,
167
- additional_inductor_config,
168
- compilation_config: CompilationConfig,
169
- vllm_backend: "VllmBackend",
170
- graph_index: int = 0,
171
- num_graphs: int = 1,
172
- runtime_shape: Optional[int] = None,
173
- use_inductor: bool = True) -> Any:
174
- if graph_index == 0:
175
- # before compiling the first graph, record the start time
176
- global compilation_start_time
177
- compilation_start_time = time.time()
178
-
179
- if not use_inductor:
180
- return graph
181
-
182
- compilation_counter.num_inductor_compilations += 1
183
-
184
- from torch._inductor import config
185
- current_config = config.get_config_copy()
186
- from torch._inductor.compile_fx import compile_fx
187
-
188
- if additional_inductor_config is not None:
189
- current_config.update(additional_inductor_config)
190
-
191
- if isinstance(runtime_shape, int):
192
- # for a specific batchsize, tuning triton kernel parameters
193
- # can be beneficial
194
- current_config["max_autotune"] = True
195
- current_config["coordinate_descent_tuning"] = True
196
-
197
- # inductor can inplace modify the graph, so we need to copy it
198
- # see https://github.com/pytorch/pytorch/issues/138980
199
- graph = copy.deepcopy(graph)
200
-
201
- cache_data = vllm_backend.inductor_hash_cache
202
- if (runtime_shape, graph_index) in cache_data:
203
- # we compiled this graph before
204
- # so we can directly lookup the compiled graph via hash
205
- inductor_artifact = cache_data[(runtime_shape, graph_index)]
206
- hash_str = inductor_artifact.hash_str
207
- if graph_index == 0:
208
- # adds some info logging for the first graph
209
- logger.info(
210
- "Directly lookup the graph for shape %s from the cache",
211
- str(runtime_shape)) # noqa
72
+ printer = pprint.PrettyPrinter(indent=4)
73
+ data = printer.pformat(self.cache)
74
+ f.write(data)
75
+
76
+ def load(self,
77
+ graph: fx.GraphModule,
78
+ example_inputs: List[Any],
79
+ graph_index: int,
80
+ runtime_shape: Optional[int] = None) -> Optional[Callable]:
81
+ if (runtime_shape, graph_index, self.compiler.name) not in self.cache:
82
+ return None
83
+ handle = self.cache[(runtime_shape, graph_index, self.compiler.name)]
84
+ compiled_graph = self.compiler.load(handle, graph, example_inputs,
85
+ graph_index, runtime_shape)
86
logger.debug(
213
- "directly lookup the %s-th graph for shape %s via hash %s",
214
- graph_index, str(runtime_shape), hash_str)
215
- from torch._inductor.codecache import FxGraphCache
216
- with patch("torch._inductor.codecache.FxGraphCache._get_shape_env",
217
- lambda *args, **kwargs: AlwaysHitShapeEnv()):
218
- inductor_compiled_graph = FxGraphCache._lookup_graph(
219
- hash_str, example_inputs, True, False)
220
- assert inductor_compiled_graph is not None, (
221
- "Inductor cache lookup failed. Please remove"
222
- f"the cache file {cache_data.cache_file_path} and try again." # noqa
223
- )
224
- inductor_artifact.file_path = inductor_compiled_graph.current_callable.__code__.co_filename # noqa
225
-
226
- # Inductor calling convention (function signature):
227
- # f(list) -> tuple
228
- # Dynamo calling convention (function signature):
229
- # f(*args) -> Any
230
-
231
- # need to know if the graph returns a tuple
232
- from torch._inductor.compile_fx import graph_returns_tuple
233
- returns_tuple = graph_returns_tuple(graph)
234
-
235
- # this is the callable we return to Dynamo to run
236
- def compiled_graph(*args):
237
- # convert args to list
238
- list_args = list(args)
239
- graph_output = inductor_compiled_graph(list_args)
240
- # unpack the tuple if needed
241
- if returns_tuple:
242
- return graph_output
243
- else:
244
- return graph_output[0]
245
- else:
246
- # it's the first time we compile this graph
247
- # the assumption is that we don't have nested Inductor compilation.
248
- # compiled_fx_graph_hash will only be called once, and we can hook
249
- # it to get the hash of the compiled graph directly.
250
-
251
- inductor_artifact = InductorArtifact()
252
- from torch._inductor.codecache import (FxGraphCache,
253
- compiled_fx_graph_hash)
254
- original_load = FxGraphCache.load
255
-
256
- def hijack_load(*args, **kwargs):
257
- inductor_compiled_graph = original_load(*args, **kwargs)
258
- inductor_artifact.file_path = inductor_compiled_graph.current_callable.__code__.co_filename # noqa
259
- return inductor_compiled_graph
260
-
261
- def hijack_compiled_fx_graph_hash(*args, **kwargs):
262
- out = compiled_fx_graph_hash(*args, **kwargs)
263
- inductor_artifact.hash_str = out[0]
264
- return out
265
-
266
- def _check_can_cache(*args, **kwargs):
267
- # no error means it can be cached.
268
- # Inductor refuses to cache the graph outside of Dynamo
269
- # tracing context, and also disables caching for graphs
270
- # with high-order ops.
271
- # For vLLM, in either case, we want to cache the graph.
272
- # see https://github.com/pytorch/pytorch/blob/9f5ebf3fc609105a74eab4ccc24932d6353ff566/torch/_inductor/codecache.py#L1221 # noqa
273
- return
274
-
275
- def _get_shape_env() -> AlwaysHitShapeEnv:
276
- return AlwaysHitShapeEnv()
277
-
278
- with ExitStack() as stack:
279
- if not cache_data.disabled:
280
- # compilation cache is enabled, patch several functions
281
-
282
- # hijack to get the compiled graph itself
283
- stack.enter_context(
284
- patch("torch._inductor.codecache.FxGraphCache.load",
285
- hijack_load))
286
-
287
- # for hijacking the hash of the compiled graph
288
- stack.enter_context(
289
- patch("torch._inductor.codecache.compiled_fx_graph_hash",
290
- hijack_compiled_fx_graph_hash))
291
-
292
- # for providing a dummy shape environment
293
- stack.enter_context(
294
- patch(
295
- "torch._inductor.codecache.FxGraphCache._get_shape_env",
296
- _get_shape_env))
297
-
298
- # for forcing the graph to be cached
299
- stack.enter_context(
300
- patch(
301
- "torch._inductor.codecache.FxGraphCache._check_can_cache",
302
- _check_can_cache))
303
-
304
- compiled_graph = compile_fx(graph,
305
- example_inputs,
306
- config_patches=current_config)
307
- # store the inductor_artifact in the cache
308
- cache_data[(runtime_shape, graph_index)] = inductor_artifact
87
+ "Directly load the %s-th graph for shape %s from %s via "
88
+ "handle %s", graph_index, str(runtime_shape), self.compiler.name,
89
+ handle)
90
+ return compiled_graph
91
+
92
+ def compile(self,
93
+ graph: fx.GraphModule,
94
+ example_inputs,
95
+ additional_inductor_config,
96
+ compilation_config: CompilationConfig,
97
+ graph_index: int = 0,
98
+ num_graphs: int = 1,
99
+ runtime_shape: Optional[int] = None) -> Any:
100
if graph_index == 0:
310
- # adds some info logging for the first graph
311
- logger.info("Cache the graph of shape %s for later use",
312
- str(runtime_shape))
313
- logger.debug(
314
- "store the %s-th graph for shape %s via hash %s from file %s",
315
- graph_index, str(runtime_shape), inductor_artifact.hash_str,
316
- inductor_artifact.file_path)
317
- # after compiling the last graph, record the end time
318
- if graph_index == num_graphs - 1:
319
- now = time.time()
320
- elapsed = now - compilation_start_time
321
- compilation_config.compilation_time += elapsed
322
- if runtime_shape is None:
323
- logger.info("Compiling a graph for general shape takes %.2f s",
324
- elapsed)
325
- else:
326
- logger.info("Compiling a graph for shape %s takes %.2f s",
327
- runtime_shape, elapsed)
101
+ # before compiling the first graph, record the start time
102
+ global compilation_start_time
103
+ compilation_start_time = time.time()
104
+
105
+ compilation_counter.num_backend_compilations += 1
106
+
107
+ compiled_graph = None
108
+
109
+ # try to load from the cache
110
+ compiled_graph = self.load(graph, example_inputs, graph_index,
111
+ runtime_shape)
112
+ if compiled_graph is not None:
113
+ if graph_index == 0:
114
+ # adds some info logging for the first graph
115
+ logger.info("Directly load the compiled graph for shape %s "
116
+ "from the cache", str(runtime_shape)) # noqa
117
+ return compiled_graph
118
+
119
+ # no compiler cached the graph, or the cache is disabled,
120
+ # we need to compile it
121
+ compiled_graph, handle = self.compiler.compile(
122
+ graph, example_inputs, additional_inductor_config, runtime_shape)
123
+
124
+ assert compiled_graph is not None, "Failed to compile the graph"
125
+
126
+ # store the artifact in the cache
127
+ if handle is not None:
128
+ self.cache[(runtime_shape, graph_index,
129
+ self.compiler.name)] = handle
130
+ if graph_index == 0:
131
+ # adds some info logging for the first graph
132
+ logger.info("Cache the graph of shape %s for later use",
133
+ str(runtime_shape))
134
+ logger.debug(
135
+ "store the %s-th graph for shape %s from %s via handle %s",
136
+ graph_index, str(runtime_shape), self.compiler.name, handle)
137
+
138
+ # after compiling the last graph, record the end time
139
+ if graph_index == num_graphs - 1:
140
+ now = time.time()
141
+ elapsed = now - compilation_start_time
142
+ compilation_config.compilation_time += elapsed
143
+ if runtime_shape is None:
144
+ logger.info("Compiling a graph for general shape takes %.2f s",
145
+ elapsed)
146
+ else:
147
+ logger.info("Compiling a graph for shape %s takes %.2f s",
148
+ runtime_shape, elapsed)
149
329
- return compiled_graph
150
+ return compiled_graph
151
152
153
@dataclasses.dataclass
@@ -436,16 +257,15 @@ class PiecewiseCompileInterpreter(torch.fx.Interpreter):
257
i for i, x in enumerate(args) if isinstance(x, torch.SymInt)
258
]
259
global compilation_start_time
439
- compiled_graph_for_general_shape = wrap_inductor(
260
+ compiled_graph_for_general_shape = self.vllm_backend.\
261
+ compiler_manager.compile(
262
submod,
263
args,
264
self.compilation_config.inductor_compile_config,
265
self.compilation_config,
444
- self.vllm_backend,
266
graph_index=index,
267
num_graphs=len(self.compile_submod_names),
447
- runtime_shape=None,
448
- use_inductor=self.compilation_config.use_inductor)
268
+ runtime_shape=None)
269
270
self.module.__dict__[target] = PiecewiseBackend(
271
submod, self.vllm_config, self.graph_pool, index,
@@ -483,7 +303,7 @@ class VllmBackend:
303
post_grad_passes: Sequence[Callable]
304
sym_tensor_indices: List[int]
305
input_buffers: List[torch.Tensor]
486
- inductor_hash_cache: InductorHashCache
306
+ compiler_manager: CompilerManager
307
308
def __init__(
309
self,
@@ -507,6 +327,9 @@ class VllmBackend:
327
self.vllm_config = vllm_config
328
self.compilation_config = vllm_config.compilation_config
329
330
+ self.compiler_manager: CompilerManager = CompilerManager(
331
+ self.compilation_config.use_inductor)
332
+
333
# `torch.compile` is JIT compiled, so we don't need to
334
# do anything here
335
@@ -533,9 +356,11 @@ class VllmBackend:
356
# the cache dir will be the same so that we can reuse the compiled
357
# graph.
358
359
+ factors = []
360
# 1. factors come from the vllm_config (it mainly summarizes how the
361
# model is created)
362
config_hash = vllm_config.compute_hash()
363
+ factors.append(config_hash)
364
365
# 2. factors come from the code files that are traced by Dynamo (
366
# it mainly summarizes how the model is used in forward pass)
@@ -553,10 +378,15 @@ class VllmBackend:
378
import hashlib
379
code_hash = hashlib.md5(
380
"\n".join(hash_content).encode()).hexdigest()
381
+ factors.append(code_hash)
382
+
383
+ # 3. compiler hash
384
+ compiler_hash = self.compiler_manager.compute_hash(vllm_config)
385
+ factors.append(compiler_hash)
386
+
387
+ # combine all factors to generate the cache dir
388
+ hash_key = hashlib.md5(str(factors).encode()).hexdigest()[:10]
389
557
- # combine the two hashes to generate the cache dir
558
- hash_key = hashlib.md5(
559
- f"{config_hash}_{code_hash}".encode()).hexdigest()[:10]
390
cache_dir = os.path.join(
391
envs.VLLM_CACHE_ROOT,
392
"torch_compile_cache",
@@ -570,15 +400,16 @@ class VllmBackend:
400
cache_dir, f"rank_{vllm_config.parallel_config.rank}")
401
self.compilation_config.local_cache_dir = local_cache_dir
402
573
- disabled = envs.VLLM_DISABLE_COMPILE_CACHE
574
- self.inductor_hash_cache: InductorHashCache = InductorHashCache(
575
- local_cache_dir, disabled=disabled)
576
- if disabled:
403
+ disable_cache = envs.VLLM_DISABLE_COMPILE_CACHE
404
+
405
+ if disable_cache:
406
logger.info("vLLM's torch.compile cache is disabled.")
407
else:
408
logger.info("Using cache directory: %s for vLLM's torch.compile",
409
local_cache_dir)
410
411
+ self.compiler_manager.initialize_cache(local_cache_dir, disable_cache)
412
+
413
# when dynamo calls the backend, it means the bytecode
414
# transform and analysis are done
415
compilation_counter.num_graphs_seen += 1
@@ -759,7 +590,7 @@ class PiecewiseBackend:
590
if self.is_last_graph and not self.to_be_compiled_sizes:
591
# no specific sizes to compile
592
# save the hash of the inductor graph for the next run
762
- self.vllm_backend.inductor_hash_cache.save_to_file()
593
+ self.vllm_backend.compiler_manager.save_to_file()
594
end_monitoring_torch_compile(self.vllm_config)
595
596
def __call__(self, *args) -> Any:
@@ -782,16 +613,14 @@ class PiecewiseBackend:
613
entry.compiled = True
614
self.to_be_compiled_sizes.remove(runtime_shape)
615
# args are real arguments
785
- entry.runnable = wrap_inductor(
616
+ entry.runnable = self.vllm_backend.compiler_manager.compile(
617
self.graph,
618
args,
619
self.compilation_config.inductor_compile_config,
620
self.compilation_config,
790
- self.vllm_backend,
621
graph_index=self.piecewise_compile_index,
622
num_graphs=self.total_piecewise_compiles,
793
- runtime_shape=runtime_shape,
794
- use_inductor=self.compilation_config.use_inductor)
623
+ runtime_shape=runtime_shape)
624
625
# finished compilations for all required shapes
626
if self.is_last_graph and not self.to_be_compiled_sizes:
vllm/compilation/compiler_interface.py
new
+340
@@ -0,0 +1,340 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+import copy
3
+import hashlib
4
+import os
5
+from contextlib import ExitStack
6
+from typing import Any, Callable, Dict, List, Optional, Tuple
7
+from unittest.mock import patch
8
+
9
+import torch
10
+import torch._inductor.compile_fx
11
+import torch.fx as fx
12
+
13
+from vllm.config import VllmConfig
14
+
15
+
16
+class CompilerInterface:
17
+ """
18
+ The interface for a compiler that can be used by vLLM.
19
+ """
20
+ # The name of the compiler, e.g. inductor.
21
+ # This is a class-level attribute.
22
+ name: str
23
+
24
+ def initialize_cache(self, cache_dir: str, disable_cache: bool = False):
25
+ """
26
+ when the vLLM process uses `cache_dir` as the cache directory,
27
+ the compiler should initialize itself with the cache directory,
28
+ e.g. by re-directing its own cache directory to a sub-directory.
29
+ """
30
+ pass
31
+
32
+ def compute_hash(self, vllm_config: VllmConfig) -> str:
33
+ """
34
+ Gather all the relevant information from the VLLM config,
35
+ to compute a hash so that we can cache the compiled model.
36
+
37
+ See :meth:`VllmConfig.compute_hash` to check what information
38
+ is already considered by default. This function should only
39
+ consider the information that is specific to the compiler.
40
+ """
41
+ return ""
42
+
43
+ def compile(
44
+ self,
45
+ graph: fx.GraphModule,
46
+ example_inputs: List[Any],
47
+ compiler_config: Dict[str, Any],
48
+ runtime_shape: Optional[int] = None
49
+ ) -> Tuple[Optional[Callable], Optional[Any]]:
50
+ """
51
+ Compile the graph with the given example inputs and compiler config,
52
+ with a runtime shape. If the `runtime_shape` is None, it means
53
+ the `example_inputs` have a dynamic shape. Otherwise, the
54
+ `runtime_shape` specifies the shape of the inputs. Right now we only
55
+ support one variable shape for all inputs, which is the batchsize
56
+ (number of tokens) during inference.
57
+
58
+ Dynamo will make sure `graph(*example_inputs)` is valid.
59
+
60
+ The function should return a compiled callable function, as well as
61
+ a handle that can be used to directly load the compiled function.
62
+
63
+ The handle should be a plain Python object, preferably a string or a
64
+ file path for readability.
65
+
66
+ If the compiler doesn't support caching, it should return None for the
67
+ handle. If the compiler fails to compile the graph, it should return
68
+ None for the compiled function as well.
69
+ """
70
+ return None, None
71
+
72
+ def load(self,
73
+ handle: Any,
74
+ graph: fx.GraphModule,
75
+ example_inputs: List[Any],
76
+ graph_index: int,
77
+ runtime_shape: Optional[int] = None) -> Callable:
78
+ """
79
+ Load the compiled function from the handle.
80
+ Raises an error if the handle is invalid.
81
+
82
+ The handle is the second return value of the `compile` function.
83
+ """
84
+ raise NotImplementedError("caching is not supported")
85
+
86
+
87
+class AlwaysHitShapeEnv:
88
+ """
89
+ Why do we need this class:
90
+
91
+ For normal `torch.compile` usage, every compilation will have
92
+ one Dynamo bytecode compilation and one Inductor compilation.
93
+ The Inductor compilation happens under the context of the
94
+ Dynamo bytecode compilation, and that context is used to
95
+ determine the dynamic shape information, etc.
96
+
97
+ For our use case, we only run Dynamo bytecode compilation once,
98
+ and run Inductor compilation multiple times with different shapes
99
+ plus a general shape. The compilation for specific shapes happens
100
+ outside of the context of the Dynamo bytecode compilation. At that
101
+ time, we don't have shape environment to provide to Inductor, and
102
+ it will fail the Inductor code cache lookup.
103
+
104
+ By providing a dummy shape environment that always hits, we can
105
+ make the Inductor code cache lookup always hit, and we can
106
+ compile the graph for different shapes as needed.
107
+
108
+ The following dummy methods are obtained by trial-and-error
109
+ until it works.
110
+ """
111
+
112
+ def __init__(self) -> None:
113
+ self.guards: List[Any] = []
114
+
115
+ def evaluate_guards_expression(self, *args, **kwargs):
116
+ return True
117
+
118
+ def get_pruned_guards(self, *args, **kwargs):
119
+ return []
120
+
121
+ def produce_guards_expression(self, *args, **kwargs):
122
+ return ""
123
+
124
+
125
+class InductorAdaptor(CompilerInterface):
126
+ """
127
+ The adaptor for the Inductor compiler, version 2.5 and 2.6.
128
+ """
129
+ name = "inductor"
130
+
131
+ def compute_hash(self, vllm_config: VllmConfig) -> str:
132
+ factors: List[Any] = []
133
+ # summarize system state
134
+ from torch._inductor.codecache import CacheBase
135
+ system_factors = CacheBase.get_system()
136
+ factors.append(system_factors)
137
+
138
+ # summarize pytorch state
139
+ from torch._inductor.codecache import torch_key
140
+ torch_factors = torch_key()
141
+ factors.append(torch_factors)
142
+ hash_str = hashlib.md5(str(factors).encode()).hexdigest()[:10]
143
+ return hash_str
144
+
145
+ def initialize_cache(self, cache_dir: str, disable_cache: bool = False):
146
+ if disable_cache:
147
+ return
148
+ # redirect the cache directory to a sub-directory
149
+ # set flags so that Inductor and Triton store their cache
150
+ # in the cache_dir, then users only need to copy the cache_dir
151
+ # to another machine to reuse the cache.
152
+ inductor_cache = os.path.join(cache_dir, "inductor_cache")
153
+ os.makedirs(inductor_cache, exist_ok=True)
154
+ os.environ["TORCHINDUCTOR_CACHE_DIR"] = inductor_cache
155
+ triton_cache = os.path.join(cache_dir, "triton_cache")
156
+ os.makedirs(triton_cache, exist_ok=True)
157
+ os.environ["TRITON_CACHE_DIR"] = triton_cache
158
+
159
+ def compile(
160
+ self,
161
+ graph: fx.GraphModule,
162
+ example_inputs: List[Any],
163
+ compiler_config: Dict[str, Any],
164
+ runtime_shape: Optional[int] = None
165
+ ) -> Tuple[Optional[Callable], Optional[Any]]:
166
+ from torch._inductor import config
167
+ current_config = config.get_config_copy()
168
+ from torch._inductor.compile_fx import compile_fx
169
+
170
+ # disable remote cache
171
+ current_config["fx_graph_cache"] = True
172
+ current_config["fx_graph_remote_cache"] = False
173
+
174
+ if compiler_config is not None:
175
+ current_config.update(compiler_config)
176
+
177
+ if isinstance(runtime_shape, int):
178
+ # for a specific batchsize, tuning triton kernel parameters
179
+ # can be beneficial
180
+ current_config["max_autotune"] = True
181
+ current_config["coordinate_descent_tuning"] = True
182
+
183
+ # inductor can inplace modify the graph, so we need to copy it
184
+ # see https://github.com/pytorch/pytorch/issues/138980
185
+ graph = copy.deepcopy(graph)
186
+
187
+ # it's the first time we compile this graph
188
+ # the assumption is that we don't have nested Inductor compilation.
189
+ # compiled_fx_graph_hash will only be called once, and we can hook
190
+ # it to get the hash of the compiled graph directly.
191
+
192
+ hash_str, file_path = None, None
193
+ from torch._inductor.codecache import (FxGraphCache,
194
+ compiled_fx_graph_hash)
195
+
196
+ if torch.__version__.startswith("2.5"):
197
+ original_load = FxGraphCache.load
198
+ original_load_name = "torch._inductor.codecache.FxGraphCache.load"
199
+
200
+ def hijack_load(*args, **kwargs):
201
+ inductor_compiled_graph = original_load(*args, **kwargs)
202
+ nonlocal file_path
203
+ file_path = inductor_compiled_graph.current_callable.__code__.co_filename # noqa
204
+ return inductor_compiled_graph
205
+
206
+ hijacked_compile_fx_inner = torch._inductor.compile_fx.compile_fx_inner # noqa
207
+ elif torch.__version__ >= "2.6":
208
+ # function renamed in 2.6
209
+ original_load_name = None
210
+
211
+ def hijacked_compile_fx_inner(*args, **kwargs):
212
+ output = torch._inductor.compile_fx.compile_fx_inner(
213
+ *args, **kwargs)
214
+ nonlocal hash_str
215
+ inductor_compiled_graph = output
216
+ if inductor_compiled_graph is not None:
217
+ nonlocal file_path
218
+ file_path = inductor_compiled_graph.current_callable.__code__.co_filename # noqa
219
+ hash_str = inductor_compiled_graph._fx_graph_cache_key
220
+ return output
221
+
222
+ def hijack_compiled_fx_graph_hash(*args, **kwargs):
223
+ out = compiled_fx_graph_hash(*args, **kwargs)
224
+ nonlocal hash_str
225
+ hash_str = out[0]
226
+ return out
227
+
228
+ def _check_can_cache(*args, **kwargs):
229
+ # no error means it can be cached.
230
+ # Inductor refuses to cache the graph outside of Dynamo
231
+ # tracing context, and also disables caching for graphs
232
+ # with high-order ops.
233
+ # For vLLM, in either case, we want to cache the graph.
234
+ # see https://github.com/pytorch/pytorch/blob/9f5ebf3fc609105a74eab4ccc24932d6353ff566/torch/_inductor/codecache.py#L1221 # noqa
235
+ return
236
+
237
+ def _get_shape_env() -> AlwaysHitShapeEnv:
238
+ return AlwaysHitShapeEnv()
239
+
240
+ with ExitStack() as stack:
241
+ # hijack to get the compiled graph itself
242
+ if original_load_name is not None:
243
+ stack.enter_context(patch(original_load_name, hijack_load))
244
+
245
+ # for hijacking the hash of the compiled graph
246
+ stack.enter_context(
247
+ patch("torch._inductor.codecache.compiled_fx_graph_hash",
248
+ hijack_compiled_fx_graph_hash))
249
+
250
+ # for providing a dummy shape environment
251
+ stack.enter_context(
252
+ patch("torch._inductor.codecache.FxGraphCache._get_shape_env",
253
+ _get_shape_env))
254
+
255
+ # for forcing the graph to be cached
256
+ stack.enter_context(
257
+ patch(
258
+ "torch._inductor.codecache.FxGraphCache._check_can_cache",
259
+ _check_can_cache))
260
+
261
+ compiled_graph = compile_fx(
262
+ graph,
263
+ example_inputs,
264
+ inner_compile=hijacked_compile_fx_inner,
265
+ config_patches=current_config)
266
+
267
+ assert hash_str is not None, (
268
+ "failed to get the hash of the compiled graph")
269
+ assert file_path is not None, (
270
+ "failed to get the file path of the compiled graph")
271
+ return compiled_graph, (hash_str, file_path)
272
+
273
+ def load(self,
274
+ handle: Any,
275
+ graph: fx.GraphModule,
276
+ example_inputs: List[Any],
277
+ graph_index: int,
278
+ runtime_shape: Optional[int] = None) -> Callable:
279
+ assert isinstance(handle, tuple)
280
+ assert isinstance(handle[0], str)
281
+ assert isinstance(handle[1], str)
282
+ hash_str = handle[0]
283
+
284
+ from torch._inductor.codecache import FxGraphCache
285
+ with patch("torch._inductor.codecache.FxGraphCache._get_shape_env",
286
+ lambda *args, **kwargs: AlwaysHitShapeEnv()):
287
+ if torch.__version__.startswith("2.5"):
288
+ inductor_compiled_graph = FxGraphCache._lookup_graph(
289
+ hash_str, example_inputs, True, False)
290
+ assert inductor_compiled_graph is not None, (
291
+ "Inductor cache lookup failed. Please remove"
292
+ f"the cache directory and try again." # noqa
293
+ )
294
+ elif torch.__version__ >= "2.6":
295
+ from torch._inductor.output_code import (
296
+ CompiledFxGraphConstantsWithGm)
297
+ constants = CompiledFxGraphConstantsWithGm(graph)
298
+ inductor_compiled_graph, _ = FxGraphCache._lookup_graph(
299
+ hash_str, example_inputs, True, None, constants)
300
+ assert inductor_compiled_graph is not None, (
301
+ "Inductor cache lookup failed. Please remove"
302
+ f"the cache directory and try again." # noqa
303
+ )
304
+
305
+ # Inductor calling convention (function signature):
306
+ # f(list) -> tuple
307
+ # Dynamo calling convention (function signature):
308
+ # f(*args) -> Any
309
+
310
+ # need to know if the graph returns a tuple
311
+ from torch._inductor.compile_fx import graph_returns_tuple
312
+ returns_tuple = graph_returns_tuple(graph)
313
+
314
+ # this is the callable we return to Dynamo to run
315
+ def compiled_graph(*args):
316
+ # convert args to list
317
+ list_args = list(args)
318
+ graph_output = inductor_compiled_graph(list_args)
319
+ # unpack the tuple if needed
320
+ if returns_tuple:
321
+ return graph_output
322
+ else:
323
+ return graph_output[0]
324
+
325
+ return compiled_graph
326
+
327
+
328
+class EagerAdaptor(CompilerInterface):
329
+ name = "eager"
330
+
331
+ def compile(
332
+ self,
333
+ graph: fx.GraphModule,
334
+ example_inputs: List[Any],
335
+ compiler_config: Dict[str, Any],
336
+ runtime_shape: Optional[int] = None
337
+ ) -> Tuple[Optional[Callable], Optional[Any]]:
338
+ # we don't need to compile the graph, just return the graph itself.
339
+ # It does not support caching, return None for the handle.
340
+ return graph, None
vllm/compilation/counter.py
+1
-1
@@ -13,7 +13,7 @@ class CompilationCounter:
13
num_piecewise_graphs_seen: int = 0
14
# not including the splitting ops
15
num_piecewise_capturable_graphs_seen: int = 0
16
- num_inductor_compilations: int = 0
16
+ num_backend_compilations: int = 0
17
num_cudagraph_caputured: int = 0
18
19
def clone(self) -> "CompilationCounter":
vllm/compilation/inductor_pass.py
-1
@@ -13,7 +13,6 @@ from torch import fx
13
class InductorPass(ABC):
14
"""
15
General custom inductor pass interface.
16
- TODO(torch==2.6) use torch._inductor.custom_graph_pass.CustomGraphPass
16
"""
17
18
@abstractmethod
vllm/compilation/pass_manager.py
+15
-1
@@ -2,6 +2,7 @@
2
3
from typing import Any, Dict, List
4
5
+import torch
6
from torch import fx as fx
7
8
from vllm.config import CompilationConfig
@@ -15,7 +16,17 @@ from .reshapes import RedundantReshapesPass
16
logger = init_logger(__name__)
17
18
18
-class PostGradPassManager:
19
+class PlaceHolder:
20
+ pass
21
+
22
+
23
+if torch.__version__ < "2.6":
24
+ Parent = PlaceHolder # type: ignore
25
+else:
26
+ Parent = torch._inductor.custom_graph_pass.CustomGraphPass # type: ignore
27
+
28
+
29
+class PostGradPassManager(Parent):
30
"""
31
The pass manager for post-grad passes.
32
It handles configuration, adding custom passes, and running passes.
@@ -55,6 +66,9 @@ class PostGradPassManager:
66
assert isinstance(pass_, InductorPass)
67
self.passes.append(pass_)
68
69
+ def uuid(self):
70
+ return self.__getstate__()
71
+
72
def __getstate__(self) -> Dict[str, List[Any]]:
73
"""
74
Custom pickling for the pass manager, as some passes cannot be pickled.
vllm/config.py
-9
@@ -3072,15 +3072,6 @@ class VllmConfig:
3072
the final hidden states.
3073
"""
3074
factors: List[Any] = []
3075
- # summarize system state
3076
- from torch._inductor.codecache import CacheBase
3077
- system_factors = CacheBase.get_system()
3078
- factors.append(system_factors)
3079
-
3080
- # summarize pytorch state
3081
- from torch._inductor.codecache import torch_key
3082
- torch_factors = torch_key()
3083
- factors.append(torch_factors)
3075
3076
# summarize vllm config
3077
vllm_factors: List[Any] = []