@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 09b95e36ab
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[torch.compile] PyTorch 2.6 and nightly compatibility (#12393)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Feb 7, 2025 at 01:09 UTC 09b95e36abbce747b52b9c3e7ae4cceaf40076ad
8 files changed +493 -320
tests/compile/piecewise/test_simple.py
+1 -1
@@ -92,7 +92,7 @@ def test_simple_piecewise_compile():
92 num_graphs_seen=1, # one graph for the model
93 num_piecewise_graphs_seen=5, # 2 * num_layers + 1
94 num_piecewise_capturable_graphs_seen=3, # 1 + num_layers
95 - num_inductor_compilations=3, # num_piecewise_capturable_graphs_seen
95 + num_backend_compilations=3, # num_piecewise_capturable_graphs_seen
96 num_cudagraph_caputured=
97 6, # num_cudagraph_sizes * num_piecewise_capturable_graphs_seen
98 ):
tests/compile/piecewise/test_toy_llama.py
+3 -3
@@ -322,7 +322,7 @@ def test_toy_llama():
322 num_graphs_seen=0,
323 num_piecewise_graphs_seen=0,
324 num_piecewise_capturable_graphs_seen=0,
325 - num_inductor_compilations=0,
325 + num_backend_compilations=0,
326 num_cudagraph_caputured=0,
327 ):
328 outputs.append(run_model(llama_config, use_compile=False))
@@ -332,7 +332,7 @@ def test_toy_llama():
332 num_graphs_seen=1, # one graph for the model
333 num_piecewise_graphs_seen=1,
334 num_piecewise_capturable_graphs_seen=1,
335 - num_inductor_compilations=1, # num_piecewise_capturable_graphs_seen
335 + num_backend_compilations=1, # num_piecewise_capturable_graphs_seen
336 num_cudagraph_caputured=
337 2, # num_cudagraph_sizes * num_piecewise_capturable_graphs_seen
338 ):
@@ -345,7 +345,7 @@ def test_toy_llama():
345 1, # 2 * num_layers + 1
346 num_piecewise_capturable_graphs_seen=1 +
347 llama_config.num_layers, # 1 + num_layers
348 - num_inductor_compilations=1 +
348 + num_backend_compilations=1 +
349 llama_config.num_layers, # num_piecewise_capturable_graphs_seen
350 num_cudagraph_caputured=2 *
351 (1 + llama_config.num_layers
vllm/compilation/backends.py
+133 -304
@@ -1,12 +1,10 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 import ast
4 -import copy
4 import dataclasses
5 import os
6 import pprint
7 import time
9 -from collections import defaultdict
8 from contextlib import ExitStack
9 from typing import Any, Callable, Dict, List, Optional, Sequence, Set, Tuple
10 from unittest.mock import patch
@@ -19,6 +17,7 @@ from vllm.config import CompilationConfig, VllmConfig
17 from vllm.logger import init_logger
18 from vllm.utils import weak_ref_tensors
19
20 +from .compiler_interface import EagerAdaptor, InductorAdaptor
21 from .counter import compilation_counter
22 from .inductor_pass import InductorPass
23 from .monitor import end_monitoring_torch_compile
@@ -27,306 +26,128 @@ from .pass_manager import PostGradPassManager
26 logger = init_logger(__name__)
27
28
30 -@dataclasses.dataclass
31 -class InductorArtifact:
32 - hash_str: str = ""
33 - file_path: str = ""
29 +class CompilerManager:
30 + """
31 + A manager to manage the compilation process, including
32 + caching the compiled graph, loading the compiled graph,
33 + and compiling the graph.
34
35 + The cache is a dict mapping
36 + `(runtime_shape, graph_index, backend_name)`
37 + to `any_data` returned from the compiler.
38
36 -class InductorHashCache:
39 + When serializing the cache, we save it to a Python file
40 + for readability. We don't use json here because json doesn't
41 + support int as key.
42 """
38 - Disk format: a Python list of tuples, each tuple is
39 - (runtime_shape, graph_index, hash_str, file_path)
40 - We use list of tuple for readability.
43
42 - In-memory format: a defaultdict of dict, where the key is
43 - runtime_shape, and the value is a dict of graph_index to hash_str.
44 + def __init__(self, use_inductor: bool):
45 + self.cache: Dict[Tuple[Optional[int], int, str], Any] = dict()
46 + cls = InductorAdaptor if use_inductor else EagerAdaptor
47 + self.compiler = cls()
48
45 - The data is essentially `Dict[Optional[int], Dict[int, InductorArtifact]]`,
46 - we don't use json here because json doesn't support int as key.
47 -
48 - TODO: better off-the-shelf solution to serialize the data?
49 - """
49 + def compute_hash(self, vllm_config: VllmConfig) -> str:
50 + return self.compiler.compute_hash(vllm_config)
51
51 - def __init__(self, cache_dir: str, disabled: bool = False):
52 - self.cache: Dict[Optional[int],
53 - Dict[int, InductorArtifact]] = defaultdict(dict)
54 - self.disabled = disabled
52 + def initialize_cache(self, cache_dir: str, disable_cache: bool = False):
53 + self.disable_cache = disable_cache
54 self.cache_dir = cache_dir
56 - self.cache_file_path = os.path.join(cache_dir,
57 - "inductor_hash_cache.py")
58 - if disabled:
59 - return
60 - # set flags so that Inductor and Triton store their cache
61 - # in the cache_dir, then users only need to copy the cache_dir
62 - # to another machine to reuse the cache.
63 - inductor_cache = os.path.join(cache_dir, "inductor_cache")
64 - os.makedirs(inductor_cache, exist_ok=True)
65 - os.environ["TORCHINDUCTOR_CACHE_DIR"] = inductor_cache
66 - triton_cache = os.path.join(cache_dir, "triton_cache")
67 - os.makedirs(triton_cache, exist_ok=True)
68 - os.environ["TRITON_CACHE_DIR"] = triton_cache
69 - if os.path.exists(self.cache_file_path):
55 + self.cache_file_path = os.path.join(cache_dir, "vllm_compile_cache.py")
56 +
57 + if not disable_cache and os.path.exists(self.cache_file_path):
58 + # load the cache from the file
59 with open(self.cache_file_path) as f:
71 - self.deserialize(f.read())
72 -
73 - def deserialize(self, data: str):
74 - # we use ast.literal_eval to parse the data
75 - # because it is a safe way to parse Python literals.
76 - # do not use eval(), it is unsafe.
77 - list_data = ast.literal_eval(data)
78 - for item in list_data:
79 - runtime_shape = item[0]
80 - graph_index = item[1]
81 - hash_str = item[2]
82 - # for compatibility of old version,
83 - # where we don't have file_path.
84 - # NOTE: after running the new code, the file_path
85 - # will be updated.
86 - file_path = "" if len(item) == 3 else item[3]
87 - self.cache[runtime_shape][graph_index] = InductorArtifact(
88 - hash_str=hash_str, file_path=file_path)
89 -
90 - def serialize(self) -> str:
91 - data = []
92 - for runtime_shape, value in self.cache.items():
93 - for graph_index, inductor_artifact in value.items():
94 - data.append(
95 - (runtime_shape, graph_index, inductor_artifact.hash_str,
96 - inductor_artifact.file_path))
97 - printer = pprint.PrettyPrinter(indent=4)
98 - return printer.pformat(data)
60 + # we use ast.literal_eval to parse the data
61 + # because it is a safe way to parse Python literals.
62 + # do not use eval(), it is unsafe.
63 + self.cache = ast.literal_eval(f.read())
64 +
65 + self.compiler.initialize_cache(cache_dir=cache_dir,
66 + disable_cache=disable_cache)
67
68 def save_to_file(self):
101 - if self.disabled:
69 + if self.disable_cache:
70 return
71 with open(self.cache_file_path, "w") as f:
104 - f.write(self.serialize())
105 -
106 - def __contains__(self, key: Tuple[Optional[int], int]) -> bool:
107 - if self.disabled:
108 - return False
109 - runtime_shape, graph_index = key
110 - return runtime_shape in self.cache and graph_index in self.cache[
111 - runtime_shape]
112 -
113 - def __getitem__(self, key: Tuple[Optional[int], int]) -> InductorArtifact:
114 - if self.disabled:
115 - raise KeyError("cannot read from disabled cache")
116 - runtime_shape, graph_index = key
117 - return self.cache[runtime_shape][graph_index]
118 -
119 - def __setitem__(self, key: Tuple[Optional[int], int],
120 - value: InductorArtifact):
121 - # setitem for disabled cache is fine, because we
122 - # don't actually write to the disk
123 - runtime_shape, graph_index = key
124 - self.cache[runtime_shape][graph_index] = value
125 -
126 -
127 -class AlwaysHitShapeEnv:
128 - """
129 - Why do we need this class:
130 -
131 - For normal `torch.compile` usage, every compilation will have
132 - one Dynamo bytecode compilation and one Inductor compilation.
133 - The Inductor compilation happens under the context of the
134 - Dynamo bytecode compilation, and that context is used to
135 - determine the dynamic shape information, etc.
136 -
137 - For our use case, we only run Dynamo bytecode compilation once,
138 - and run Inductor compilation multiple times with different shapes
139 - plus a general shape. The compilation for specific shapes happens
140 - outside of the context of the Dynamo bytecode compilation. At that
141 - time, we don't have shape environment to provide to Inductor, and
142 - it will fail the Inductor code cache lookup.
143 -
144 - By providing a dummy shape environment that always hits, we can
145 - make the Inductor code cache lookup always hit, and we can
146 - compile the graph for different shapes as needed.
147 -
148 - The following dummy methods are obtained by trial-and-error
149 - until it works.
150 - """
151 -
152 - def __init__(self) -> None:
153 - self.guards: List[Any] = []
154 -
155 - def evaluate_guards_expression(self, *args, **kwargs):
156 - return True
157 -
158 - def get_pruned_guards(self, *args, **kwargs):
159 - return []
160 -
161 - def produce_guards_expression(self, *args, **kwargs):
162 - return ""
163 -
164 -
165 -def wrap_inductor(graph: fx.GraphModule,
166 - example_inputs,
167 - additional_inductor_config,
168 - compilation_config: CompilationConfig,
169 - vllm_backend: "VllmBackend",
170 - graph_index: int = 0,
171 - num_graphs: int = 1,
172 - runtime_shape: Optional[int] = None,
173 - use_inductor: bool = True) -> Any:
174 - if graph_index == 0:
175 - # before compiling the first graph, record the start time
176 - global compilation_start_time
177 - compilation_start_time = time.time()
178 -
179 - if not use_inductor:
180 - return graph
181 -
182 - compilation_counter.num_inductor_compilations += 1
183 -
184 - from torch._inductor import config
185 - current_config = config.get_config_copy()
186 - from torch._inductor.compile_fx import compile_fx
187 -
188 - if additional_inductor_config is not None:
189 - current_config.update(additional_inductor_config)
190 -
191 - if isinstance(runtime_shape, int):
192 - # for a specific batchsize, tuning triton kernel parameters
193 - # can be beneficial
194 - current_config["max_autotune"] = True
195 - current_config["coordinate_descent_tuning"] = True
196 -
197 - # inductor can inplace modify the graph, so we need to copy it
198 - # see https://github.com/pytorch/pytorch/issues/138980
199 - graph = copy.deepcopy(graph)
200 -
201 - cache_data = vllm_backend.inductor_hash_cache
202 - if (runtime_shape, graph_index) in cache_data:
203 - # we compiled this graph before
204 - # so we can directly lookup the compiled graph via hash
205 - inductor_artifact = cache_data[(runtime_shape, graph_index)]
206 - hash_str = inductor_artifact.hash_str
207 - if graph_index == 0:
208 - # adds some info logging for the first graph
209 - logger.info(
210 - "Directly lookup the graph for shape %s from the cache",
211 - str(runtime_shape)) # noqa
72 + printer = pprint.PrettyPrinter(indent=4)
73 + data = printer.pformat(self.cache)
74 + f.write(data)
75 +
76 + def load(self,
77 + graph: fx.GraphModule,
78 + example_inputs: List[Any],
79 + graph_index: int,
80 + runtime_shape: Optional[int] = None) -> Optional[Callable]:
81 + if (runtime_shape, graph_index, self.compiler.name) not in self.cache:
82 + return None
83 + handle = self.cache[(runtime_shape, graph_index, self.compiler.name)]
84 + compiled_graph = self.compiler.load(handle, graph, example_inputs,
85 + graph_index, runtime_shape)
86 logger.debug(
213 - "directly lookup the %s-th graph for shape %s via hash %s",
214 - graph_index, str(runtime_shape), hash_str)
215 - from torch._inductor.codecache import FxGraphCache
216 - with patch("torch._inductor.codecache.FxGraphCache._get_shape_env",
217 - lambda *args, **kwargs: AlwaysHitShapeEnv()):
218 - inductor_compiled_graph = FxGraphCache._lookup_graph(
219 - hash_str, example_inputs, True, False)
220 - assert inductor_compiled_graph is not None, (
221 - "Inductor cache lookup failed. Please remove"
222 - f"the cache file {cache_data.cache_file_path} and try again." # noqa
223 - )
224 - inductor_artifact.file_path = inductor_compiled_graph.current_callable.__code__.co_filename # noqa
225 -
226 - # Inductor calling convention (function signature):
227 - # f(list) -> tuple
228 - # Dynamo calling convention (function signature):
229 - # f(*args) -> Any
230 -
231 - # need to know if the graph returns a tuple
232 - from torch._inductor.compile_fx import graph_returns_tuple
233 - returns_tuple = graph_returns_tuple(graph)
234 -
235 - # this is the callable we return to Dynamo to run
236 - def compiled_graph(*args):
237 - # convert args to list
238 - list_args = list(args)
239 - graph_output = inductor_compiled_graph(list_args)
240 - # unpack the tuple if needed
241 - if returns_tuple:
242 - return graph_output
243 - else:
244 - return graph_output[0]
245 - else:
246 - # it's the first time we compile this graph
247 - # the assumption is that we don't have nested Inductor compilation.
248 - # compiled_fx_graph_hash will only be called once, and we can hook
249 - # it to get the hash of the compiled graph directly.
250 -
251 - inductor_artifact = InductorArtifact()
252 - from torch._inductor.codecache import (FxGraphCache,
253 - compiled_fx_graph_hash)
254 - original_load = FxGraphCache.load
255 -
256 - def hijack_load(*args, **kwargs):
257 - inductor_compiled_graph = original_load(*args, **kwargs)
258 - inductor_artifact.file_path = inductor_compiled_graph.current_callable.__code__.co_filename # noqa
259 - return inductor_compiled_graph
260 -
261 - def hijack_compiled_fx_graph_hash(*args, **kwargs):
262 - out = compiled_fx_graph_hash(*args, **kwargs)
263 - inductor_artifact.hash_str = out[0]
264 - return out
265 -
266 - def _check_can_cache(*args, **kwargs):
267 - # no error means it can be cached.
268 - # Inductor refuses to cache the graph outside of Dynamo
269 - # tracing context, and also disables caching for graphs
270 - # with high-order ops.
271 - # For vLLM, in either case, we want to cache the graph.
272 - # see https://github.com/pytorch/pytorch/blob/9f5ebf3fc609105a74eab4ccc24932d6353ff566/torch/_inductor/codecache.py#L1221 # noqa
273 - return
274 -
275 - def _get_shape_env() -> AlwaysHitShapeEnv:
276 - return AlwaysHitShapeEnv()
277 -
278 - with ExitStack() as stack:
279 - if not cache_data.disabled:
280 - # compilation cache is enabled, patch several functions
281 -
282 - # hijack to get the compiled graph itself
283 - stack.enter_context(
284 - patch("torch._inductor.codecache.FxGraphCache.load",
285 - hijack_load))
286 -
287 - # for hijacking the hash of the compiled graph
288 - stack.enter_context(
289 - patch("torch._inductor.codecache.compiled_fx_graph_hash",
290 - hijack_compiled_fx_graph_hash))
291 -
292 - # for providing a dummy shape environment
293 - stack.enter_context(
294 - patch(
295 - "torch._inductor.codecache.FxGraphCache._get_shape_env",
296 - _get_shape_env))
297 -
298 - # for forcing the graph to be cached
299 - stack.enter_context(
300 - patch(
301 - "torch._inductor.codecache.FxGraphCache._check_can_cache",
302 - _check_can_cache))
303 -
304 - compiled_graph = compile_fx(graph,
305 - example_inputs,
306 - config_patches=current_config)
307 - # store the inductor_artifact in the cache
308 - cache_data[(runtime_shape, graph_index)] = inductor_artifact
87 + "Directly load the %s-th graph for shape %s from %s via "
88 + "handle %s", graph_index, str(runtime_shape), self.compiler.name,
89 + handle)
90 + return compiled_graph
91 +
92 + def compile(self,
93 + graph: fx.GraphModule,
94 + example_inputs,
95 + additional_inductor_config,
96 + compilation_config: CompilationConfig,
97 + graph_index: int = 0,
98 + num_graphs: int = 1,
99 + runtime_shape: Optional[int] = None) -> Any:
100 if graph_index == 0:
310 - # adds some info logging for the first graph
311 - logger.info("Cache the graph of shape %s for later use",
312 - str(runtime_shape))
313 - logger.debug(
314 - "store the %s-th graph for shape %s via hash %s from file %s",
315 - graph_index, str(runtime_shape), inductor_artifact.hash_str,
316 - inductor_artifact.file_path)
317 - # after compiling the last graph, record the end time
318 - if graph_index == num_graphs - 1:
319 - now = time.time()
320 - elapsed = now - compilation_start_time
321 - compilation_config.compilation_time += elapsed
322 - if runtime_shape is None:
323 - logger.info("Compiling a graph for general shape takes %.2f s",
324 - elapsed)
325 - else:
326 - logger.info("Compiling a graph for shape %s takes %.2f s",
327 - runtime_shape, elapsed)
101 + # before compiling the first graph, record the start time
102 + global compilation_start_time
103 + compilation_start_time = time.time()
104 +
105 + compilation_counter.num_backend_compilations += 1
106 +
107 + compiled_graph = None
108 +
109 + # try to load from the cache
110 + compiled_graph = self.load(graph, example_inputs, graph_index,
111 + runtime_shape)
112 + if compiled_graph is not None:
113 + if graph_index == 0:
114 + # adds some info logging for the first graph
115 + logger.info("Directly load the compiled graph for shape %s "
116 + "from the cache", str(runtime_shape)) # noqa
117 + return compiled_graph
118 +
119 + # no compiler cached the graph, or the cache is disabled,
120 + # we need to compile it
121 + compiled_graph, handle = self.compiler.compile(
122 + graph, example_inputs, additional_inductor_config, runtime_shape)
123 +
124 + assert compiled_graph is not None, "Failed to compile the graph"
125 +
126 + # store the artifact in the cache
127 + if handle is not None:
128 + self.cache[(runtime_shape, graph_index,
129 + self.compiler.name)] = handle
130 + if graph_index == 0:
131 + # adds some info logging for the first graph
132 + logger.info("Cache the graph of shape %s for later use",
133 + str(runtime_shape))
134 + logger.debug(
135 + "store the %s-th graph for shape %s from %s via handle %s",
136 + graph_index, str(runtime_shape), self.compiler.name, handle)
137 +
138 + # after compiling the last graph, record the end time
139 + if graph_index == num_graphs - 1:
140 + now = time.time()
141 + elapsed = now - compilation_start_time
142 + compilation_config.compilation_time += elapsed
143 + if runtime_shape is None:
144 + logger.info("Compiling a graph for general shape takes %.2f s",
145 + elapsed)
146 + else:
147 + logger.info("Compiling a graph for shape %s takes %.2f s",
148 + runtime_shape, elapsed)
149
329 - return compiled_graph
150 + return compiled_graph
151
152
153 @dataclasses.dataclass
@@ -436,16 +257,15 @@ class PiecewiseCompileInterpreter(torch.fx.Interpreter):
257 i for i, x in enumerate(args) if isinstance(x, torch.SymInt)
258 ]
259 global compilation_start_time
439 - compiled_graph_for_general_shape = wrap_inductor(
260 + compiled_graph_for_general_shape = self.vllm_backend.\
261 + compiler_manager.compile(
262 submod,
263 args,
264 self.compilation_config.inductor_compile_config,
265 self.compilation_config,
444 - self.vllm_backend,
266 graph_index=index,
267 num_graphs=len(self.compile_submod_names),
447 - runtime_shape=None,
448 - use_inductor=self.compilation_config.use_inductor)
268 + runtime_shape=None)
269
270 self.module.__dict__[target] = PiecewiseBackend(
271 submod, self.vllm_config, self.graph_pool, index,
@@ -483,7 +303,7 @@ class VllmBackend:
303 post_grad_passes: Sequence[Callable]
304 sym_tensor_indices: List[int]
305 input_buffers: List[torch.Tensor]
486 - inductor_hash_cache: InductorHashCache
306 + compiler_manager: CompilerManager
307
308 def __init__(
309 self,
@@ -507,6 +327,9 @@ class VllmBackend:
327 self.vllm_config = vllm_config
328 self.compilation_config = vllm_config.compilation_config
329
330 + self.compiler_manager: CompilerManager = CompilerManager(
331 + self.compilation_config.use_inductor)
332 +
333 # `torch.compile` is JIT compiled, so we don't need to
334 # do anything here
335
@@ -533,9 +356,11 @@ class VllmBackend:
356 # the cache dir will be the same so that we can reuse the compiled
357 # graph.
358
359 + factors = []
360 # 1. factors come from the vllm_config (it mainly summarizes how the
361 # model is created)
362 config_hash = vllm_config.compute_hash()
363 + factors.append(config_hash)
364
365 # 2. factors come from the code files that are traced by Dynamo (
366 # it mainly summarizes how the model is used in forward pass)
@@ -553,10 +378,15 @@ class VllmBackend:
378 import hashlib
379 code_hash = hashlib.md5(
380 "\n".join(hash_content).encode()).hexdigest()
381 + factors.append(code_hash)
382 +
383 + # 3. compiler hash
384 + compiler_hash = self.compiler_manager.compute_hash(vllm_config)
385 + factors.append(compiler_hash)
386 +
387 + # combine all factors to generate the cache dir
388 + hash_key = hashlib.md5(str(factors).encode()).hexdigest()[:10]
389
557 - # combine the two hashes to generate the cache dir
558 - hash_key = hashlib.md5(
559 - f"{config_hash}_{code_hash}".encode()).hexdigest()[:10]
390 cache_dir = os.path.join(
391 envs.VLLM_CACHE_ROOT,
392 "torch_compile_cache",
@@ -570,15 +400,16 @@ class VllmBackend:
400 cache_dir, f"rank_{vllm_config.parallel_config.rank}")
401 self.compilation_config.local_cache_dir = local_cache_dir
402
573 - disabled = envs.VLLM_DISABLE_COMPILE_CACHE
574 - self.inductor_hash_cache: InductorHashCache = InductorHashCache(
575 - local_cache_dir, disabled=disabled)
576 - if disabled:
403 + disable_cache = envs.VLLM_DISABLE_COMPILE_CACHE
404 +
405 + if disable_cache:
406 logger.info("vLLM's torch.compile cache is disabled.")
407 else:
408 logger.info("Using cache directory: %s for vLLM's torch.compile",
409 local_cache_dir)
410
411 + self.compiler_manager.initialize_cache(local_cache_dir, disable_cache)
412 +
413 # when dynamo calls the backend, it means the bytecode
414 # transform and analysis are done
415 compilation_counter.num_graphs_seen += 1
@@ -759,7 +590,7 @@ class PiecewiseBackend:
590 if self.is_last_graph and not self.to_be_compiled_sizes:
591 # no specific sizes to compile
592 # save the hash of the inductor graph for the next run
762 - self.vllm_backend.inductor_hash_cache.save_to_file()
593 + self.vllm_backend.compiler_manager.save_to_file()
594 end_monitoring_torch_compile(self.vllm_config)
595
596 def __call__(self, *args) -> Any:
@@ -782,16 +613,14 @@ class PiecewiseBackend:
613 entry.compiled = True
614 self.to_be_compiled_sizes.remove(runtime_shape)
615 # args are real arguments
785 - entry.runnable = wrap_inductor(
616 + entry.runnable = self.vllm_backend.compiler_manager.compile(
617 self.graph,
618 args,
619 self.compilation_config.inductor_compile_config,
620 self.compilation_config,
790 - self.vllm_backend,
621 graph_index=self.piecewise_compile_index,
622 num_graphs=self.total_piecewise_compiles,
793 - runtime_shape=runtime_shape,
794 - use_inductor=self.compilation_config.use_inductor)
623 + runtime_shape=runtime_shape)
624
625 # finished compilations for all required shapes
626 if self.is_last_graph and not self.to_be_compiled_sizes:
vllm/compilation/compiler_interface.py new
+340
@@ -0,0 +1,340 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +import copy
3 +import hashlib
4 +import os
5 +from contextlib import ExitStack
6 +from typing import Any, Callable, Dict, List, Optional, Tuple
7 +from unittest.mock import patch
8 +
9 +import torch
10 +import torch._inductor.compile_fx
11 +import torch.fx as fx
12 +
13 +from vllm.config import VllmConfig
14 +
15 +
16 +class CompilerInterface:
17 + """
18 + The interface for a compiler that can be used by vLLM.
19 + """
20 + # The name of the compiler, e.g. inductor.
21 + # This is a class-level attribute.
22 + name: str
23 +
24 + def initialize_cache(self, cache_dir: str, disable_cache: bool = False):
25 + """
26 + when the vLLM process uses `cache_dir` as the cache directory,
27 + the compiler should initialize itself with the cache directory,
28 + e.g. by re-directing its own cache directory to a sub-directory.
29 + """
30 + pass
31 +
32 + def compute_hash(self, vllm_config: VllmConfig) -> str:
33 + """
34 + Gather all the relevant information from the VLLM config,
35 + to compute a hash so that we can cache the compiled model.
36 +
37 + See :meth:`VllmConfig.compute_hash` to check what information
38 + is already considered by default. This function should only
39 + consider the information that is specific to the compiler.
40 + """
41 + return ""
42 +
43 + def compile(
44 + self,
45 + graph: fx.GraphModule,
46 + example_inputs: List[Any],
47 + compiler_config: Dict[str, Any],
48 + runtime_shape: Optional[int] = None
49 + ) -> Tuple[Optional[Callable], Optional[Any]]:
50 + """
51 + Compile the graph with the given example inputs and compiler config,
52 + with a runtime shape. If the `runtime_shape` is None, it means
53 + the `example_inputs` have a dynamic shape. Otherwise, the
54 + `runtime_shape` specifies the shape of the inputs. Right now we only
55 + support one variable shape for all inputs, which is the batchsize
56 + (number of tokens) during inference.
57 +
58 + Dynamo will make sure `graph(*example_inputs)` is valid.
59 +
60 + The function should return a compiled callable function, as well as
61 + a handle that can be used to directly load the compiled function.
62 +
63 + The handle should be a plain Python object, preferably a string or a
64 + file path for readability.
65 +
66 + If the compiler doesn't support caching, it should return None for the
67 + handle. If the compiler fails to compile the graph, it should return
68 + None for the compiled function as well.
69 + """
70 + return None, None
71 +
72 + def load(self,
73 + handle: Any,
74 + graph: fx.GraphModule,
75 + example_inputs: List[Any],
76 + graph_index: int,
77 + runtime_shape: Optional[int] = None) -> Callable:
78 + """
79 + Load the compiled function from the handle.
80 + Raises an error if the handle is invalid.
81 +
82 + The handle is the second return value of the `compile` function.
83 + """
84 + raise NotImplementedError("caching is not supported")
85 +
86 +
87 +class AlwaysHitShapeEnv:
88 + """
89 + Why do we need this class:
90 +
91 + For normal `torch.compile` usage, every compilation will have
92 + one Dynamo bytecode compilation and one Inductor compilation.
93 + The Inductor compilation happens under the context of the
94 + Dynamo bytecode compilation, and that context is used to
95 + determine the dynamic shape information, etc.
96 +
97 + For our use case, we only run Dynamo bytecode compilation once,
98 + and run Inductor compilation multiple times with different shapes
99 + plus a general shape. The compilation for specific shapes happens
100 + outside of the context of the Dynamo bytecode compilation. At that
101 + time, we don't have shape environment to provide to Inductor, and
102 + it will fail the Inductor code cache lookup.
103 +
104 + By providing a dummy shape environment that always hits, we can
105 + make the Inductor code cache lookup always hit, and we can
106 + compile the graph for different shapes as needed.
107 +
108 + The following dummy methods are obtained by trial-and-error
109 + until it works.
110 + """
111 +
112 + def __init__(self) -> None:
113 + self.guards: List[Any] = []
114 +
115 + def evaluate_guards_expression(self, *args, **kwargs):
116 + return True
117 +
118 + def get_pruned_guards(self, *args, **kwargs):
119 + return []
120 +
121 + def produce_guards_expression(self, *args, **kwargs):
122 + return ""
123 +
124 +
125 +class InductorAdaptor(CompilerInterface):
126 + """
127 + The adaptor for the Inductor compiler, version 2.5 and 2.6.
128 + """
129 + name = "inductor"
130 +
131 + def compute_hash(self, vllm_config: VllmConfig) -> str:
132 + factors: List[Any] = []
133 + # summarize system state
134 + from torch._inductor.codecache import CacheBase
135 + system_factors = CacheBase.get_system()
136 + factors.append(system_factors)
137 +
138 + # summarize pytorch state
139 + from torch._inductor.codecache import torch_key
140 + torch_factors = torch_key()
141 + factors.append(torch_factors)
142 + hash_str = hashlib.md5(str(factors).encode()).hexdigest()[:10]
143 + return hash_str
144 +
145 + def initialize_cache(self, cache_dir: str, disable_cache: bool = False):
146 + if disable_cache:
147 + return
148 + # redirect the cache directory to a sub-directory
149 + # set flags so that Inductor and Triton store their cache
150 + # in the cache_dir, then users only need to copy the cache_dir
151 + # to another machine to reuse the cache.
152 + inductor_cache = os.path.join(cache_dir, "inductor_cache")
153 + os.makedirs(inductor_cache, exist_ok=True)
154 + os.environ["TORCHINDUCTOR_CACHE_DIR"] = inductor_cache
155 + triton_cache = os.path.join(cache_dir, "triton_cache")
156 + os.makedirs(triton_cache, exist_ok=True)
157 + os.environ["TRITON_CACHE_DIR"] = triton_cache
158 +
159 + def compile(
160 + self,
161 + graph: fx.GraphModule,
162 + example_inputs: List[Any],
163 + compiler_config: Dict[str, Any],
164 + runtime_shape: Optional[int] = None
165 + ) -> Tuple[Optional[Callable], Optional[Any]]:
166 + from torch._inductor import config
167 + current_config = config.get_config_copy()
168 + from torch._inductor.compile_fx import compile_fx
169 +
170 + # disable remote cache
171 + current_config["fx_graph_cache"] = True
172 + current_config["fx_graph_remote_cache"] = False
173 +
174 + if compiler_config is not None:
175 + current_config.update(compiler_config)
176 +
177 + if isinstance(runtime_shape, int):
178 + # for a specific batchsize, tuning triton kernel parameters
179 + # can be beneficial
180 + current_config["max_autotune"] = True
181 + current_config["coordinate_descent_tuning"] = True
182 +
183 + # inductor can inplace modify the graph, so we need to copy it
184 + # see https://github.com/pytorch/pytorch/issues/138980
185 + graph = copy.deepcopy(graph)
186 +
187 + # it's the first time we compile this graph
188 + # the assumption is that we don't have nested Inductor compilation.
189 + # compiled_fx_graph_hash will only be called once, and we can hook
190 + # it to get the hash of the compiled graph directly.
191 +
192 + hash_str, file_path = None, None
193 + from torch._inductor.codecache import (FxGraphCache,
194 + compiled_fx_graph_hash)
195 +
196 + if torch.__version__.startswith("2.5"):
197 + original_load = FxGraphCache.load
198 + original_load_name = "torch._inductor.codecache.FxGraphCache.load"
199 +
200 + def hijack_load(*args, **kwargs):
201 + inductor_compiled_graph = original_load(*args, **kwargs)
202 + nonlocal file_path
203 + file_path = inductor_compiled_graph.current_callable.__code__.co_filename # noqa
204 + return inductor_compiled_graph
205 +
206 + hijacked_compile_fx_inner = torch._inductor.compile_fx.compile_fx_inner # noqa
207 + elif torch.__version__ >= "2.6":
208 + # function renamed in 2.6
209 + original_load_name = None
210 +
211 + def hijacked_compile_fx_inner(*args, **kwargs):
212 + output = torch._inductor.compile_fx.compile_fx_inner(
213 + *args, **kwargs)
214 + nonlocal hash_str
215 + inductor_compiled_graph = output
216 + if inductor_compiled_graph is not None:
217 + nonlocal file_path
218 + file_path = inductor_compiled_graph.current_callable.__code__.co_filename # noqa
219 + hash_str = inductor_compiled_graph._fx_graph_cache_key
220 + return output
221 +
222 + def hijack_compiled_fx_graph_hash(*args, **kwargs):
223 + out = compiled_fx_graph_hash(*args, **kwargs)
224 + nonlocal hash_str
225 + hash_str = out[0]
226 + return out
227 +
228 + def _check_can_cache(*args, **kwargs):
229 + # no error means it can be cached.
230 + # Inductor refuses to cache the graph outside of Dynamo
231 + # tracing context, and also disables caching for graphs
232 + # with high-order ops.
233 + # For vLLM, in either case, we want to cache the graph.
234 + # see https://github.com/pytorch/pytorch/blob/9f5ebf3fc609105a74eab4ccc24932d6353ff566/torch/_inductor/codecache.py#L1221 # noqa
235 + return
236 +
237 + def _get_shape_env() -> AlwaysHitShapeEnv:
238 + return AlwaysHitShapeEnv()
239 +
240 + with ExitStack() as stack:
241 + # hijack to get the compiled graph itself
242 + if original_load_name is not None:
243 + stack.enter_context(patch(original_load_name, hijack_load))
244 +
245 + # for hijacking the hash of the compiled graph
246 + stack.enter_context(
247 + patch("torch._inductor.codecache.compiled_fx_graph_hash",
248 + hijack_compiled_fx_graph_hash))
249 +
250 + # for providing a dummy shape environment
251 + stack.enter_context(
252 + patch("torch._inductor.codecache.FxGraphCache._get_shape_env",
253 + _get_shape_env))
254 +
255 + # for forcing the graph to be cached
256 + stack.enter_context(
257 + patch(
258 + "torch._inductor.codecache.FxGraphCache._check_can_cache",
259 + _check_can_cache))
260 +
261 + compiled_graph = compile_fx(
262 + graph,
263 + example_inputs,
264 + inner_compile=hijacked_compile_fx_inner,
265 + config_patches=current_config)
266 +
267 + assert hash_str is not None, (
268 + "failed to get the hash of the compiled graph")
269 + assert file_path is not None, (
270 + "failed to get the file path of the compiled graph")
271 + return compiled_graph, (hash_str, file_path)
272 +
273 + def load(self,
274 + handle: Any,
275 + graph: fx.GraphModule,
276 + example_inputs: List[Any],
277 + graph_index: int,
278 + runtime_shape: Optional[int] = None) -> Callable:
279 + assert isinstance(handle, tuple)
280 + assert isinstance(handle[0], str)
281 + assert isinstance(handle[1], str)
282 + hash_str = handle[0]
283 +
284 + from torch._inductor.codecache import FxGraphCache
285 + with patch("torch._inductor.codecache.FxGraphCache._get_shape_env",
286 + lambda *args, **kwargs: AlwaysHitShapeEnv()):
287 + if torch.__version__.startswith("2.5"):
288 + inductor_compiled_graph = FxGraphCache._lookup_graph(
289 + hash_str, example_inputs, True, False)
290 + assert inductor_compiled_graph is not None, (
291 + "Inductor cache lookup failed. Please remove"
292 + f"the cache directory and try again." # noqa
293 + )
294 + elif torch.__version__ >= "2.6":
295 + from torch._inductor.output_code import (
296 + CompiledFxGraphConstantsWithGm)
297 + constants = CompiledFxGraphConstantsWithGm(graph)
298 + inductor_compiled_graph, _ = FxGraphCache._lookup_graph(
299 + hash_str, example_inputs, True, None, constants)
300 + assert inductor_compiled_graph is not None, (
301 + "Inductor cache lookup failed. Please remove"
302 + f"the cache directory and try again." # noqa
303 + )
304 +
305 + # Inductor calling convention (function signature):
306 + # f(list) -> tuple
307 + # Dynamo calling convention (function signature):
308 + # f(*args) -> Any
309 +
310 + # need to know if the graph returns a tuple
311 + from torch._inductor.compile_fx import graph_returns_tuple
312 + returns_tuple = graph_returns_tuple(graph)
313 +
314 + # this is the callable we return to Dynamo to run
315 + def compiled_graph(*args):
316 + # convert args to list
317 + list_args = list(args)
318 + graph_output = inductor_compiled_graph(list_args)
319 + # unpack the tuple if needed
320 + if returns_tuple:
321 + return graph_output
322 + else:
323 + return graph_output[0]
324 +
325 + return compiled_graph
326 +
327 +
328 +class EagerAdaptor(CompilerInterface):
329 + name = "eager"
330 +
331 + def compile(
332 + self,
333 + graph: fx.GraphModule,
334 + example_inputs: List[Any],
335 + compiler_config: Dict[str, Any],
336 + runtime_shape: Optional[int] = None
337 + ) -> Tuple[Optional[Callable], Optional[Any]]:
338 + # we don't need to compile the graph, just return the graph itself.
339 + # It does not support caching, return None for the handle.
340 + return graph, None
vllm/compilation/counter.py
+1 -1
@@ -13,7 +13,7 @@ class CompilationCounter:
13 num_piecewise_graphs_seen: int = 0
14 # not including the splitting ops
15 num_piecewise_capturable_graphs_seen: int = 0
16 - num_inductor_compilations: int = 0
16 + num_backend_compilations: int = 0
17 num_cudagraph_caputured: int = 0
18
19 def clone(self) -> "CompilationCounter":
vllm/compilation/inductor_pass.py
-1
@@ -13,7 +13,6 @@ from torch import fx
13 class InductorPass(ABC):
14 """
15 General custom inductor pass interface.
16 - TODO(torch==2.6) use torch._inductor.custom_graph_pass.CustomGraphPass
16 """
17
18 @abstractmethod
vllm/compilation/pass_manager.py
+15 -1
@@ -2,6 +2,7 @@
2
3 from typing import Any, Dict, List
4
5 +import torch
6 from torch import fx as fx
7
8 from vllm.config import CompilationConfig
@@ -15,7 +16,17 @@ from .reshapes import RedundantReshapesPass
16 logger = init_logger(__name__)
17
18
18 -class PostGradPassManager:
19 +class PlaceHolder:
20 + pass
21 +
22 +
23 +if torch.__version__ < "2.6":
24 + Parent = PlaceHolder # type: ignore
25 +else:
26 + Parent = torch._inductor.custom_graph_pass.CustomGraphPass # type: ignore
27 +
28 +
29 +class PostGradPassManager(Parent):
30 """
31 The pass manager for post-grad passes.
32 It handles configuration, adding custom passes, and running passes.
@@ -55,6 +66,9 @@ class PostGradPassManager:
66 assert isinstance(pass_, InductorPass)
67 self.passes.append(pass_)
68
69 + def uuid(self):
70 + return self.__getstate__()
71 +
72 def __getstate__(self) -> Dict[str, List[Any]]:
73 """
74 Custom pickling for the pass manager, as some passes cannot be pickled.
vllm/config.py
-9
@@ -3072,15 +3072,6 @@ class VllmConfig:
3072 the final hidden states.
3073 """
3074 factors: List[Any] = []
3075 - # summarize system state
3076 - from torch._inductor.codecache import CacheBase
3077 - system_factors = CacheBase.get_system()
3078 - factors.append(system_factors)
3079 -
3080 - # summarize pytorch state
3081 - from torch._inductor.codecache import torch_key
3082 - torch_factors = torch_key()
3083 - factors.append(torch_factors)
3075
3076 # summarize vllm config
3077 vllm_factors: List[Any] = []