A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[torch.compile] Inductor code caching fix (#10273)
Signed-off-by: luka <luka@neuralmagic.com> Signed-off-by: Luka Govedic <luka.govedic@gmail.com>
Luka Govedič committed
Nov 21, 2024 at 00:44 UTC
8b0fe06c890a202eba24d517cc77562e4a8b0d0c
14 files changed
+604
-288
tests/compile/backend.py
+10
-6
@@ -1,7 +1,9 @@
1
from copy import deepcopy
2
-from typing import Callable
2
+from typing import Callable, Union
3
4
-import torch
4
+from torch import fx
5
+
6
+from vllm.compilation.inductor_pass import InductorPass
7
8
9
class TestBackend:
@@ -11,19 +13,21 @@ class TestBackend:
13
It also saves the graph before and after the custom passes for inspection.
14
"""
15
14
- def __init__(self, *args: Callable[[torch.fx.Graph], None]):
15
- self.custom_passes = args
16
+ def __init__(self, *passes: Union[InductorPass, Callable[[fx.Graph],
17
+ None]]):
18
+ self.custom_passes = list(passes)
19
from torch._inductor import config
20
self.current_config = config.shallow_copy_dict()
21
+ self.current_config['force_disable_caches'] = True
22
self.current_config['post_grad_custom_post_pass'] = self.post_pass
23
20
- def __call__(self, graph: torch.fx.GraphModule, example_inputs):
24
+ def __call__(self, graph: fx.GraphModule, example_inputs):
25
from torch._inductor.compile_fx import compile_fx
26
return compile_fx(graph,
27
example_inputs,
28
config_patches=self.current_config)
29
26
- def post_pass(self, graph: torch.fx.Graph):
30
+ def post_pass(self, graph: fx.Graph):
31
self.graph_pre_pass = deepcopy(graph)
32
for pass_ in self.custom_passes:
33
pass_(graph)
tests/compile/test_functionalization.py
new
+95
@@ -0,0 +1,95 @@
1
+import pytest
2
+import torch
3
+
4
+import vllm.envs as envs
5
+from vllm import LLM, SamplingParams
6
+from vllm.compilation.fix_functionalization import FixFunctionalizationPass
7
+from vllm.compilation.fusion import (FusionPass, find_auto_fn,
8
+ find_auto_fn_maybe)
9
+from vllm.compilation.reshapes import RedundantReshapesPass
10
+from vllm.compilation.vllm_inductor_pass import is_func
11
+from vllm.config import CompilationConfig
12
+
13
+from .backend import TestBackend
14
+
15
+OPS_IN_MODEL = [
16
+ torch.ops._C.rotary_embedding.default,
17
+ torch.ops._C.fused_add_rms_norm.default,
18
+ torch.ops._C.silu_and_mul.default,
19
+]
20
+
21
+RMS_OP = torch.ops._C.rms_norm.default
22
+
23
+RMS_QUANT_OPS = {
24
+ "static_fp8": [
25
+ torch.ops._C.rms_norm_static_fp8_quant.default,
26
+ torch.ops._C.fused_add_rms_norm_static_fp8_quant.default
27
+ ],
28
+}
29
+
30
+prompts = [
31
+ "Hello, my name is",
32
+ "The president of the United States is",
33
+ "The capital of France is",
34
+ "The future of AI is",
35
+]
36
+
37
+
38
+@pytest.mark.parametrize("model",
39
+ ["nm-testing/TinyLlama-1.1B-Chat-v1.0-FP8-e2e"])
40
+@pytest.mark.parametrize("do_fusion", [True, False])
41
+@pytest.mark.skipif(envs.VLLM_TARGET_DEVICE != "cuda",
42
+ reason="Only test on CUDA")
43
+def test_fix_functionalization(model: str, do_fusion: bool):
44
+ torch.set_default_device("cuda")
45
+
46
+ config = CompilationConfig.PassConfig(enable_fusion=do_fusion,
47
+ enable_reshape=True)
48
+ reshape_pass = RedundantReshapesPass(config)
49
+ fusion_pass = FusionPass.instance(config)
50
+
51
+ passes = [reshape_pass, fusion_pass] if do_fusion else [reshape_pass]
52
+ func_pass = FixFunctionalizationPass(config)
53
+ backend_func = TestBackend(*passes, func_pass)
54
+ backend_no_func = TestBackend(*passes)
55
+
56
+ # instantiate a full engine and manually compile the model 2x
57
+ # (with and without FixFunctionalizationPass)
58
+ llm = LLM(model=model, enforce_eager=True)
59
+ model_runner = llm.llm_engine.model_executor.driver_worker.model_runner
60
+ orig_model = model_runner.model
61
+ # TODO mark inputs dynamic? (currently torch.compile is triggered 4x)
62
+ # Can only do that by using the decorator but then we'd have to instantiate
63
+ # 2 LLM instances.
64
+
65
+ sampling_params = SamplingParams(temperature=0.0, top_p=1.0)
66
+ model_runner.model = torch.compile(orig_model,
67
+ fullgraph=True,
68
+ backend=backend_func)
69
+ gen_func = llm.generate(prompts, sampling_params)
70
+
71
+ model_runner.model = torch.compile(orig_model,
72
+ fullgraph=True,
73
+ backend=backend_no_func)
74
+ gen_no_func = llm.generate(prompts, sampling_params)
75
+
76
+ for output_func, output_no_func in zip(gen_func, gen_no_func):
77
+ assert output_func.outputs[0].text == output_no_func.outputs[0].text
78
+
79
+ # OPS_IN_MODEL always appear. RMS_OP is fused away if we run fusion,
80
+ # and replaced by fused quantized ops in RMS_QUANT_OPS.
81
+ ops = OPS_IN_MODEL + (RMS_QUANT_OPS["static_fp8"]
82
+ if do_fusion else [RMS_OP])
83
+
84
+ for op in ops:
85
+ find_auto_fn(backend_no_func.graph_post_pass.nodes, op)
86
+ assert find_auto_fn_maybe(backend_func.graph_post_pass.nodes,
87
+ op) is None # noqa: E501
88
+
89
+ # make sure the ops were all de-functionalized
90
+ found = dict()
91
+ for node in backend_func.graph_post_pass.nodes:
92
+ for op in ops:
93
+ if is_func(node, op):
94
+ found[op] = True
95
+ assert all(found[op] for op in ops)
tests/compile/test_fusion.py
+5
-6
@@ -38,12 +38,6 @@ class TestModel(torch.nn.Module):
38
return y3
39
40
41
-# Init does pattern registration, which can only happen once
42
-config = CompilationConfig(enable_fusion=True)
43
-reshape_pass = RedundantReshapesPass(config)
44
-fusion_pass = FusionPass.instance(config)
45
-
46
-
41
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
42
@pytest.mark.parametrize("hidden_size", [64, 3392, 4096])
43
@pytest.mark.parametrize("num_tokens", [7, 256, 533, 2048, 2049])
@@ -58,6 +52,11 @@ def test_fusion_rmsnorm_quant(dtype, hidden_size, num_tokens, eps):
52
pytest.skip("Only test eps=1e-5 for now")
53
54
# Reshape pass is needed for the fusion pass to work
55
+ config = CompilationConfig.PassConfig(enable_fusion=True,
56
+ enable_reshape=True)
57
+ reshape_pass = RedundantReshapesPass(config)
58
+ fusion_pass = FusionPass.instance(config)
59
+
60
backend = TestBackend(reshape_pass, fusion_pass)
61
model = TestModel(hidden_size, eps)
62
tests/compile/test_pass_manager.py
new
+35
@@ -0,0 +1,35 @@
1
+import pickle
2
+
3
+import pytest
4
+import torch
5
+from torch._inductor.codecache import BypassFxGraphCache
6
+
7
+from vllm.compilation.config import CompilationConfig
8
+from vllm.compilation.inductor_pass import (CallableInductorPass,
9
+ as_inductor_pass)
10
+from vllm.compilation.pass_manager import PostGradPassManager
11
+
12
+
13
+def simple_callable(graph: torch.fx.Graph):
14
+ pass
15
+
16
+
17
+@as_inductor_pass(files=(__file__, ))
18
+def callable_decorated(graph: torch.fx.Graph):
19
+ pass
20
+
21
+
22
+@pytest.mark.parametrize(
23
+ "works, callable",
24
+ [(False, simple_callable), (True, callable_decorated),
25
+ (True, CallableInductorPass(simple_callable, "simple_callable"))])
26
+def test_pass_manager(works: bool, callable):
27
+ config = CompilationConfig().pass_config
28
+ pass_manager = PostGradPassManager([callable])
29
+ pass_manager.configure(config) # Adds default passes
30
+
31
+ if works:
32
+ pickle.dumps(pass_manager)
33
+ else:
34
+ with pytest.raises(BypassFxGraphCache):
35
+ pickle.dumps(pass_manager)
vllm/compilation/backends.py
+19
-217
@@ -1,6 +1,5 @@
1
import copy
2
import dataclasses
3
-import operator
3
from contextlib import ExitStack
4
from typing import Any, Callable, Dict, List, Optional, Sequence, Set, Tuple
5
from unittest.mock import patch
@@ -11,205 +10,15 @@ import torch.fx as fx
10
import vllm.envs as envs
11
from vllm.config import CompilationConfig
12
from vllm.logger import init_logger
14
-from vllm.utils import combine_fx_passes, weak_ref_tensors
13
+from vllm.utils import weak_ref_tensors
14
15
from .counter import compilation_counter
17
-from .fusion import FusionPass
18
-from .reshapes import RedundantReshapesPass
16
+from .inductor_pass import InductorPass
17
+from .pass_manager import PostGradPassManager
18
19
logger = init_logger(__name__)
20
21
23
-def fix_functionalization(graph: fx.Graph):
24
- """
25
- Rewrite the graph module to replace the pattern involving
26
- torch._higher_order_ops.auto_functionalize.auto_functionalized
27
- with a direct call to the inplace custom op.
28
-
29
- # TODO: check if PyTorch nightly has fixed this issue
30
- """
31
-
32
- # debug code, if we want to see the graph before the transformation
33
- # with open("before.py", "w") as f:
34
- # print(graph.python_code(root_module="self", verbose=True).src, file=f)
35
-
36
- nodes_to_remove = []
37
-
38
- for node in graph.nodes:
39
- # Identify the auto_functionalized node
40
- if node.op == 'call_function' and node.target == torch._higher_order_ops.auto_functionalize.auto_functionalized: # noqa
41
- if node.args[0] == torch.ops._C.rotary_embedding.default:
42
- # manual replace for rotary_embedding
43
-
44
- # Now, collect the arguments
45
- kwargs = node.kwargs
46
-
47
- query = kwargs['query']
48
- mm_node = query.args[0].args[0]
49
-
50
- # Create a new call to torch.ops._C.rotary_embedding.default
51
- with graph.inserting_before(node):
52
- # just insert the call to the custom op
53
- # NOTE: don't run dead code elimination,
54
- # otherwise this op will be removed
55
- graph.call_function(torch.ops._C.rotary_embedding.default,
56
- kwargs=kwargs)
57
-
58
- # Remove the auto_functionalized node
59
- # Since the node may have outputs, we need to handle its users
60
- # Replace uses of the outputs (getitem nodes) with mm_node
61
- for user in list(node.users):
62
- if user.op == 'call_function' and user.target == operator.getitem: # noqa
63
- # Remove the getitem node
64
- for getitem_user in list(user.users):
65
- if (getitem_user.op == 'call_function'
66
- and getitem_user.target
67
- == torch.ops.aten.slice_scatter.default):
68
- # Replace the uses of slice_scatter node
69
- # with mm_node
70
- getitem_user.replace_all_uses_with(mm_node)
71
- nodes_to_remove.append(getitem_user)
72
- nodes_to_remove.append(user)
73
- nodes_to_remove.append(node)
74
-
75
- elif node.args[0] == torch.ops._C.fused_add_rms_norm.default:
76
- # manual replace for fused_add_rms_norm
77
- # this is the most effective optimization for llama
78
- # failing to do this will result in many unnecessary copies
79
-
80
- kwargs = node.kwargs
81
-
82
- input = kwargs['input']
83
- residual = kwargs['residual']
84
-
85
- # Create a new call to torch.ops._C.rotary_embedding.default
86
- with graph.inserting_before(node):
87
- # just insert the call to the custom op
88
- # NOTE: don't run dead code elimination,
89
- # otherwise this op will be removed
90
- graph.call_function(
91
- torch.ops._C.fused_add_rms_norm.default, kwargs=kwargs)
92
-
93
- for user in list(node.users):
94
- if user.op == 'call_function' and user.target == operator.getitem: # noqa
95
- # Remove the getitem node
96
- if user.args[1] == 1:
97
- replace_node = input
98
- elif user.args[1] == 2:
99
- replace_node = residual
100
- user.replace_all_uses_with(replace_node)
101
- nodes_to_remove.append(user)
102
- nodes_to_remove.append(node)
103
- elif (node.args[0] ==
104
- torch.ops._C.fused_add_rms_norm_static_fp8_quant.default):
105
- # manual replace for fused_add_rms_norm_static_fp8_quant
106
- # this is the most effective optimization for llama
107
- # failing to do this will result in many unnecessary copies
108
-
109
- kwargs = node.kwargs
110
-
111
- result = kwargs['result']
112
- residual = kwargs['residual']
113
-
114
- # Create a new call to
115
- # torch.ops._C.fused_add_rms_norm_static_fp8_quant.default
116
- with graph.inserting_before(node):
117
- # just insert the call to the custom op
118
- # NOTE: don't run dead code elimination,
119
- # otherwise this op will be removed
120
- graph.call_function(
121
- torch.ops._C.fused_add_rms_norm_static_fp8_quant.
122
- default,
123
- kwargs=kwargs)
124
-
125
- for user in list(node.users):
126
- if user.op == 'call_function' and user.target == operator.getitem: # noqa
127
- # Remove the getitem node
128
- if user.args[1] == 1:
129
- replace_node = result
130
- elif user.args[1] == 2:
131
- replace_node = residual
132
- user.replace_all_uses_with(replace_node)
133
- nodes_to_remove.append(user)
134
- nodes_to_remove.append(node)
135
-
136
- elif node.args[0] == torch.ops._C.rms_norm.default:
137
- # manual replace for rms_norm
138
-
139
- kwargs = node.kwargs
140
-
141
- replace_node = kwargs['result']
142
- # Create a new call to torch.ops._C.rms_norm.default
143
- with graph.inserting_before(node):
144
- # just insert the call to the custom op
145
- # NOTE: don't run dead code elimination,
146
- # otherwise this op will be removed
147
- graph.call_function(torch.ops._C.rms_norm.default,
148
- kwargs=kwargs)
149
-
150
- for user in list(node.users):
151
- if user.op == 'call_function' and user.target == operator.getitem: # noqa
152
- user.replace_all_uses_with(replace_node)
153
- nodes_to_remove.append(user)
154
- nodes_to_remove.append(node)
155
-
156
- elif node.args[
157
- 0] == torch.ops._C.rms_norm_static_fp8_quant.default: # noqa
158
- # manual replace for rms_norm_static_fp8_quant
159
-
160
- kwargs = node.kwargs
161
-
162
- replace_node = kwargs['result']
163
- # Create a new call to torch.ops._C.rms_norm_static_fp8_quant.default # noqa
164
- with graph.inserting_before(node):
165
- # just insert the call to the custom op
166
- # NOTE: don't run dead code elimination,
167
- # otherwise this op will be removed
168
- graph.call_function(
169
- torch.ops._C.rms_norm_static_fp8_quant.default,
170
- kwargs=kwargs)
171
-
172
- for user in list(node.users):
173
- if user.op == 'call_function' and user.target == operator.getitem: # noqa
174
- user.replace_all_uses_with(replace_node)
175
- nodes_to_remove.append(user)
176
- nodes_to_remove.append(node)
177
-
178
- elif node.args[0] == torch.ops._C.silu_and_mul.default:
179
- # manual replace for silu_and_mul
180
-
181
- kwargs = node.kwargs
182
-
183
- input = kwargs['input']
184
- out = kwargs['out']
185
-
186
- # Create a new call to torch.ops._C.silu_and_mul.default
187
- # cannot use kwargs, because we have an `out`, see https://github.com/pytorch/pytorch/blob/a00faf440888ffb724bad413f329a49e2b6388e7/torch/_inductor/lowering.py#L351 # noqa
188
- with graph.inserting_before(node):
189
- # just insert the call to the custom op
190
- # NOTE: don't run dead code elimination,
191
- # otherwise this op will be removed
192
- graph.call_function(
193
- torch.ops._C.silu_and_mul.default,
194
- args=(out, input),
195
- )
196
- replace_node = out
197
-
198
- for user in list(node.users):
199
- if user.op == 'call_function' and user.target == operator.getitem: # noqa
200
- user.replace_all_uses_with(replace_node)
201
- nodes_to_remove.append(user)
202
- nodes_to_remove.append(node)
203
-
204
- # Remove the nodes all at once
205
- for node in nodes_to_remove:
206
- graph.erase_node(node)
207
-
208
- # debug code, if we want to see the graph after the transformation
209
- # with open("after.py", "w") as f:
210
- # print(graph.python_code(root_module="self", verbose=True).src, file=f)
211
-
212
-
22
def wrap_inductor(graph,
23
example_inputs,
24
additional_inductor_config,
@@ -368,12 +177,8 @@ class VllmBackend:
177
The major work of this backend is to split the graph into
178
piecewise graphs, and pass them to the piecewise backend.
179
371
- This backend also handles custom passes and adds them to Inductor config.
372
- The order of the post-grad post-passes is:
373
- 1. post_grad_passes (constructor parameter)
374
- 2. config["post_grad_custom_post_pass"]
375
- 3. fix_functionalization
376
- This way, all passes operate on a functionalized graph.
180
+ This backend also adds the PostGradPassManager to Inductor config,
181
+ which handles the post-grad passes.
182
"""
183
184
compilation_configs: CompilationConfig
@@ -402,7 +207,9 @@ class VllmBackend:
207
# streams, it might not be safe to share a global pool.
208
# only investigate this when we use multiple streams
209
self.graph_pool = global_graph_pool
405
- self.post_grad_passes = []
210
+
211
+ # Passes to run on the graph post-grad.
212
+ self.post_grad_pass_manager = PostGradPassManager()
213
214
self.sym_tensor_indices = []
215
self.input_buffers = []
@@ -412,24 +219,19 @@ class VllmBackend:
219
# `torch.compile` is JIT compiled, so we don't need to
220
# do anything here
221
415
- def add_passes_to_config(self):
222
+ def configure_post_pass(self):
223
config = self.compilation_configs
417
- passes = list(self.post_grad_passes)
418
-
419
- passes = passes + [RedundantReshapesPass(config)]
420
-
421
- if config.enable_fusion:
422
- passes = passes + [FusionPass.instance(config)]
224
+ self.post_grad_pass_manager.configure(config.pass_config)
225
226
+ # Post-grad custom passes are run using the post_grad_custom_post_pass
227
+ # hook. If a pass for that hook exists, add it to the pass manager.
228
inductor_config = config.inductor_compile_config
425
- if "post_grad_custom_post_pass" in inductor_config:
426
- passes = passes + [inductor_config["post_grad_custom_post_pass"]]
427
-
428
- # add the fix_functionalization pass last, so that all other
429
- # passes operate on a functionalized graph
430
- passes = passes + [fix_functionalization]
431
- combined_pass = combine_fx_passes(passes)
432
- inductor_config["post_grad_custom_post_pass"] = combined_pass
229
+ PASS_KEY = "post_grad_custom_post_pass"
230
+ if PASS_KEY in inductor_config:
231
+ # Config should automatically wrap all inductor passes
232
+ assert isinstance(inductor_config[PASS_KEY], InductorPass)
233
+ self.post_grad_pass_manager.add(inductor_config[PASS_KEY])
234
+ inductor_config[PASS_KEY] = self.post_grad_pass_manager
235
236
def __call__(self, graph: fx.GraphModule, example_inputs) -> Callable:
237
@@ -444,7 +246,7 @@ class VllmBackend:
246
# we get the sizes to capture for cudagraph
247
# from compilation context
248
self.compilation_configs.init_during_runtime()
447
- self.add_passes_to_config()
249
+ self.configure_post_pass()
250
251
self.split_gm, self.piecewise_graphs = split_graph(
252
graph, self.compilation_configs.splitting_ops)
vllm/compilation/fix_functionalization.py
new
+177
@@ -0,0 +1,177 @@
1
+import operator
2
+from typing import Dict, Iterable, List, Optional, Tuple, Union
3
+
4
+import torch
5
+from torch._higher_order_ops.auto_functionalize import auto_functionalized
6
+
7
+from vllm.logger import init_logger
8
+
9
+from .vllm_inductor_pass import VllmInductorPass, is_func
10
+
11
+logger = init_logger(__name__)
12
+
13
+
14
+class FixFunctionalizationPass(VllmInductorPass):
15
+ """
16
+ This pass defunctionalizes certain nodes to avoid redundant tensor copies.
17
+ After this pass, DCE (dead-code elimination) should never be run,
18
+ as de-functionalized nodes may appear as dead code.
19
+
20
+ To add new nodes to defunctionalize, add to the if-elif chain in __call__.
21
+ """
22
+
23
+ def __call__(self, graph: torch.fx.Graph):
24
+ self.begin()
25
+ self.dump_graph(graph, "before_fix_functionalization")
26
+
27
+ self.nodes_to_remove: List[torch.fx.Node] = []
28
+ count = 0
29
+ for node in graph.nodes:
30
+ if not is_func(node, auto_functionalized):
31
+ continue # Avoid deep if-elif nesting
32
+
33
+ kwargs = node.kwargs
34
+ at_target = node.args[0]
35
+
36
+ if at_target == torch.ops._C.rotary_embedding.default:
37
+ query = kwargs['query']
38
+ mm_node = query.args[0].args[0]
39
+
40
+ # rotary_embedding is a special case: the two mutating inputs
41
+ # are query and key, which are slices of mm_node.
42
+ # While functionalized, results at[1] and at[2] are scattered
43
+ # back into mm_node. After de-functionalization, we can just
44
+ # use mm_node directly.
45
+ for idx, user in self.getitem_users(node).items():
46
+ for user_of_getitem in user.users:
47
+ if is_func(user_of_getitem,
48
+ torch.ops.aten.slice_scatter.default):
49
+ user_of_getitem.replace_all_uses_with(mm_node)
50
+ self._remove(user_of_getitem)
51
+ self._remove(user)
52
+
53
+ self.insert_defunctionalized(graph, node)
54
+ self._remove(node)
55
+
56
+ # These 2 replacements avoid the most copies for LLaMa.
57
+ elif at_target == torch.ops._C.fused_add_rms_norm.default:
58
+ mutated_args = {1: 'input', 2: 'residual'}
59
+ self.defunctionalize(graph, node, mutated_args)
60
+ elif at_target == torch.ops._C.fused_add_rms_norm_static_fp8_quant.default: # noqa: E501
61
+ mutated_args = {1: 'result', 2: 'residual'}
62
+ self.defunctionalize(graph, node, mutated_args)
63
+
64
+ elif at_target in [
65
+ torch.ops._C.rms_norm.default,
66
+ torch.ops._C.rms_norm_static_fp8_quant.default
67
+ ]:
68
+ mutated_args = {1: 'result'}
69
+ self.defunctionalize(graph, node, mutated_args)
70
+
71
+ elif at_target == torch.ops._C.silu_and_mul.default:
72
+ mutated_args = {1: 'out'}
73
+ # Because we have an 'out', need to specify args directly
74
+ self.defunctionalize(graph,
75
+ node,
76
+ mutated_args,
77
+ args=('out', 'input'))
78
+ else:
79
+ continue # skip the count
80
+
81
+ count += 1
82
+
83
+ self.dump_graph(graph, "before_fix_functionalization_cleanup")
84
+
85
+ # Remove the nodes all at once
86
+ count_removed = len(self.nodes_to_remove)
87
+ for node in self.nodes_to_remove:
88
+ graph.erase_node(node)
89
+
90
+ logger.debug("De-functionalized %s nodes, removed %s nodes", count,
91
+ count_removed)
92
+ self.dump_graph(graph, "after_fix_functionalization")
93
+ self.end_and_log()
94
+
95
+ def _remove(self, node_or_nodes: Union[torch.fx.Node,
96
+ Iterable[torch.fx.Node]]):
97
+ """
98
+ Stage a node (or nodes) for removal at the end of the pass.
99
+ """
100
+ if isinstance(node_or_nodes, torch.fx.Node):
101
+ self.nodes_to_remove.append(node_or_nodes)
102
+ else:
103
+ self.nodes_to_remove.extend(node_or_nodes)
104
+
105
+ def defunctionalize(self,
106
+ graph: torch.fx.Graph,
107
+ node: torch.fx.Node,
108
+ mutated_args: Dict[int, Union[torch.fx.Node, str]],
109
+ args: Optional[Tuple[Union[torch.fx.Node, str],
110
+ ...]] = None):
111
+ """
112
+ De-functionalize a node by replacing it with a call to the original.
113
+ It also replaces the getitem users with the mutated arguments.
114
+ See replace_users_with_mutated_args and insert_defunctionalized.
115
+ """
116
+ self.replace_users_with_mutated_args(node, mutated_args)
117
+ self.insert_defunctionalized(graph, node, args=args)
118
+ self._remove(node)
119
+
120
+ def replace_users_with_mutated_args(self, node: torch.fx.Node,
121
+ mutated_args: Dict[int,
122
+ Union[torch.fx.Node,
123
+ str]]):
124
+ """
125
+ Replace all getitem users of the auto-functionalized node with the
126
+ mutated arguments.
127
+ :param node: The auto-functionalized node
128
+ :param mutated_args: The mutated arguments, indexed by getitem index.
129
+ If the value of an arg is a string, `node.kwargs[arg]` is used.
130
+ """
131
+ for idx, user in self.getitem_users(node).items():
132
+ arg = mutated_args[idx]
133
+ arg = node.kwargs[arg] if isinstance(arg, str) else arg
134
+ user.replace_all_uses_with(arg)
135
+ self._remove(user)
136
+
137
+ def getitem_users(self, node: torch.fx.Node) -> Dict[int, torch.fx.Node]:
138
+ """
139
+ Returns the operator.getitem users of the auto-functionalized node,
140
+ indexed by the index they are getting.
141
+ """
142
+ users = {}
143
+ for user in node.users:
144
+ if is_func(user, operator.getitem):
145
+ idx = user.args[1]
146
+ users[idx] = user
147
+ return users
148
+
149
+ def insert_defunctionalized(self,
150
+ graph: torch.fx.Graph,
151
+ node: torch.fx.Node,
152
+ args: Optional[Tuple[Union[torch.fx.Node, str],
153
+ ...]] = None):
154
+ """
155
+ Insert a new defunctionalized node into the graph before node.
156
+ If one of the kwargs is 'out', provide args directly,
157
+ as node.kwargs cannot be used.
158
+ See https://github.com/pytorch/pytorch/blob/a00faf440888ffb724bad413f329a49e2b6388e7/torch/_inductor/lowering.py#L351
159
+
160
+ :param graph: Graph to insert the defunctionalized node into
161
+ :param node: The auto-functionalized node to defunctionalize
162
+ :param args: If we cannot use kwargs, specify args directly.
163
+ If an arg is a string, `node.kwargs[arg]` is used.
164
+ """ # noqa: E501
165
+ assert is_func(node, auto_functionalized), \
166
+ f"node must be auto-functionalized, is {node} instead"
167
+
168
+ # Create a new call to the original function
169
+ with graph.inserting_before(node):
170
+ function = node.args[0]
171
+ if args is None:
172
+ graph.call_function(function, kwargs=node.kwargs)
173
+ else:
174
+ # Args passed as strings refer to items in node.kwargs
175
+ args = tuple(node.kwargs[arg] if isinstance(arg, str) else arg
176
+ for arg in args)
177
+ graph.call_function(function, args=args)
vllm/compilation/fusion.py
+7
-6
@@ -6,10 +6,11 @@ from torch._higher_order_ops.auto_functionalize import auto_functionalized
6
from torch._inductor.pattern_matcher import (Match, PatternMatcherPass,
7
fwd_only, register_replacement)
8
9
-from vllm.compilation.inductor_pass import InductorPass
9
from vllm.config import CompilationConfig
10
from vllm.logger import init_logger
11
12
+from .vllm_inductor_pass import VllmInductorPass, is_func
13
+
14
logger = init_logger(__name__)
15
16
@@ -90,8 +91,6 @@ def empty_fp32(*args, **kwargs):
91
92
93
# Utilities for post-processing multi-output matches
93
-def is_func(node: torch.fx.Node, target) -> bool:
94
- return node.op == "call_function" and node.target == target
94
95
96
# Returns the first auto_functionalized node with the given op (if it exists)
@@ -127,7 +126,7 @@ def find_getitem(node: torch.fx.Node, idx: int) -> torch.fx.Node:
126
return ret
127
128
130
-class FusionPass(InductorPass):
129
+class FusionPass(VllmInductorPass):
130
"""
131
This pass fuses a pre-defined set of custom ops into fused ops.
132
It uses the torch pattern matcher to find the patterns and replace them.
@@ -142,7 +141,7 @@ class FusionPass(InductorPass):
141
_instance: 'Optional[FusionPass]' = None
142
143
@classmethod
145
- def instance(cls, config: CompilationConfig):
144
+ def instance(cls, config: CompilationConfig.PassConfig):
145
"""
146
Get the singleton instance of the FusionPass.
147
If the instance exists, the config is updated but
@@ -154,7 +153,7 @@ class FusionPass(InductorPass):
153
cls._instance.config = config
154
return cls._instance
155
157
- def __init__(self, config: CompilationConfig):
156
+ def __init__(self, config: CompilationConfig.PassConfig):
157
assert self.__class__._instance is None, \
158
"FusionPass singleton instance already exists"
159
super().__init__(config)
@@ -278,6 +277,7 @@ class FusionPass(InductorPass):
277
for node in match.nodes)
278
279
def __call__(self, graph: torch.fx.Graph):
280
+ self.begin()
281
self.dump_graph(graph, "before_fusion")
282
283
count = self.patterns.apply(graph)
@@ -289,3 +289,4 @@ class FusionPass(InductorPass):
289
logger.debug("Post-processed %s matches", len(self.matches))
290
self.dump_graph(graph, "after_fusion")
291
self.matches.clear()
292
+ self.end_and_log()
vllm/compilation/inductor_pass.py
+73
-27
@@ -1,38 +1,84 @@
1
+import hashlib
2
+import inspect
3
+import types
4
from abc import ABC, abstractmethod
5
+from typing import Any, Callable, Optional, Union
6
7
import torch
4
-
5
-from vllm.config import CompilationConfig
6
-# yapf: disable
7
-from vllm.distributed import get_tensor_model_parallel_rank as get_tp_rank
8
-from vllm.distributed import (
9
- get_tensor_model_parallel_world_size as get_tp_world_size)
10
-from vllm.distributed import model_parallel_is_initialized as p_is_init
11
-# yapf: enable
12
-from vllm.logger import init_logger
13
-
14
-logger = init_logger(__name__)
8
+from torch import fx
9
10
11
class InductorPass(ABC):
12
+ """
13
+ General custom inductor pass interface.
14
+ TODO(torch==2.6) use torch._inductor.custom_graph_pass.CustomGraphPass
15
+ """
16
17
@abstractmethod
18
def __call__(self, graph: torch.fx.Graph):
19
+ """
20
+ Execute the pass on the given graph.
21
+ """
22
raise NotImplementedError
23
23
- def __init__(self, config: CompilationConfig):
24
- self.config = config
25
-
26
- def dump_graph(self, graph: torch.fx.Graph, stage: str):
27
- if stage in self.config.dump_graph_stages:
28
- # Make sure filename includes rank in the distributed setting
29
- parallel = p_is_init() and get_tp_world_size() > 1
30
- rank = f"-{get_tp_rank()}" if parallel else ""
31
- filepath = self.config.dump_graph_dir / f"{stage}{rank}.py"
32
-
33
- logger.info("Printing graph to %s", filepath)
34
- with open(filepath, "w") as f:
35
- src = graph.python_code(root_module="self", verbose=True).src
36
- # Add imports so it's not full of errors
37
- print("import torch; from torch import device", file=f)
38
- print(src, file=f)
24
+ def uuid(self) -> Any:
25
+ """
26
+ Provide a unique identifier for the pass, used in Inductor code cache.
27
+ This should depend on the pass implementation, so that changes to the
28
+ pass result in recompilation.
29
+ By default, the object source is hashed.
30
+ """
31
+ return InductorPass.hash_source(self)
32
+
33
+ @staticmethod
34
+ def hash_source(*srcs: Union[str, Any]):
35
+ """
36
+ Utility method to hash the sources of functions or objects.
37
+ :param srcs: strings or objects to add to the hash.
38
+ Objects and functions have their source inspected.
39
+ :return:
40
+ """
41
+ hasher = hashlib.sha256()
42
+ for src in srcs:
43
+ if isinstance(src, str):
44
+ src_str = src
45
+ elif isinstance(src, types.FunctionType):
46
+ src_str = inspect.getsource(src)
47
+ else:
48
+ src_str = inspect.getsource(src.__class__)
49
+ hasher.update(src_str.encode("utf-8"))
50
+ return hasher.digest()
51
+
52
+
53
+class CallableInductorPass(InductorPass):
54
+ """
55
+ This class is a wrapper for a callable that automatically provides an
56
+ implementation of the UUID.
57
+ """
58
+
59
+ def __init__(self,
60
+ callable: Callable[[fx.Graph], None],
61
+ uuid: Optional[Any] = None):
62
+ self.callable = callable
63
+ if uuid is None:
64
+ uuid = InductorPass.hash_source(callable)
65
+ self._uuid = uuid
66
+
67
+ def __call__(self, graph: torch.fx.Graph):
68
+ self.callable(graph)
69
+
70
+ def uuid(self) -> Any:
71
+ return self._uuid
72
+
73
+ def __getstate__(self):
74
+ """
75
+ Pickling occurs in the Inductor code cache if a pass is not given to
76
+ the pass manager but is instead directly added to config as a pass.
77
+ See PostGradPassManager for more.
78
+
79
+ TODO(torch==2.6), use the `uuid` method in CustomGraphPass instead.
80
+ """
81
+ return self._uuid
82
+
83
+ def __setstate__(self, state):
84
+ raise ValueError("Cannot unpickle CallableInductorPass")
vllm/compilation/pass_manager.py
new
+77
@@ -0,0 +1,77 @@
1
+from typing import List
2
+
3
+from torch import fx as fx
4
+
5
+from vllm.config import CompilationConfig
6
+from vllm.logger import init_logger
7
+
8
+from .fix_functionalization import FixFunctionalizationPass
9
+from .fusion import FusionPass
10
+from .inductor_pass import InductorPass
11
+from .reshapes import RedundantReshapesPass
12
+
13
+logger = init_logger(__name__)
14
+
15
+
16
+class PostGradPassManager:
17
+ """
18
+ The pass manager for post-grad passes.
19
+ It handles configuration, adding custom passes, and running passes.
20
+ It also supports pickling, which is used by the Inductor code cache.
21
+ TODO(torch==2.6), use CustomGraphPass
22
+ (torch._inductor.custom_graph_pass.CustomGraphPass)
23
+
24
+ The order of the post-grad post-passes is:
25
+ 1. passes (constructor parameter)
26
+ 2. default passes (RedundantReshapesPass, FusionPass)
27
+ 3. config["post_grad_custom_post_pass"] (if it exists)
28
+ 4. fix_functionalization
29
+ This way, all passes operate on a functionalized graph.
30
+ """
31
+
32
+ def __init__(self):
33
+ self.passes: List[InductorPass] = []
34
+
35
+ def __call__(self, graph: fx.Graph):
36
+ for pass_ in self.passes:
37
+ pass_(graph)
38
+
39
+ # always run fix_functionalization last
40
+ self.fix_functionalization(graph)
41
+
42
+ def configure(self, pass_config: CompilationConfig.PassConfig):
43
+ self.pass_config = pass_config
44
+ if pass_config.enable_reshape:
45
+ self.passes += [RedundantReshapesPass(pass_config)]
46
+
47
+ if pass_config.enable_fusion:
48
+ self.passes += [FusionPass.instance(pass_config)]
49
+
50
+ self.fix_functionalization = FixFunctionalizationPass(pass_config)
51
+
52
+ def add(self, pass_: InductorPass):
53
+ assert isinstance(pass_, InductorPass)
54
+ self.passes.append(pass_)
55
+
56
+ def __getstate__(self):
57
+ """
58
+ Custom pickling for the pass manager, as some passes cannot be pickled.
59
+ Pickling occurs because the pass manager is set as the value of
60
+ `config["post_grad_custom_post_pass"]` in the Inductor config.
61
+ The config is pickled to act as a key in the Inductor code cache.
62
+ Any other passes in the config are pickled as well.
63
+
64
+ TODO(torch==2.6), use the `uuid` method in CustomGraphPass instead.
65
+ """
66
+ state = {"pass_config": self.pass_config.uuid(), "passes": []}
67
+ for pass_ in self.passes:
68
+ state["passes"].append(pass_.uuid())
69
+ state["passes"].append(self.fix_functionalization.uuid())
70
+ return state
71
+
72
+ def __setstate__(self, state):
73
+ """
74
+ Do not allow unpickling of the pass manager.
75
+ If this is needed in the future, it should properly pickle the passes.
76
+ """
77
+ raise ValueError("Cannot unpickle PostGradPassManager")
vllm/compilation/reshapes.py
+5
-3
@@ -3,14 +3,14 @@ from typing import Union
3
import torch.fx
4
from torch import SymInt
5
6
-from vllm.compilation.fusion import is_func
7
-from vllm.compilation.inductor_pass import InductorPass
6
from vllm.logger import init_logger
7
8
+from .vllm_inductor_pass import VllmInductorPass, is_func
9
+
10
logger = init_logger(__name__)
11
12
13
-class RedundantReshapesPass(InductorPass):
13
+class RedundantReshapesPass(VllmInductorPass):
14
"""
15
This is an inductor pass that removes redundant reshape operations.
16
It is required for RMSNorm-quant fusion to work properly.
@@ -31,6 +31,7 @@ class RedundantReshapesPass(InductorPass):
31
"""
32
33
def __call__(self, graph: torch.fx.Graph):
34
+ self.begin()
35
self.dump_graph(graph, "before_reshapes")
36
count = 0
37
# Remove no-op reshapes/views:
@@ -56,6 +57,7 @@ class RedundantReshapesPass(InductorPass):
57
logger.debug("Removed %s no-op reshapes", count)
58
59
self.dump_graph(graph, "after_reshapes")
60
+ self.end_and_log()
61
62
def dims_equivalent(self, dim: Union[int, torch.fx.Node],
63
i_dim: Union[int, SymInt]) -> bool:
vllm/compilation/vllm_inductor_pass.py
new
+53
@@ -0,0 +1,53 @@
1
+import time
2
+
3
+import torch
4
+
5
+from vllm.config import CompilationConfig
6
+# yapf: disable
7
+from vllm.distributed import get_tensor_model_parallel_rank as get_tp_rank
8
+from vllm.distributed import (
9
+ get_tensor_model_parallel_world_size as get_tp_world_size)
10
+from vllm.distributed import model_parallel_is_initialized as p_is_init
11
+# yapf: enable
12
+from vllm.logger import init_logger
13
+
14
+from .inductor_pass import InductorPass
15
+
16
+logger = init_logger(__name__)
17
+
18
+
19
+def is_func(node: torch.fx.Node, target) -> bool:
20
+ return node.op == "call_function" and node.target == target
21
+
22
+
23
+class VllmInductorPass(InductorPass):
24
+ """
25
+ An inductor pass with access to vLLM PassConfig.
26
+ It provides timing, logging, and dumping utilities.
27
+ """
28
+
29
+ def __init__(self, config: CompilationConfig.PassConfig):
30
+ self.config = config
31
+ self.pass_name = self.__class__.__name__
32
+
33
+ def dump_graph(self, graph: torch.fx.Graph, stage: str):
34
+ if stage in self.config.dump_graph_stages:
35
+ # Make sure filename includes rank in the distributed setting
36
+ parallel = p_is_init() and get_tp_world_size() > 1
37
+ rank = f"-{get_tp_rank()}" if parallel else ""
38
+ filepath = self.config.dump_graph_dir / f"{stage}{rank}.py"
39
+
40
+ logger.info("%s printing graph to %s", self.pass_name, filepath)
41
+ with open(filepath, "w") as f:
42
+ src = graph.python_code(root_module="self", verbose=True).src
43
+ # Add imports so it's not full of errors
44
+ print("import torch; from torch import device", file=f)
45
+ print(src, file=f)
46
+
47
+ def begin(self):
48
+ self._start_time = time.perf_counter_ns()
49
+
50
+ def end_and_log(self):
51
+ self._end_time = time.perf_counter_ns()
52
+ duration_ms = float(self._end_time - self._start_time) / 1.0e6
53
+ logger.debug("%s completed in %.1f ms", self.pass_name, duration_ms)
vllm/config.py
+47
-13
@@ -1,5 +1,6 @@
1
import copy
2
import enum
3
+import hashlib
4
import json
5
import warnings
6
from dataclasses import dataclass, field, replace
@@ -13,6 +14,7 @@ from pydantic import BaseModel, Field, PrivateAttr
14
from transformers import PretrainedConfig
15
16
import vllm.envs as envs
17
+from vllm.compilation.inductor_pass import CallableInductorPass, InductorPass
18
from vllm.logger import init_logger
19
from vllm.model_executor.layers.quantization import (QUANTIZATION_METHODS,
20
get_quantization_config)
@@ -2120,12 +2122,7 @@ class CompilationConfig(BaseModel):
2122
name because the config uses json format. If we pass the config
2123
from Python, functions can also be passed directly via Python object
2124
constructor, e.g. `CompilationConfig(inductor_passes={"a": func})`
2123
- - custom inductor passes:
2124
- - dump_graph_stages: list of stages for which we want to dump the graph.
2125
- Each pass defines its own stages (before, after, maybe in-between).
2126
- - dump_graph_dir: directory to dump the graph. Default is .
2127
- - enable_fusion: whether to enable the custom fusion pass.
2128
- TODO better pass enabling system.
2125
+ - custom inductor passes: see PassConfig for more details
2126
2127
Why we have different sizes for cudagraph and inductor:
2128
- cudagraph: a cudagraph captured for a specific size can only be used
@@ -2157,9 +2154,43 @@ class CompilationConfig(BaseModel):
2154
cudagraph_capture_sizes: Optional[List[int]] = None
2155
cudagraph_copy_inputs: bool = False
2156
2160
- dump_graph_stages: List[str] = Field(default_factory=list)
2161
- dump_graph_dir: Path = Field(default=Path("."))
2162
- enable_fusion: bool = True
2157
+ class PassConfig(BaseModel):
2158
+ """
2159
+ Configuration for custom Inductor passes.
2160
+ This is separate from general CompilationConfig so that inductor passes
2161
+ don't all have access to full configuration - that would create a cycle
2162
+ as the PassManager is set as a property of config.
2163
+ - dump_graph_stages: list of stages for which we want to dump the graph.
2164
+ Each pass defines its own stages (before, after, maybe in-between).
2165
+ - dump_graph_dir: directory to dump the graphs. Default is .
2166
+ - enable_fusion: whether to enable the custom fusion pass.
2167
+ - enable_reshape: whether to enable the custom reshape elimination pass.
2168
+ TODO better pass enabling system.
2169
+ """
2170
+ dump_graph_stages: List[str] = Field(default_factory=list)
2171
+ dump_graph_dir: Path = Field(default=Path("."))
2172
+ enable_fusion: bool = True
2173
+ enable_reshape: bool = True
2174
+
2175
+ def uuid(self):
2176
+ """
2177
+ Produces a hash unique to the pass configuration.
2178
+ Any new fields that affect compilation should be added to the hash.
2179
+ Do not include dump_graph_* in the hash - they don't affect
2180
+ compilation.
2181
+ """
2182
+ dict_ = self.model_dump(
2183
+ include={"enable_fusion", "enable_reshape"})
2184
+ encoded = json.dumps(dict_, sort_keys=True).encode("utf-8")
2185
+ return hashlib.sha256(encoded).digest()
2186
+
2187
+ def model_post_init(self, __context: Any) -> None:
2188
+ if not self.enable_reshape and self.enable_fusion:
2189
+ print_warning_once(
2190
+ "Fusion enabled but reshape elimination disabled."
2191
+ "RMSNorm + quant (fp8) fusion might not work")
2192
+
2193
+ pass_config: PassConfig = Field(default_factory=PassConfig)
2194
2195
# not configurable, computed after init
2196
compile_sizes: List[int] = PrivateAttr
@@ -2185,8 +2216,9 @@ class CompilationConfig(BaseModel):
2216
for k, v in self.inductor_passes.items():
2217
if not isinstance(v, str):
2218
assert callable(v), (
2188
- f"pass {k} should be a function or a qualified name")
2189
- self.inductor_compile_config[k] = v
2219
+ f"pass {k} should be callable or a qualified name")
2220
+ self.inductor_compile_config[k] = v if isinstance(
2221
+ v, InductorPass) else CallableInductorPass(v)
2222
continue
2223
2224
# resolve function from qualified name
@@ -2194,7 +2226,8 @@ class CompilationConfig(BaseModel):
2226
module = ".".join(names[:-1])
2227
func_name = names[-1]
2228
func = __import__(module).__dict__[func_name]
2197
- self.inductor_compile_config[k] = func
2229
+ self.inductor_compile_config[k] = func if isinstance(
2230
+ func, InductorPass) else CallableInductorPass(func)
2231
2232
self.enabled_custom_ops = Counter()
2233
self.disabled_custom_ops = Counter()
@@ -2344,7 +2377,8 @@ class VllmConfig:
2377
self.compilation_config.custom_ops = ["none"]
2378
self.compilation_config.use_cudagraph = True
2379
self.compilation_config.use_inductor = True
2347
- self.compilation_config.enable_fusion = False
2380
+ self.compilation_config.pass_config.enable_fusion = False
2381
+ self.compilation_config.pass_config.enable_reshape = False
2382
2383
current_platform.check_and_update_config(self)
2384
vllm/utils.py
-9
@@ -1501,15 +1501,6 @@ class LazyDict(Mapping, Generic[T]):
1501
return len(self._factory)
1502
1503
1504
-def combine_fx_passes(passes: List[Callable]) -> Callable:
1505
-
1506
- def combined_fx(graph) -> None:
1507
- for fx in passes:
1508
- fx(graph)
1509
-
1510
- return combined_fx
1511
-
1512
-
1504
def weak_ref_tensor(tensor: torch.Tensor) -> torch.Tensor:
1505
"""
1506
Create a weak reference to a tensor.
vllm/v1/worker/gpu_model_runner.py
+1
-1
@@ -548,7 +548,7 @@ class GPUModelRunner:
548
if not self.use_cuda_graph:
549
logger.warning(
550
"Skipping CUDA graph capture. Please add "
551
- "-O 3 to use CUDA graphs.", CompilationLevel.PIECEWISE)
551
+ "-O %s to use CUDA graphs.", CompilationLevel.PIECEWISE)
552
return
553
554
start_time = time.perf_counter()