@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8b0fe06c89
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[torch.compile] Inductor code caching fix (#10273)

Signed-off-by: luka <luka@neuralmagic.com> Signed-off-by: Luka Govedic <luka.govedic@gmail.com>

Luka Govedič committed Nov 21, 2024 at 00:44 UTC 8b0fe06c890a202eba24d517cc77562e4a8b0d0c
14 files changed +604 -288
tests/compile/backend.py
+10 -6
@@ -1,7 +1,9 @@
1 from copy import deepcopy
2 -from typing import Callable
2 +from typing import Callable, Union
3
4 -import torch
4 +from torch import fx
5 +
6 +from vllm.compilation.inductor_pass import InductorPass
7
8
9 class TestBackend:
@@ -11,19 +13,21 @@ class TestBackend:
13 It also saves the graph before and after the custom passes for inspection.
14 """
15
14 - def __init__(self, *args: Callable[[torch.fx.Graph], None]):
15 - self.custom_passes = args
16 + def __init__(self, *passes: Union[InductorPass, Callable[[fx.Graph],
17 + None]]):
18 + self.custom_passes = list(passes)
19 from torch._inductor import config
20 self.current_config = config.shallow_copy_dict()
21 + self.current_config['force_disable_caches'] = True
22 self.current_config['post_grad_custom_post_pass'] = self.post_pass
23
20 - def __call__(self, graph: torch.fx.GraphModule, example_inputs):
24 + def __call__(self, graph: fx.GraphModule, example_inputs):
25 from torch._inductor.compile_fx import compile_fx
26 return compile_fx(graph,
27 example_inputs,
28 config_patches=self.current_config)
29
26 - def post_pass(self, graph: torch.fx.Graph):
30 + def post_pass(self, graph: fx.Graph):
31 self.graph_pre_pass = deepcopy(graph)
32 for pass_ in self.custom_passes:
33 pass_(graph)
tests/compile/test_functionalization.py new
+95
@@ -0,0 +1,95 @@
1 +import pytest
2 +import torch
3 +
4 +import vllm.envs as envs
5 +from vllm import LLM, SamplingParams
6 +from vllm.compilation.fix_functionalization import FixFunctionalizationPass
7 +from vllm.compilation.fusion import (FusionPass, find_auto_fn,
8 + find_auto_fn_maybe)
9 +from vllm.compilation.reshapes import RedundantReshapesPass
10 +from vllm.compilation.vllm_inductor_pass import is_func
11 +from vllm.config import CompilationConfig
12 +
13 +from .backend import TestBackend
14 +
15 +OPS_IN_MODEL = [
16 + torch.ops._C.rotary_embedding.default,
17 + torch.ops._C.fused_add_rms_norm.default,
18 + torch.ops._C.silu_and_mul.default,
19 +]
20 +
21 +RMS_OP = torch.ops._C.rms_norm.default
22 +
23 +RMS_QUANT_OPS = {
24 + "static_fp8": [
25 + torch.ops._C.rms_norm_static_fp8_quant.default,
26 + torch.ops._C.fused_add_rms_norm_static_fp8_quant.default
27 + ],
28 +}
29 +
30 +prompts = [
31 + "Hello, my name is",
32 + "The president of the United States is",
33 + "The capital of France is",
34 + "The future of AI is",
35 +]
36 +
37 +
38 +@pytest.mark.parametrize("model",
39 + ["nm-testing/TinyLlama-1.1B-Chat-v1.0-FP8-e2e"])
40 +@pytest.mark.parametrize("do_fusion", [True, False])
41 +@pytest.mark.skipif(envs.VLLM_TARGET_DEVICE != "cuda",
42 + reason="Only test on CUDA")
43 +def test_fix_functionalization(model: str, do_fusion: bool):
44 + torch.set_default_device("cuda")
45 +
46 + config = CompilationConfig.PassConfig(enable_fusion=do_fusion,
47 + enable_reshape=True)
48 + reshape_pass = RedundantReshapesPass(config)
49 + fusion_pass = FusionPass.instance(config)
50 +
51 + passes = [reshape_pass, fusion_pass] if do_fusion else [reshape_pass]
52 + func_pass = FixFunctionalizationPass(config)
53 + backend_func = TestBackend(*passes, func_pass)
54 + backend_no_func = TestBackend(*passes)
55 +
56 + # instantiate a full engine and manually compile the model 2x
57 + # (with and without FixFunctionalizationPass)
58 + llm = LLM(model=model, enforce_eager=True)
59 + model_runner = llm.llm_engine.model_executor.driver_worker.model_runner
60 + orig_model = model_runner.model
61 + # TODO mark inputs dynamic? (currently torch.compile is triggered 4x)
62 + # Can only do that by using the decorator but then we'd have to instantiate
63 + # 2 LLM instances.
64 +
65 + sampling_params = SamplingParams(temperature=0.0, top_p=1.0)
66 + model_runner.model = torch.compile(orig_model,
67 + fullgraph=True,
68 + backend=backend_func)
69 + gen_func = llm.generate(prompts, sampling_params)
70 +
71 + model_runner.model = torch.compile(orig_model,
72 + fullgraph=True,
73 + backend=backend_no_func)
74 + gen_no_func = llm.generate(prompts, sampling_params)
75 +
76 + for output_func, output_no_func in zip(gen_func, gen_no_func):
77 + assert output_func.outputs[0].text == output_no_func.outputs[0].text
78 +
79 + # OPS_IN_MODEL always appear. RMS_OP is fused away if we run fusion,
80 + # and replaced by fused quantized ops in RMS_QUANT_OPS.
81 + ops = OPS_IN_MODEL + (RMS_QUANT_OPS["static_fp8"]
82 + if do_fusion else [RMS_OP])
83 +
84 + for op in ops:
85 + find_auto_fn(backend_no_func.graph_post_pass.nodes, op)
86 + assert find_auto_fn_maybe(backend_func.graph_post_pass.nodes,
87 + op) is None # noqa: E501
88 +
89 + # make sure the ops were all de-functionalized
90 + found = dict()
91 + for node in backend_func.graph_post_pass.nodes:
92 + for op in ops:
93 + if is_func(node, op):
94 + found[op] = True
95 + assert all(found[op] for op in ops)
tests/compile/test_fusion.py
+5 -6
@@ -38,12 +38,6 @@ class TestModel(torch.nn.Module):
38 return y3
39
40
41 -# Init does pattern registration, which can only happen once
42 -config = CompilationConfig(enable_fusion=True)
43 -reshape_pass = RedundantReshapesPass(config)
44 -fusion_pass = FusionPass.instance(config)
45 -
46 -
41 @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
42 @pytest.mark.parametrize("hidden_size", [64, 3392, 4096])
43 @pytest.mark.parametrize("num_tokens", [7, 256, 533, 2048, 2049])
@@ -58,6 +52,11 @@ def test_fusion_rmsnorm_quant(dtype, hidden_size, num_tokens, eps):
52 pytest.skip("Only test eps=1e-5 for now")
53
54 # Reshape pass is needed for the fusion pass to work
55 + config = CompilationConfig.PassConfig(enable_fusion=True,
56 + enable_reshape=True)
57 + reshape_pass = RedundantReshapesPass(config)
58 + fusion_pass = FusionPass.instance(config)
59 +
60 backend = TestBackend(reshape_pass, fusion_pass)
61 model = TestModel(hidden_size, eps)
62
tests/compile/test_pass_manager.py new
+35
@@ -0,0 +1,35 @@
1 +import pickle
2 +
3 +import pytest
4 +import torch
5 +from torch._inductor.codecache import BypassFxGraphCache
6 +
7 +from vllm.compilation.config import CompilationConfig
8 +from vllm.compilation.inductor_pass import (CallableInductorPass,
9 + as_inductor_pass)
10 +from vllm.compilation.pass_manager import PostGradPassManager
11 +
12 +
13 +def simple_callable(graph: torch.fx.Graph):
14 + pass
15 +
16 +
17 +@as_inductor_pass(files=(__file__, ))
18 +def callable_decorated(graph: torch.fx.Graph):
19 + pass
20 +
21 +
22 +@pytest.mark.parametrize(
23 + "works, callable",
24 + [(False, simple_callable), (True, callable_decorated),
25 + (True, CallableInductorPass(simple_callable, "simple_callable"))])
26 +def test_pass_manager(works: bool, callable):
27 + config = CompilationConfig().pass_config
28 + pass_manager = PostGradPassManager([callable])
29 + pass_manager.configure(config) # Adds default passes
30 +
31 + if works:
32 + pickle.dumps(pass_manager)
33 + else:
34 + with pytest.raises(BypassFxGraphCache):
35 + pickle.dumps(pass_manager)
vllm/compilation/backends.py
+19 -217
@@ -1,6 +1,5 @@
1 import copy
2 import dataclasses
3 -import operator
3 from contextlib import ExitStack
4 from typing import Any, Callable, Dict, List, Optional, Sequence, Set, Tuple
5 from unittest.mock import patch
@@ -11,205 +10,15 @@ import torch.fx as fx
10 import vllm.envs as envs
11 from vllm.config import CompilationConfig
12 from vllm.logger import init_logger
14 -from vllm.utils import combine_fx_passes, weak_ref_tensors
13 +from vllm.utils import weak_ref_tensors
14
15 from .counter import compilation_counter
17 -from .fusion import FusionPass
18 -from .reshapes import RedundantReshapesPass
16 +from .inductor_pass import InductorPass
17 +from .pass_manager import PostGradPassManager
18
19 logger = init_logger(__name__)
20
21
23 -def fix_functionalization(graph: fx.Graph):
24 - """
25 - Rewrite the graph module to replace the pattern involving
26 - torch._higher_order_ops.auto_functionalize.auto_functionalized
27 - with a direct call to the inplace custom op.
28 -
29 - # TODO: check if PyTorch nightly has fixed this issue
30 - """
31 -
32 - # debug code, if we want to see the graph before the transformation
33 - # with open("before.py", "w") as f:
34 - # print(graph.python_code(root_module="self", verbose=True).src, file=f)
35 -
36 - nodes_to_remove = []
37 -
38 - for node in graph.nodes:
39 - # Identify the auto_functionalized node
40 - if node.op == 'call_function' and node.target == torch._higher_order_ops.auto_functionalize.auto_functionalized: # noqa
41 - if node.args[0] == torch.ops._C.rotary_embedding.default:
42 - # manual replace for rotary_embedding
43 -
44 - # Now, collect the arguments
45 - kwargs = node.kwargs
46 -
47 - query = kwargs['query']
48 - mm_node = query.args[0].args[0]
49 -
50 - # Create a new call to torch.ops._C.rotary_embedding.default
51 - with graph.inserting_before(node):
52 - # just insert the call to the custom op
53 - # NOTE: don't run dead code elimination,
54 - # otherwise this op will be removed
55 - graph.call_function(torch.ops._C.rotary_embedding.default,
56 - kwargs=kwargs)
57 -
58 - # Remove the auto_functionalized node
59 - # Since the node may have outputs, we need to handle its users
60 - # Replace uses of the outputs (getitem nodes) with mm_node
61 - for user in list(node.users):
62 - if user.op == 'call_function' and user.target == operator.getitem: # noqa
63 - # Remove the getitem node
64 - for getitem_user in list(user.users):
65 - if (getitem_user.op == 'call_function'
66 - and getitem_user.target
67 - == torch.ops.aten.slice_scatter.default):
68 - # Replace the uses of slice_scatter node
69 - # with mm_node
70 - getitem_user.replace_all_uses_with(mm_node)
71 - nodes_to_remove.append(getitem_user)
72 - nodes_to_remove.append(user)
73 - nodes_to_remove.append(node)
74 -
75 - elif node.args[0] == torch.ops._C.fused_add_rms_norm.default:
76 - # manual replace for fused_add_rms_norm
77 - # this is the most effective optimization for llama
78 - # failing to do this will result in many unnecessary copies
79 -
80 - kwargs = node.kwargs
81 -
82 - input = kwargs['input']
83 - residual = kwargs['residual']
84 -
85 - # Create a new call to torch.ops._C.rotary_embedding.default
86 - with graph.inserting_before(node):
87 - # just insert the call to the custom op
88 - # NOTE: don't run dead code elimination,
89 - # otherwise this op will be removed
90 - graph.call_function(
91 - torch.ops._C.fused_add_rms_norm.default, kwargs=kwargs)
92 -
93 - for user in list(node.users):
94 - if user.op == 'call_function' and user.target == operator.getitem: # noqa
95 - # Remove the getitem node
96 - if user.args[1] == 1:
97 - replace_node = input
98 - elif user.args[1] == 2:
99 - replace_node = residual
100 - user.replace_all_uses_with(replace_node)
101 - nodes_to_remove.append(user)
102 - nodes_to_remove.append(node)
103 - elif (node.args[0] ==
104 - torch.ops._C.fused_add_rms_norm_static_fp8_quant.default):
105 - # manual replace for fused_add_rms_norm_static_fp8_quant
106 - # this is the most effective optimization for llama
107 - # failing to do this will result in many unnecessary copies
108 -
109 - kwargs = node.kwargs
110 -
111 - result = kwargs['result']
112 - residual = kwargs['residual']
113 -
114 - # Create a new call to
115 - # torch.ops._C.fused_add_rms_norm_static_fp8_quant.default
116 - with graph.inserting_before(node):
117 - # just insert the call to the custom op
118 - # NOTE: don't run dead code elimination,
119 - # otherwise this op will be removed
120 - graph.call_function(
121 - torch.ops._C.fused_add_rms_norm_static_fp8_quant.
122 - default,
123 - kwargs=kwargs)
124 -
125 - for user in list(node.users):
126 - if user.op == 'call_function' and user.target == operator.getitem: # noqa
127 - # Remove the getitem node
128 - if user.args[1] == 1:
129 - replace_node = result
130 - elif user.args[1] == 2:
131 - replace_node = residual
132 - user.replace_all_uses_with(replace_node)
133 - nodes_to_remove.append(user)
134 - nodes_to_remove.append(node)
135 -
136 - elif node.args[0] == torch.ops._C.rms_norm.default:
137 - # manual replace for rms_norm
138 -
139 - kwargs = node.kwargs
140 -
141 - replace_node = kwargs['result']
142 - # Create a new call to torch.ops._C.rms_norm.default
143 - with graph.inserting_before(node):
144 - # just insert the call to the custom op
145 - # NOTE: don't run dead code elimination,
146 - # otherwise this op will be removed
147 - graph.call_function(torch.ops._C.rms_norm.default,
148 - kwargs=kwargs)
149 -
150 - for user in list(node.users):
151 - if user.op == 'call_function' and user.target == operator.getitem: # noqa
152 - user.replace_all_uses_with(replace_node)
153 - nodes_to_remove.append(user)
154 - nodes_to_remove.append(node)
155 -
156 - elif node.args[
157 - 0] == torch.ops._C.rms_norm_static_fp8_quant.default: # noqa
158 - # manual replace for rms_norm_static_fp8_quant
159 -
160 - kwargs = node.kwargs
161 -
162 - replace_node = kwargs['result']
163 - # Create a new call to torch.ops._C.rms_norm_static_fp8_quant.default # noqa
164 - with graph.inserting_before(node):
165 - # just insert the call to the custom op
166 - # NOTE: don't run dead code elimination,
167 - # otherwise this op will be removed
168 - graph.call_function(
169 - torch.ops._C.rms_norm_static_fp8_quant.default,
170 - kwargs=kwargs)
171 -
172 - for user in list(node.users):
173 - if user.op == 'call_function' and user.target == operator.getitem: # noqa
174 - user.replace_all_uses_with(replace_node)
175 - nodes_to_remove.append(user)
176 - nodes_to_remove.append(node)
177 -
178 - elif node.args[0] == torch.ops._C.silu_and_mul.default:
179 - # manual replace for silu_and_mul
180 -
181 - kwargs = node.kwargs
182 -
183 - input = kwargs['input']
184 - out = kwargs['out']
185 -
186 - # Create a new call to torch.ops._C.silu_and_mul.default
187 - # cannot use kwargs, because we have an `out`, see https://github.com/pytorch/pytorch/blob/a00faf440888ffb724bad413f329a49e2b6388e7/torch/_inductor/lowering.py#L351 # noqa
188 - with graph.inserting_before(node):
189 - # just insert the call to the custom op
190 - # NOTE: don't run dead code elimination,
191 - # otherwise this op will be removed
192 - graph.call_function(
193 - torch.ops._C.silu_and_mul.default,
194 - args=(out, input),
195 - )
196 - replace_node = out
197 -
198 - for user in list(node.users):
199 - if user.op == 'call_function' and user.target == operator.getitem: # noqa
200 - user.replace_all_uses_with(replace_node)
201 - nodes_to_remove.append(user)
202 - nodes_to_remove.append(node)
203 -
204 - # Remove the nodes all at once
205 - for node in nodes_to_remove:
206 - graph.erase_node(node)
207 -
208 - # debug code, if we want to see the graph after the transformation
209 - # with open("after.py", "w") as f:
210 - # print(graph.python_code(root_module="self", verbose=True).src, file=f)
211 -
212 -
22 def wrap_inductor(graph,
23 example_inputs,
24 additional_inductor_config,
@@ -368,12 +177,8 @@ class VllmBackend:
177 The major work of this backend is to split the graph into
178 piecewise graphs, and pass them to the piecewise backend.
179
371 - This backend also handles custom passes and adds them to Inductor config.
372 - The order of the post-grad post-passes is:
373 - 1. post_grad_passes (constructor parameter)
374 - 2. config["post_grad_custom_post_pass"]
375 - 3. fix_functionalization
376 - This way, all passes operate on a functionalized graph.
180 + This backend also adds the PostGradPassManager to Inductor config,
181 + which handles the post-grad passes.
182 """
183
184 compilation_configs: CompilationConfig
@@ -402,7 +207,9 @@ class VllmBackend:
207 # streams, it might not be safe to share a global pool.
208 # only investigate this when we use multiple streams
209 self.graph_pool = global_graph_pool
405 - self.post_grad_passes = []
210 +
211 + # Passes to run on the graph post-grad.
212 + self.post_grad_pass_manager = PostGradPassManager()
213
214 self.sym_tensor_indices = []
215 self.input_buffers = []
@@ -412,24 +219,19 @@ class VllmBackend:
219 # `torch.compile` is JIT compiled, so we don't need to
220 # do anything here
221
415 - def add_passes_to_config(self):
222 + def configure_post_pass(self):
223 config = self.compilation_configs
417 - passes = list(self.post_grad_passes)
418 -
419 - passes = passes + [RedundantReshapesPass(config)]
420 -
421 - if config.enable_fusion:
422 - passes = passes + [FusionPass.instance(config)]
224 + self.post_grad_pass_manager.configure(config.pass_config)
225
226 + # Post-grad custom passes are run using the post_grad_custom_post_pass
227 + # hook. If a pass for that hook exists, add it to the pass manager.
228 inductor_config = config.inductor_compile_config
425 - if "post_grad_custom_post_pass" in inductor_config:
426 - passes = passes + [inductor_config["post_grad_custom_post_pass"]]
427 -
428 - # add the fix_functionalization pass last, so that all other
429 - # passes operate on a functionalized graph
430 - passes = passes + [fix_functionalization]
431 - combined_pass = combine_fx_passes(passes)
432 - inductor_config["post_grad_custom_post_pass"] = combined_pass
229 + PASS_KEY = "post_grad_custom_post_pass"
230 + if PASS_KEY in inductor_config:
231 + # Config should automatically wrap all inductor passes
232 + assert isinstance(inductor_config[PASS_KEY], InductorPass)
233 + self.post_grad_pass_manager.add(inductor_config[PASS_KEY])
234 + inductor_config[PASS_KEY] = self.post_grad_pass_manager
235
236 def __call__(self, graph: fx.GraphModule, example_inputs) -> Callable:
237
@@ -444,7 +246,7 @@ class VllmBackend:
246 # we get the sizes to capture for cudagraph
247 # from compilation context
248 self.compilation_configs.init_during_runtime()
447 - self.add_passes_to_config()
249 + self.configure_post_pass()
250
251 self.split_gm, self.piecewise_graphs = split_graph(
252 graph, self.compilation_configs.splitting_ops)
vllm/compilation/fix_functionalization.py new
+177
@@ -0,0 +1,177 @@
1 +import operator
2 +from typing import Dict, Iterable, List, Optional, Tuple, Union
3 +
4 +import torch
5 +from torch._higher_order_ops.auto_functionalize import auto_functionalized
6 +
7 +from vllm.logger import init_logger
8 +
9 +from .vllm_inductor_pass import VllmInductorPass, is_func
10 +
11 +logger = init_logger(__name__)
12 +
13 +
14 +class FixFunctionalizationPass(VllmInductorPass):
15 + """
16 + This pass defunctionalizes certain nodes to avoid redundant tensor copies.
17 + After this pass, DCE (dead-code elimination) should never be run,
18 + as de-functionalized nodes may appear as dead code.
19 +
20 + To add new nodes to defunctionalize, add to the if-elif chain in __call__.
21 + """
22 +
23 + def __call__(self, graph: torch.fx.Graph):
24 + self.begin()
25 + self.dump_graph(graph, "before_fix_functionalization")
26 +
27 + self.nodes_to_remove: List[torch.fx.Node] = []
28 + count = 0
29 + for node in graph.nodes:
30 + if not is_func(node, auto_functionalized):
31 + continue # Avoid deep if-elif nesting
32 +
33 + kwargs = node.kwargs
34 + at_target = node.args[0]
35 +
36 + if at_target == torch.ops._C.rotary_embedding.default:
37 + query = kwargs['query']
38 + mm_node = query.args[0].args[0]
39 +
40 + # rotary_embedding is a special case: the two mutating inputs
41 + # are query and key, which are slices of mm_node.
42 + # While functionalized, results at[1] and at[2] are scattered
43 + # back into mm_node. After de-functionalization, we can just
44 + # use mm_node directly.
45 + for idx, user in self.getitem_users(node).items():
46 + for user_of_getitem in user.users:
47 + if is_func(user_of_getitem,
48 + torch.ops.aten.slice_scatter.default):
49 + user_of_getitem.replace_all_uses_with(mm_node)
50 + self._remove(user_of_getitem)
51 + self._remove(user)
52 +
53 + self.insert_defunctionalized(graph, node)
54 + self._remove(node)
55 +
56 + # These 2 replacements avoid the most copies for LLaMa.
57 + elif at_target == torch.ops._C.fused_add_rms_norm.default:
58 + mutated_args = {1: 'input', 2: 'residual'}
59 + self.defunctionalize(graph, node, mutated_args)
60 + elif at_target == torch.ops._C.fused_add_rms_norm_static_fp8_quant.default: # noqa: E501
61 + mutated_args = {1: 'result', 2: 'residual'}
62 + self.defunctionalize(graph, node, mutated_args)
63 +
64 + elif at_target in [
65 + torch.ops._C.rms_norm.default,
66 + torch.ops._C.rms_norm_static_fp8_quant.default
67 + ]:
68 + mutated_args = {1: 'result'}
69 + self.defunctionalize(graph, node, mutated_args)
70 +
71 + elif at_target == torch.ops._C.silu_and_mul.default:
72 + mutated_args = {1: 'out'}
73 + # Because we have an 'out', need to specify args directly
74 + self.defunctionalize(graph,
75 + node,
76 + mutated_args,
77 + args=('out', 'input'))
78 + else:
79 + continue # skip the count
80 +
81 + count += 1
82 +
83 + self.dump_graph(graph, "before_fix_functionalization_cleanup")
84 +
85 + # Remove the nodes all at once
86 + count_removed = len(self.nodes_to_remove)
87 + for node in self.nodes_to_remove:
88 + graph.erase_node(node)
89 +
90 + logger.debug("De-functionalized %s nodes, removed %s nodes", count,
91 + count_removed)
92 + self.dump_graph(graph, "after_fix_functionalization")
93 + self.end_and_log()
94 +
95 + def _remove(self, node_or_nodes: Union[torch.fx.Node,
96 + Iterable[torch.fx.Node]]):
97 + """
98 + Stage a node (or nodes) for removal at the end of the pass.
99 + """
100 + if isinstance(node_or_nodes, torch.fx.Node):
101 + self.nodes_to_remove.append(node_or_nodes)
102 + else:
103 + self.nodes_to_remove.extend(node_or_nodes)
104 +
105 + def defunctionalize(self,
106 + graph: torch.fx.Graph,
107 + node: torch.fx.Node,
108 + mutated_args: Dict[int, Union[torch.fx.Node, str]],
109 + args: Optional[Tuple[Union[torch.fx.Node, str],
110 + ...]] = None):
111 + """
112 + De-functionalize a node by replacing it with a call to the original.
113 + It also replaces the getitem users with the mutated arguments.
114 + See replace_users_with_mutated_args and insert_defunctionalized.
115 + """
116 + self.replace_users_with_mutated_args(node, mutated_args)
117 + self.insert_defunctionalized(graph, node, args=args)
118 + self._remove(node)
119 +
120 + def replace_users_with_mutated_args(self, node: torch.fx.Node,
121 + mutated_args: Dict[int,
122 + Union[torch.fx.Node,
123 + str]]):
124 + """
125 + Replace all getitem users of the auto-functionalized node with the
126 + mutated arguments.
127 + :param node: The auto-functionalized node
128 + :param mutated_args: The mutated arguments, indexed by getitem index.
129 + If the value of an arg is a string, `node.kwargs[arg]` is used.
130 + """
131 + for idx, user in self.getitem_users(node).items():
132 + arg = mutated_args[idx]
133 + arg = node.kwargs[arg] if isinstance(arg, str) else arg
134 + user.replace_all_uses_with(arg)
135 + self._remove(user)
136 +
137 + def getitem_users(self, node: torch.fx.Node) -> Dict[int, torch.fx.Node]:
138 + """
139 + Returns the operator.getitem users of the auto-functionalized node,
140 + indexed by the index they are getting.
141 + """
142 + users = {}
143 + for user in node.users:
144 + if is_func(user, operator.getitem):
145 + idx = user.args[1]
146 + users[idx] = user
147 + return users
148 +
149 + def insert_defunctionalized(self,
150 + graph: torch.fx.Graph,
151 + node: torch.fx.Node,
152 + args: Optional[Tuple[Union[torch.fx.Node, str],
153 + ...]] = None):
154 + """
155 + Insert a new defunctionalized node into the graph before node.
156 + If one of the kwargs is 'out', provide args directly,
157 + as node.kwargs cannot be used.
158 + See https://github.com/pytorch/pytorch/blob/a00faf440888ffb724bad413f329a49e2b6388e7/torch/_inductor/lowering.py#L351
159 +
160 + :param graph: Graph to insert the defunctionalized node into
161 + :param node: The auto-functionalized node to defunctionalize
162 + :param args: If we cannot use kwargs, specify args directly.
163 + If an arg is a string, `node.kwargs[arg]` is used.
164 + """ # noqa: E501
165 + assert is_func(node, auto_functionalized), \
166 + f"node must be auto-functionalized, is {node} instead"
167 +
168 + # Create a new call to the original function
169 + with graph.inserting_before(node):
170 + function = node.args[0]
171 + if args is None:
172 + graph.call_function(function, kwargs=node.kwargs)
173 + else:
174 + # Args passed as strings refer to items in node.kwargs
175 + args = tuple(node.kwargs[arg] if isinstance(arg, str) else arg
176 + for arg in args)
177 + graph.call_function(function, args=args)
vllm/compilation/fusion.py
+7 -6
@@ -6,10 +6,11 @@ from torch._higher_order_ops.auto_functionalize import auto_functionalized
6 from torch._inductor.pattern_matcher import (Match, PatternMatcherPass,
7 fwd_only, register_replacement)
8
9 -from vllm.compilation.inductor_pass import InductorPass
9 from vllm.config import CompilationConfig
10 from vllm.logger import init_logger
11
12 +from .vllm_inductor_pass import VllmInductorPass, is_func
13 +
14 logger = init_logger(__name__)
15
16
@@ -90,8 +91,6 @@ def empty_fp32(*args, **kwargs):
91
92
93 # Utilities for post-processing multi-output matches
93 -def is_func(node: torch.fx.Node, target) -> bool:
94 - return node.op == "call_function" and node.target == target
94
95
96 # Returns the first auto_functionalized node with the given op (if it exists)
@@ -127,7 +126,7 @@ def find_getitem(node: torch.fx.Node, idx: int) -> torch.fx.Node:
126 return ret
127
128
130 -class FusionPass(InductorPass):
129 +class FusionPass(VllmInductorPass):
130 """
131 This pass fuses a pre-defined set of custom ops into fused ops.
132 It uses the torch pattern matcher to find the patterns and replace them.
@@ -142,7 +141,7 @@ class FusionPass(InductorPass):
141 _instance: 'Optional[FusionPass]' = None
142
143 @classmethod
145 - def instance(cls, config: CompilationConfig):
144 + def instance(cls, config: CompilationConfig.PassConfig):
145 """
146 Get the singleton instance of the FusionPass.
147 If the instance exists, the config is updated but
@@ -154,7 +153,7 @@ class FusionPass(InductorPass):
153 cls._instance.config = config
154 return cls._instance
155
157 - def __init__(self, config: CompilationConfig):
156 + def __init__(self, config: CompilationConfig.PassConfig):
157 assert self.__class__._instance is None, \
158 "FusionPass singleton instance already exists"
159 super().__init__(config)
@@ -278,6 +277,7 @@ class FusionPass(InductorPass):
277 for node in match.nodes)
278
279 def __call__(self, graph: torch.fx.Graph):
280 + self.begin()
281 self.dump_graph(graph, "before_fusion")
282
283 count = self.patterns.apply(graph)
@@ -289,3 +289,4 @@ class FusionPass(InductorPass):
289 logger.debug("Post-processed %s matches", len(self.matches))
290 self.dump_graph(graph, "after_fusion")
291 self.matches.clear()
292 + self.end_and_log()
vllm/compilation/inductor_pass.py
+73 -27
@@ -1,38 +1,84 @@
1 +import hashlib
2 +import inspect
3 +import types
4 from abc import ABC, abstractmethod
5 +from typing import Any, Callable, Optional, Union
6
7 import torch
4 -
5 -from vllm.config import CompilationConfig
6 -# yapf: disable
7 -from vllm.distributed import get_tensor_model_parallel_rank as get_tp_rank
8 -from vllm.distributed import (
9 - get_tensor_model_parallel_world_size as get_tp_world_size)
10 -from vllm.distributed import model_parallel_is_initialized as p_is_init
11 -# yapf: enable
12 -from vllm.logger import init_logger
13 -
14 -logger = init_logger(__name__)
8 +from torch import fx
9
10
11 class InductorPass(ABC):
12 + """
13 + General custom inductor pass interface.
14 + TODO(torch==2.6) use torch._inductor.custom_graph_pass.CustomGraphPass
15 + """
16
17 @abstractmethod
18 def __call__(self, graph: torch.fx.Graph):
19 + """
20 + Execute the pass on the given graph.
21 + """
22 raise NotImplementedError
23
23 - def __init__(self, config: CompilationConfig):
24 - self.config = config
25 -
26 - def dump_graph(self, graph: torch.fx.Graph, stage: str):
27 - if stage in self.config.dump_graph_stages:
28 - # Make sure filename includes rank in the distributed setting
29 - parallel = p_is_init() and get_tp_world_size() > 1
30 - rank = f"-{get_tp_rank()}" if parallel else ""
31 - filepath = self.config.dump_graph_dir / f"{stage}{rank}.py"
32 -
33 - logger.info("Printing graph to %s", filepath)
34 - with open(filepath, "w") as f:
35 - src = graph.python_code(root_module="self", verbose=True).src
36 - # Add imports so it's not full of errors
37 - print("import torch; from torch import device", file=f)
38 - print(src, file=f)
24 + def uuid(self) -> Any:
25 + """
26 + Provide a unique identifier for the pass, used in Inductor code cache.
27 + This should depend on the pass implementation, so that changes to the
28 + pass result in recompilation.
29 + By default, the object source is hashed.
30 + """
31 + return InductorPass.hash_source(self)
32 +
33 + @staticmethod
34 + def hash_source(*srcs: Union[str, Any]):
35 + """
36 + Utility method to hash the sources of functions or objects.
37 + :param srcs: strings or objects to add to the hash.
38 + Objects and functions have their source inspected.
39 + :return:
40 + """
41 + hasher = hashlib.sha256()
42 + for src in srcs:
43 + if isinstance(src, str):
44 + src_str = src
45 + elif isinstance(src, types.FunctionType):
46 + src_str = inspect.getsource(src)
47 + else:
48 + src_str = inspect.getsource(src.__class__)
49 + hasher.update(src_str.encode("utf-8"))
50 + return hasher.digest()
51 +
52 +
53 +class CallableInductorPass(InductorPass):
54 + """
55 + This class is a wrapper for a callable that automatically provides an
56 + implementation of the UUID.
57 + """
58 +
59 + def __init__(self,
60 + callable: Callable[[fx.Graph], None],
61 + uuid: Optional[Any] = None):
62 + self.callable = callable
63 + if uuid is None:
64 + uuid = InductorPass.hash_source(callable)
65 + self._uuid = uuid
66 +
67 + def __call__(self, graph: torch.fx.Graph):
68 + self.callable(graph)
69 +
70 + def uuid(self) -> Any:
71 + return self._uuid
72 +
73 + def __getstate__(self):
74 + """
75 + Pickling occurs in the Inductor code cache if a pass is not given to
76 + the pass manager but is instead directly added to config as a pass.
77 + See PostGradPassManager for more.
78 +
79 + TODO(torch==2.6), use the `uuid` method in CustomGraphPass instead.
80 + """
81 + return self._uuid
82 +
83 + def __setstate__(self, state):
84 + raise ValueError("Cannot unpickle CallableInductorPass")
vllm/compilation/pass_manager.py new
+77
@@ -0,0 +1,77 @@
1 +from typing import List
2 +
3 +from torch import fx as fx
4 +
5 +from vllm.config import CompilationConfig
6 +from vllm.logger import init_logger
7 +
8 +from .fix_functionalization import FixFunctionalizationPass
9 +from .fusion import FusionPass
10 +from .inductor_pass import InductorPass
11 +from .reshapes import RedundantReshapesPass
12 +
13 +logger = init_logger(__name__)
14 +
15 +
16 +class PostGradPassManager:
17 + """
18 + The pass manager for post-grad passes.
19 + It handles configuration, adding custom passes, and running passes.
20 + It also supports pickling, which is used by the Inductor code cache.
21 + TODO(torch==2.6), use CustomGraphPass
22 + (torch._inductor.custom_graph_pass.CustomGraphPass)
23 +
24 + The order of the post-grad post-passes is:
25 + 1. passes (constructor parameter)
26 + 2. default passes (RedundantReshapesPass, FusionPass)
27 + 3. config["post_grad_custom_post_pass"] (if it exists)
28 + 4. fix_functionalization
29 + This way, all passes operate on a functionalized graph.
30 + """
31 +
32 + def __init__(self):
33 + self.passes: List[InductorPass] = []
34 +
35 + def __call__(self, graph: fx.Graph):
36 + for pass_ in self.passes:
37 + pass_(graph)
38 +
39 + # always run fix_functionalization last
40 + self.fix_functionalization(graph)
41 +
42 + def configure(self, pass_config: CompilationConfig.PassConfig):
43 + self.pass_config = pass_config
44 + if pass_config.enable_reshape:
45 + self.passes += [RedundantReshapesPass(pass_config)]
46 +
47 + if pass_config.enable_fusion:
48 + self.passes += [FusionPass.instance(pass_config)]
49 +
50 + self.fix_functionalization = FixFunctionalizationPass(pass_config)
51 +
52 + def add(self, pass_: InductorPass):
53 + assert isinstance(pass_, InductorPass)
54 + self.passes.append(pass_)
55 +
56 + def __getstate__(self):
57 + """
58 + Custom pickling for the pass manager, as some passes cannot be pickled.
59 + Pickling occurs because the pass manager is set as the value of
60 + `config["post_grad_custom_post_pass"]` in the Inductor config.
61 + The config is pickled to act as a key in the Inductor code cache.
62 + Any other passes in the config are pickled as well.
63 +
64 + TODO(torch==2.6), use the `uuid` method in CustomGraphPass instead.
65 + """
66 + state = {"pass_config": self.pass_config.uuid(), "passes": []}
67 + for pass_ in self.passes:
68 + state["passes"].append(pass_.uuid())
69 + state["passes"].append(self.fix_functionalization.uuid())
70 + return state
71 +
72 + def __setstate__(self, state):
73 + """
74 + Do not allow unpickling of the pass manager.
75 + If this is needed in the future, it should properly pickle the passes.
76 + """
77 + raise ValueError("Cannot unpickle PostGradPassManager")
vllm/compilation/reshapes.py
+5 -3
@@ -3,14 +3,14 @@ from typing import Union
3 import torch.fx
4 from torch import SymInt
5
6 -from vllm.compilation.fusion import is_func
7 -from vllm.compilation.inductor_pass import InductorPass
6 from vllm.logger import init_logger
7
8 +from .vllm_inductor_pass import VllmInductorPass, is_func
9 +
10 logger = init_logger(__name__)
11
12
13 -class RedundantReshapesPass(InductorPass):
13 +class RedundantReshapesPass(VllmInductorPass):
14 """
15 This is an inductor pass that removes redundant reshape operations.
16 It is required for RMSNorm-quant fusion to work properly.
@@ -31,6 +31,7 @@ class RedundantReshapesPass(InductorPass):
31 """
32
33 def __call__(self, graph: torch.fx.Graph):
34 + self.begin()
35 self.dump_graph(graph, "before_reshapes")
36 count = 0
37 # Remove no-op reshapes/views:
@@ -56,6 +57,7 @@ class RedundantReshapesPass(InductorPass):
57 logger.debug("Removed %s no-op reshapes", count)
58
59 self.dump_graph(graph, "after_reshapes")
60 + self.end_and_log()
61
62 def dims_equivalent(self, dim: Union[int, torch.fx.Node],
63 i_dim: Union[int, SymInt]) -> bool:
vllm/compilation/vllm_inductor_pass.py new
+53
@@ -0,0 +1,53 @@
1 +import time
2 +
3 +import torch
4 +
5 +from vllm.config import CompilationConfig
6 +# yapf: disable
7 +from vllm.distributed import get_tensor_model_parallel_rank as get_tp_rank
8 +from vllm.distributed import (
9 + get_tensor_model_parallel_world_size as get_tp_world_size)
10 +from vllm.distributed import model_parallel_is_initialized as p_is_init
11 +# yapf: enable
12 +from vllm.logger import init_logger
13 +
14 +from .inductor_pass import InductorPass
15 +
16 +logger = init_logger(__name__)
17 +
18 +
19 +def is_func(node: torch.fx.Node, target) -> bool:
20 + return node.op == "call_function" and node.target == target
21 +
22 +
23 +class VllmInductorPass(InductorPass):
24 + """
25 + An inductor pass with access to vLLM PassConfig.
26 + It provides timing, logging, and dumping utilities.
27 + """
28 +
29 + def __init__(self, config: CompilationConfig.PassConfig):
30 + self.config = config
31 + self.pass_name = self.__class__.__name__
32 +
33 + def dump_graph(self, graph: torch.fx.Graph, stage: str):
34 + if stage in self.config.dump_graph_stages:
35 + # Make sure filename includes rank in the distributed setting
36 + parallel = p_is_init() and get_tp_world_size() > 1
37 + rank = f"-{get_tp_rank()}" if parallel else ""
38 + filepath = self.config.dump_graph_dir / f"{stage}{rank}.py"
39 +
40 + logger.info("%s printing graph to %s", self.pass_name, filepath)
41 + with open(filepath, "w") as f:
42 + src = graph.python_code(root_module="self", verbose=True).src
43 + # Add imports so it's not full of errors
44 + print("import torch; from torch import device", file=f)
45 + print(src, file=f)
46 +
47 + def begin(self):
48 + self._start_time = time.perf_counter_ns()
49 +
50 + def end_and_log(self):
51 + self._end_time = time.perf_counter_ns()
52 + duration_ms = float(self._end_time - self._start_time) / 1.0e6
53 + logger.debug("%s completed in %.1f ms", self.pass_name, duration_ms)
vllm/config.py
+47 -13
@@ -1,5 +1,6 @@
1 import copy
2 import enum
3 +import hashlib
4 import json
5 import warnings
6 from dataclasses import dataclass, field, replace
@@ -13,6 +14,7 @@ from pydantic import BaseModel, Field, PrivateAttr
14 from transformers import PretrainedConfig
15
16 import vllm.envs as envs
17 +from vllm.compilation.inductor_pass import CallableInductorPass, InductorPass
18 from vllm.logger import init_logger
19 from vllm.model_executor.layers.quantization import (QUANTIZATION_METHODS,
20 get_quantization_config)
@@ -2120,12 +2122,7 @@ class CompilationConfig(BaseModel):
2122 name because the config uses json format. If we pass the config
2123 from Python, functions can also be passed directly via Python object
2124 constructor, e.g. `CompilationConfig(inductor_passes={"a": func})`
2123 - - custom inductor passes:
2124 - - dump_graph_stages: list of stages for which we want to dump the graph.
2125 - Each pass defines its own stages (before, after, maybe in-between).
2126 - - dump_graph_dir: directory to dump the graph. Default is .
2127 - - enable_fusion: whether to enable the custom fusion pass.
2128 - TODO better pass enabling system.
2125 + - custom inductor passes: see PassConfig for more details
2126
2127 Why we have different sizes for cudagraph and inductor:
2128 - cudagraph: a cudagraph captured for a specific size can only be used
@@ -2157,9 +2154,43 @@ class CompilationConfig(BaseModel):
2154 cudagraph_capture_sizes: Optional[List[int]] = None
2155 cudagraph_copy_inputs: bool = False
2156
2160 - dump_graph_stages: List[str] = Field(default_factory=list)
2161 - dump_graph_dir: Path = Field(default=Path("."))
2162 - enable_fusion: bool = True
2157 + class PassConfig(BaseModel):
2158 + """
2159 + Configuration for custom Inductor passes.
2160 + This is separate from general CompilationConfig so that inductor passes
2161 + don't all have access to full configuration - that would create a cycle
2162 + as the PassManager is set as a property of config.
2163 + - dump_graph_stages: list of stages for which we want to dump the graph.
2164 + Each pass defines its own stages (before, after, maybe in-between).
2165 + - dump_graph_dir: directory to dump the graphs. Default is .
2166 + - enable_fusion: whether to enable the custom fusion pass.
2167 + - enable_reshape: whether to enable the custom reshape elimination pass.
2168 + TODO better pass enabling system.
2169 + """
2170 + dump_graph_stages: List[str] = Field(default_factory=list)
2171 + dump_graph_dir: Path = Field(default=Path("."))
2172 + enable_fusion: bool = True
2173 + enable_reshape: bool = True
2174 +
2175 + def uuid(self):
2176 + """
2177 + Produces a hash unique to the pass configuration.
2178 + Any new fields that affect compilation should be added to the hash.
2179 + Do not include dump_graph_* in the hash - they don't affect
2180 + compilation.
2181 + """
2182 + dict_ = self.model_dump(
2183 + include={"enable_fusion", "enable_reshape"})
2184 + encoded = json.dumps(dict_, sort_keys=True).encode("utf-8")
2185 + return hashlib.sha256(encoded).digest()
2186 +
2187 + def model_post_init(self, __context: Any) -> None:
2188 + if not self.enable_reshape and self.enable_fusion:
2189 + print_warning_once(
2190 + "Fusion enabled but reshape elimination disabled."
2191 + "RMSNorm + quant (fp8) fusion might not work")
2192 +
2193 + pass_config: PassConfig = Field(default_factory=PassConfig)
2194
2195 # not configurable, computed after init
2196 compile_sizes: List[int] = PrivateAttr
@@ -2185,8 +2216,9 @@ class CompilationConfig(BaseModel):
2216 for k, v in self.inductor_passes.items():
2217 if not isinstance(v, str):
2218 assert callable(v), (
2188 - f"pass {k} should be a function or a qualified name")
2189 - self.inductor_compile_config[k] = v
2219 + f"pass {k} should be callable or a qualified name")
2220 + self.inductor_compile_config[k] = v if isinstance(
2221 + v, InductorPass) else CallableInductorPass(v)
2222 continue
2223
2224 # resolve function from qualified name
@@ -2194,7 +2226,8 @@ class CompilationConfig(BaseModel):
2226 module = ".".join(names[:-1])
2227 func_name = names[-1]
2228 func = __import__(module).__dict__[func_name]
2197 - self.inductor_compile_config[k] = func
2229 + self.inductor_compile_config[k] = func if isinstance(
2230 + func, InductorPass) else CallableInductorPass(func)
2231
2232 self.enabled_custom_ops = Counter()
2233 self.disabled_custom_ops = Counter()
@@ -2344,7 +2377,8 @@ class VllmConfig:
2377 self.compilation_config.custom_ops = ["none"]
2378 self.compilation_config.use_cudagraph = True
2379 self.compilation_config.use_inductor = True
2347 - self.compilation_config.enable_fusion = False
2380 + self.compilation_config.pass_config.enable_fusion = False
2381 + self.compilation_config.pass_config.enable_reshape = False
2382
2383 current_platform.check_and_update_config(self)
2384
vllm/utils.py
-9
@@ -1501,15 +1501,6 @@ class LazyDict(Mapping, Generic[T]):
1501 return len(self._factory)
1502
1503
1504 -def combine_fx_passes(passes: List[Callable]) -> Callable:
1505 -
1506 - def combined_fx(graph) -> None:
1507 - for fx in passes:
1508 - fx(graph)
1509 -
1510 - return combined_fx
1511 -
1512 -
1504 def weak_ref_tensor(tensor: torch.Tensor) -> torch.Tensor:
1505 """
1506 Create a weak reference to a tensor.
vllm/v1/worker/gpu_model_runner.py
+1 -1
@@ -548,7 +548,7 @@ class GPUModelRunner:
548 if not self.use_cuda_graph:
549 logger.warning(
550 "Skipping CUDA graph capture. Please add "
551 - "-O 3 to use CUDA graphs.", CompilationLevel.PIECEWISE)
551 + "-O %s to use CUDA graphs.", CompilationLevel.PIECEWISE)
552 return
553
554 start_time = time.perf_counter()