A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[torch.compile] fix functionalization (#8480)
youkaichao committed
Sep 14, 2024 at 09:46 UTC
a36e070dad7d7098f69324b8275a533140221809
3 files changed
+167
-5
tests/compile/test_full_graph.py
+9
-4
@@ -16,7 +16,12 @@ def test_full_graph(model):
16
"The future of AI is",
17
]
18
sampling_params = SamplingParams(temperature=0)
19
- llm = LLM(model="meta-llama/Meta-Llama-3-8B",
20
- enforce_eager=True,
21
- load_format="dummy")
22
- llm.generate(prompts, sampling_params)
19
+ llm = LLM(model=model, enforce_eager=True)
20
+
21
+ outputs = llm.generate(prompts, sampling_params)
22
+
23
+ # Print the outputs.
24
+ for output in outputs:
25
+ prompt = output.prompt
26
+ generated_text = output.outputs[0].text
27
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
vllm/compilation/backends.py
new
+156
@@ -0,0 +1,156 @@
1
+import operator
2
+
3
+import torch
4
+import torch.fx as fx
5
+
6
+
7
+def fix_functionalization(graph: fx.Graph):
8
+ """
9
+ Rewrite the graph module to replace the pattern involving
10
+ torch._higher_order_ops.auto_functionalize.auto_functionalized
11
+ with a direct call to the inplace custom op.
12
+
13
+ # TODO: check if PyTorch nightly has fixed this issue
14
+ """
15
+
16
+ # debug code, if we want to see the graph before the transformation
17
+ # with open("before.py", "w") as f:
18
+ # print(graph.python_code(root_module="self", verbose=True).src, file=f)
19
+
20
+ nodes_to_remove = []
21
+
22
+ for node in graph.nodes:
23
+ # Identify the auto_functionalized node
24
+ if node.op == 'call_function' and node.target == torch._higher_order_ops.auto_functionalize.auto_functionalized: # noqa
25
+ if node.args[0] == torch.ops._C.rotary_embedding.default:
26
+ # manual replace for rotary_embedding
27
+
28
+ # Now, collect the arguments
29
+ kwargs = node.kwargs
30
+
31
+ query = kwargs['query']
32
+ mm_node = query.args[0].args[0]
33
+
34
+ # Create a new call to torch.ops._C.rotary_embedding.default
35
+ with graph.inserting_before(node):
36
+ # just insert the call to the custom op
37
+ # NOTE: don't run dead code elimination,
38
+ # otherwise this op will be removed
39
+ graph.call_function(torch.ops._C.rotary_embedding.default,
40
+ kwargs=kwargs)
41
+
42
+ # Remove the auto_functionalized node
43
+ # Since the node may have outputs, we need to handle its users
44
+ # Replace uses of the outputs (getitem nodes) with mm_node
45
+ for user in list(node.users):
46
+ if user.op == 'call_function' and user.target == operator.getitem: # noqa
47
+ # Remove the getitem node
48
+ for getitem_user in list(user.users):
49
+ if (getitem_user.op == 'call_function'
50
+ and getitem_user.target
51
+ == torch.ops.aten.slice_scatter.default):
52
+ # Replace the uses of slice_scatter node
53
+ # with mm_node
54
+ getitem_user.replace_all_uses_with(mm_node)
55
+ nodes_to_remove.append(getitem_user)
56
+ nodes_to_remove.append(user)
57
+ nodes_to_remove.append(node)
58
+
59
+ elif node.args[0] == torch.ops._C.fused_add_rms_norm.default:
60
+ # manual replace for fused_add_rms_norm
61
+ # this is the most effective optimization for llama
62
+ # failing to do this will result in many unnecessary copies
63
+
64
+ kwargs = node.kwargs
65
+
66
+ input = kwargs['input']
67
+ residual = kwargs['residual']
68
+
69
+ # Create a new call to torch.ops._C.rotary_embedding.default
70
+ with graph.inserting_before(node):
71
+ # just insert the call to the custom op
72
+ # NOTE: don't run dead code elimination,
73
+ # otherwise this op will be removed
74
+ graph.call_function(
75
+ torch.ops._C.fused_add_rms_norm.default, kwargs=kwargs)
76
+
77
+ for user in list(node.users):
78
+ if user.op == 'call_function' and user.target == operator.getitem: # noqa
79
+ # Remove the getitem node
80
+ if user.args[1] == 1:
81
+ replace_node = input
82
+ elif user.args[1] == 2:
83
+ replace_node = residual
84
+ user.replace_all_uses_with(replace_node)
85
+ nodes_to_remove.append(user)
86
+ nodes_to_remove.append(node)
87
+
88
+ elif node.args[0] == torch.ops._C.rms_norm.default:
89
+ # manual replace for rms_norm
90
+
91
+ kwargs = node.kwargs
92
+
93
+ input = kwargs['input']
94
+ out = kwargs['out']
95
+ weight = kwargs['weight']
96
+ epsilon = kwargs['epsilon']
97
+ # Create a new call to torch.ops._C.rotary_embedding.default
98
+ # cannot use kwargs, because we have an `out`, see https://github.com/pytorch/pytorch/blob/a00faf440888ffb724bad413f329a49e2b6388e7/torch/_inductor/lowering.py#L351 # noqa
99
+ with graph.inserting_before(node):
100
+ # just insert the call to the custom op
101
+ # NOTE: don't run dead code elimination,
102
+ # otherwise this op will be removed
103
+ graph.call_function(
104
+ torch.ops._C.rms_norm.default,
105
+ args=(out, input, weight, epsilon),
106
+ )
107
+
108
+ replace_node = out
109
+
110
+ for user in list(node.users):
111
+ if user.op == 'call_function' and user.target == operator.getitem: # noqa
112
+ user.replace_all_uses_with(replace_node)
113
+ nodes_to_remove.append(user)
114
+ nodes_to_remove.append(node)
115
+
116
+ elif node.args[0] == torch.ops._C.silu_and_mul.default:
117
+ # manual replace for silu_and_mul
118
+
119
+ kwargs = node.kwargs
120
+
121
+ input = kwargs['input']
122
+ out = kwargs['out']
123
+
124
+ # Create a new call to torch.ops._C.rotary_embedding.default
125
+ # cannot use kwargs, because we have an `out`, see https://github.com/pytorch/pytorch/blob/a00faf440888ffb724bad413f329a49e2b6388e7/torch/_inductor/lowering.py#L351 # noqa
126
+ with graph.inserting_before(node):
127
+ # just insert the call to the custom op
128
+ # NOTE: don't run dead code elimination,
129
+ # otherwise this op will be removed
130
+ graph.call_function(
131
+ torch.ops._C.silu_and_mul.default,
132
+ args=(out, input),
133
+ )
134
+ replace_node = out
135
+
136
+ for user in list(node.users):
137
+ if user.op == 'call_function' and user.target == operator.getitem: # noqa
138
+ user.replace_all_uses_with(replace_node)
139
+ nodes_to_remove.append(user)
140
+ nodes_to_remove.append(node)
141
+
142
+ # Remove the nodes all at once
143
+ for node in nodes_to_remove:
144
+ graph.erase_node(node)
145
+
146
+ # debug code, if we want to see the graph after the transformation
147
+ # with open("after.py", "w") as f:
148
+ # print(graph.python_code(root_module="self", verbose=True).src, file=f)
149
+
150
+
151
+def vllm_backend(graph, example_inputs):
152
+ from torch._inductor import config
153
+ current_config = config.shallow_copy_dict()
154
+ from torch._inductor.compile_fx import compile_fx
155
+ current_config['post_grad_custom_post_pass'] = fix_functionalization
156
+ return compile_fx(graph, example_inputs, config_patches=current_config)
vllm/worker/model_runner.py
+2
-1
@@ -1064,8 +1064,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1064
"This may lead to less accurate results!")
1065
1066
if envs.VLLM_TEST_DYNAMO_GRAPH_CAPTURE and supports_dynamo():
1067
+ from vllm.compilation.backends import vllm_backend
1068
from vllm.plugins import get_torch_compile_backend
1068
- backend = get_torch_compile_backend() or "eager"
1069
+ backend = get_torch_compile_backend() or vllm_backend
1070
self.model = torch.compile(
1071
self.model,
1072
fullgraph=envs.VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE,