A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[torch.compile] rework compile control with piecewise cudagraph (#9715)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Oct 29, 2024 at 23:03 UTC
ff5ed6e1bcbd112a26f8eb43b6bfdbc5ec73726e
17 files changed
+983
-106
.buildkite/test-pipeline.yaml
+3
@@ -229,6 +229,9 @@ steps:
229
- tests/compile
230
commands:
231
- pytest -v -s compile/test_basic_correctness.py
232
+ # these tests need to be separated, cannot combine
233
+ - pytest -v -s compile/piecewise/test_simple.py
234
+ - pytest -v -s compile/piecewise/test_toy_llama.py
235
236
- label: "PyTorch Fullgraph Test" # 18min
237
source_file_dependencies:
tests/compile/piecewise/__init__.py
tests/compile/piecewise/piecewise_compilation_config.json
new
+4
@@ -0,0 +1,4 @@
1
+{
2
+ "use_cudagraph": true,
3
+ "non_cudagraph_ops": ["silly.attention"]
4
+}
\ No newline at end of file
tests/compile/piecewise/test_simple.py
new
+96
@@ -0,0 +1,96 @@
1
+"""
2
+Test the piecewise compilation with a simple model so that we
3
+can exactly calculate the expected output and side effects.
4
+"""
5
+import os
6
+
7
+import torch
8
+from torch import nn
9
+
10
+from vllm.compilation.compile_context import set_compile_context
11
+from vllm.compilation.counter import compilation_counter
12
+from vllm.compilation.decorators import support_torch_compile
13
+from vllm.compilation.levels import CompilationLevel
14
+
15
+os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(CompilationLevel.PIECEWISE)
16
+
17
+global_counter = 0
18
+
19
+
20
+@torch.library.custom_op("silly::attention", mutates_args=["out"])
21
+def silly_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,
22
+ out: torch.Tensor) -> None:
23
+ global global_counter
24
+ global_counter += 1
25
+ print(f"{global_counter=}")
26
+ out.copy_(q)
27
+ out[0] += 1
28
+
29
+
30
+@silly_attention.register_fake
31
+def _(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,
32
+ out: torch.Tensor) -> None:
33
+ return
34
+
35
+
36
+@support_torch_compile
37
+class SillyModel(nn.Module):
38
+
39
+ def __init__(self) -> None:
40
+ super().__init__()
41
+
42
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
43
+ """
44
+ Overall effect:
45
+ x += 1
46
+ x[0] += 2
47
+ global_counter += 2
48
+ """
49
+ x = x + 1
50
+ x = x + 2
51
+ out = torch.empty_like(x)
52
+ torch.ops.silly.attention(x, x, x, out)
53
+ x = out
54
+ x = x - 2
55
+ x = x - 1
56
+ out = torch.empty_like(x)
57
+ torch.ops.silly.attention(x, x, x, out)
58
+ x = out
59
+ x = x + 1
60
+ return x
61
+
62
+
63
+def test_simple_piecewise_compile():
64
+
65
+ model = SillyModel()
66
+
67
+ directory = os.path.dirname(__file__)
68
+ config = os.path.join(directory, "piecewise_compilation_config.json")
69
+ os.environ["VLLM_TORCH_COMPILE_CONFIG"] = config
70
+
71
+ input_buffer = torch.randn(100).cuda()
72
+
73
+ with compilation_counter.expect(
74
+ num_graphs_seen=1, # one graph for the model
75
+ num_piecewise_graphs_seen=5, # 2 * num_layers + 1
76
+ num_piecewise_capturable_graphs_seen=3, # 1 + num_layers
77
+ num_inductor_compilations=3, # num_piecewise_capturable_graphs_seen
78
+ num_cudagraph_caputured=
79
+ 6, # num_cudagraph_sizes * num_piecewise_capturable_graphs_seen
80
+ ):
81
+
82
+ with set_compile_context([1, 2]):
83
+ model(input_buffer)
84
+
85
+ model(input_buffer[:2])
86
+ model(input_buffer[:1])
87
+
88
+ input_buffer[:2].zero_()
89
+ global global_counter
90
+ global_counter = 0
91
+ output = model(input_buffer[:2])
92
+ assert global_counter == 2
93
+ assert torch.allclose(output.cpu(), torch.tensor([3., 1.]))
94
+
95
+ # clean up to avoid side effects for other tests
96
+ del os.environ["VLLM_TORCH_COMPILE_CONFIG"]
tests/compile/piecewise/test_toy_llama.py
new
+334
@@ -0,0 +1,334 @@
1
+"""
2
+Test the piecewise compilation with a simple model, comparing the output
3
+with and without the piecewise compilation.
4
+"""
5
+import os
6
+from dataclasses import dataclass
7
+from typing import Optional, Tuple
8
+
9
+import torch
10
+from torch import nn
11
+
12
+from vllm.compilation.compile_context import set_compile_context
13
+from vllm.compilation.config import CompilationConfig
14
+from vllm.compilation.counter import compilation_counter
15
+from vllm.compilation.decorators import support_torch_compile
16
+from vllm.compilation.levels import CompilationLevel
17
+from vllm.plugins import set_compilation_config
18
+
19
+
20
+@torch.library.custom_op("silly::attention", mutates_args=["out"])
21
+def silly_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,
22
+ out: torch.Tensor) -> None:
23
+ out.copy_(q)
24
+ out += k
25
+ out += v
26
+
27
+
28
+@silly_attention.register_fake
29
+def _(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,
30
+ out: torch.Tensor) -> None:
31
+ return
32
+
33
+
34
+@dataclass
35
+class LlamaConfig:
36
+ hidden_size: int = 128
37
+ mlp_size: int = 256
38
+ vocab_size: int = 128
39
+ num_layers: int = 2
40
+
41
+
42
+class LlamaMLP(nn.Module):
43
+
44
+ def __init__(self, config: LlamaConfig) -> None:
45
+ super().__init__()
46
+ self.gate_up_projection = nn.Linear(
47
+ in_features=config.hidden_size,
48
+ out_features=config.mlp_size * 2,
49
+ bias=False,
50
+ )
51
+ self.down_projection = nn.Linear(
52
+ in_features=config.mlp_size,
53
+ out_features=config.hidden_size,
54
+ bias=False,
55
+ )
56
+
57
+ self.gate_up_projection.weight.data.fill_(0.0)
58
+ self.down_projection.weight.data.fill_(0.0)
59
+
60
+ def forward(self, x):
61
+ x = self.gate_up_projection(x)
62
+ x = x[:, :x.size(1) // 2] * torch.nn.functional.relu(
63
+ x[:, x.size(1) // 2:])
64
+ x = self.down_projection(x)
65
+ return x
66
+
67
+
68
+class LlamaAttention(nn.Module):
69
+
70
+ def __init__(self, config: LlamaConfig) -> None:
71
+ super().__init__()
72
+ self.qkv_projection = nn.Linear(
73
+ in_features=config.hidden_size,
74
+ out_features=config.hidden_size * 3,
75
+ )
76
+
77
+ self.output_projection = nn.Linear(
78
+ in_features=config.hidden_size,
79
+ out_features=config.hidden_size,
80
+ )
81
+
82
+ self.qkv_projection.weight.data.fill_(0.0)
83
+ self.output_projection.weight.data.fill_(0.0)
84
+
85
+ def forward(
86
+ self,
87
+ positions: torch.Tensor,
88
+ hidden_states: torch.Tensor,
89
+ ) -> torch.Tensor:
90
+ qkv = self.qkv_projection(hidden_states)
91
+ hidden_size = qkv.size(-1) // 3
92
+ q, k, v = qkv.split([hidden_size, hidden_size, hidden_size], dim=-1)
93
+
94
+ q = q + positions.unsqueeze(1)
95
+ k = k + positions.unsqueeze(1)
96
+
97
+ attn_output = torch.empty_like(q)
98
+ torch.ops.silly.attention(q, k, v, attn_output)
99
+
100
+ output = self.output_projection(attn_output)
101
+ return output
102
+
103
+
104
+class LlamaDecoderLayer(nn.Module):
105
+
106
+ def __init__(self, config: LlamaConfig) -> None:
107
+ super().__init__()
108
+ self.self_attention = LlamaAttention(config)
109
+ self.mlp = LlamaMLP(config)
110
+
111
+ def forward(
112
+ self,
113
+ positions: torch.Tensor,
114
+ hidden_states: torch.Tensor,
115
+ residual: Optional[torch.Tensor],
116
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
117
+ if residual is None:
118
+ residual = hidden_states
119
+ hidden_states = hidden_states / 2
120
+ else:
121
+ hidden_states = hidden_states + residual
122
+ residual = hidden_states
123
+ hidden_states = hidden_states / 2
124
+
125
+ hidden_states = self.self_attention(positions=positions,
126
+ hidden_states=hidden_states)
127
+
128
+ hidden_states = hidden_states + residual
129
+ residual = hidden_states
130
+ hidden_states = hidden_states / 2
131
+ hidden_states = self.mlp(hidden_states)
132
+
133
+ return hidden_states, residual
134
+
135
+
136
+class LlamaModel(nn.Module):
137
+
138
+ def __init__(self, config: LlamaConfig) -> None:
139
+ super().__init__()
140
+ self.embedding_tokens = nn.Embedding(
141
+ num_embeddings=config.vocab_size,
142
+ embedding_dim=config.hidden_size,
143
+ )
144
+ self.layers = nn.ModuleList(
145
+ [LlamaDecoderLayer(config) for _ in range(config.num_layers)])
146
+
147
+ self.embedding_tokens.weight.data.fill_(0.0)
148
+
149
+ def forward(
150
+ self,
151
+ input_ids: Optional[torch.Tensor],
152
+ positions: torch.Tensor,
153
+ ) -> torch.Tensor:
154
+ hidden_states = self.embedding_tokens(input_ids)
155
+ residual = None
156
+ for layer in self.layers:
157
+ hidden_states, residual = layer(positions, hidden_states, residual)
158
+ return hidden_states
159
+
160
+
161
+@torch.inference_mode
162
+def run_model(llama_config,
163
+ use_compile: bool,
164
+ split_attn: bool = False) -> torch.Tensor:
165
+
166
+ if use_compile:
167
+ os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(
168
+ CompilationLevel.PIECEWISE)
169
+
170
+ if split_attn:
171
+ set_compilation_config(
172
+ CompilationConfig(
173
+ use_cudagraph=True,
174
+ non_cudagraph_ops=["silly.attention"],
175
+ ))
176
+ else:
177
+ set_compilation_config(CompilationConfig(use_cudagraph=True, ))
178
+ else:
179
+ os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(
180
+ CompilationLevel.NO_COMPILATION)
181
+ set_compilation_config(None)
182
+
183
+ cls = LlamaModel
184
+ if use_compile:
185
+ cls = support_torch_compile(LlamaModel)
186
+ model = cls(llama_config).eval().cuda()
187
+
188
+ B = 16 # max batch size
189
+ input_ids = torch.randint(0, llama_config.vocab_size, (B, )).cuda()
190
+ positions = torch.arange(B).cuda()
191
+
192
+ with set_compile_context([1, 2]):
193
+ model(input_ids, positions)
194
+ model(input_ids[:2], positions[:2])
195
+ model(input_ids[:1], positions[:1])
196
+
197
+ input_ids[:2].zero_()
198
+ output = model(input_ids[:2], positions[:2])
199
+
200
+ # manual cleanup
201
+ del os.environ["VLLM_TORCH_COMPILE_LEVEL"]
202
+ set_compilation_config(None)
203
+
204
+ return output.cpu()
205
+
206
+
207
+def test_toy_llama():
208
+ # compare output with and without piecewise compilation
209
+
210
+ llama_config = LlamaConfig(hidden_size=128,
211
+ mlp_size=256,
212
+ vocab_size=128,
213
+ num_layers=2)
214
+
215
+ outputs = []
216
+ with compilation_counter.expect(
217
+ num_graphs_seen=0,
218
+ num_piecewise_graphs_seen=0,
219
+ num_piecewise_capturable_graphs_seen=0,
220
+ num_inductor_compilations=0,
221
+ num_cudagraph_caputured=0,
222
+ ):
223
+ outputs.append(run_model(llama_config, use_compile=False))
224
+ with compilation_counter.expect(
225
+ num_graphs_seen=1, # one graph for the model
226
+ num_piecewise_graphs_seen=1,
227
+ num_piecewise_capturable_graphs_seen=1,
228
+ num_inductor_compilations=1, # num_piecewise_capturable_graphs_seen
229
+ num_cudagraph_caputured=
230
+ 2, # num_cudagraph_sizes * num_piecewise_capturable_graphs_seen
231
+ ):
232
+ outputs.append(run_model(llama_config, use_compile=True))
233
+
234
+ with compilation_counter.expect(
235
+ num_graphs_seen=1, # one graph for the model
236
+ num_piecewise_graphs_seen=2 * llama_config.num_layers +
237
+ 1, # 2 * num_layers + 1
238
+ num_piecewise_capturable_graphs_seen=1 +
239
+ llama_config.num_layers, # 1 + num_layers
240
+ num_inductor_compilations=1 +
241
+ llama_config.num_layers, # num_piecewise_capturable_graphs_seen
242
+ num_cudagraph_caputured=2 *
243
+ (1 + llama_config.num_layers
244
+ ), # num_cudagraph_sizes * num_piecewise_capturable_graphs_seen
245
+ ):
246
+ outputs.append(
247
+ run_model(llama_config, use_compile=True, split_attn=True))
248
+
249
+ for i in range(1, len(outputs)):
250
+ assert torch.allclose(outputs[0], outputs[i])
251
+
252
+
253
+@torch.inference_mode
254
+def benchmark():
255
+ os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(CompilationLevel.PIECEWISE)
256
+ from triton.testing import do_bench
257
+ cls = support_torch_compile(LlamaModel)
258
+
259
+ # similar to llama 3.1-8B
260
+ llama_config = LlamaConfig(hidden_size=4096,
261
+ mlp_size=14336,
262
+ vocab_size=128 * 1024,
263
+ num_layers=32)
264
+
265
+ # a tiny model to measure the overhead
266
+ # of piecewise cudagraph
267
+ llama_config = LlamaConfig(hidden_size=40,
268
+ mlp_size=80,
269
+ vocab_size=128,
270
+ num_layers=2)
271
+
272
+ cudagraph_sizes = [1, 2, 4] + [i * 8 for i in range(1, 33)]
273
+
274
+ eager_time = {}
275
+ full_cudagraph_time = {}
276
+ piecewise_cudagraph_time = {}
277
+
278
+ pool = torch.cuda.graph_pool_handle()
279
+
280
+ for piecewise in [False, True]:
281
+ if piecewise:
282
+ set_compilation_config(
283
+ CompilationConfig(
284
+ use_cudagraph=True,
285
+ non_cudagraph_ops=["silly.attention"],
286
+ ))
287
+ else:
288
+ set_compilation_config(None)
289
+
290
+ model = cls(llama_config).eval().cuda().to(torch.bfloat16)
291
+
292
+ B = 256 # max batch size
293
+ input_ids = torch.randint(0, llama_config.vocab_size, (B, )).cuda()
294
+ positions = torch.arange(B).cuda().to(torch.bfloat16)
295
+
296
+ graphs = {}
297
+
298
+ with set_compile_context(cudagraph_sizes):
299
+ model(input_ids, positions)
300
+ for b in cudagraph_sizes[::-1]:
301
+ if not piecewise:
302
+ graph = torch.cuda.CUDAGraph()
303
+ with torch.cuda.graph(graph, pool=pool):
304
+ output = model(input_ids[:b], positions[:b])
305
+ graphs[b] = (graph, output)
306
+ else:
307
+ output = model(input_ids[:b], positions[:b])
308
+ graphs[b] = (model, output)
309
+ for b in cudagraph_sizes:
310
+ if piecewise:
311
+ # noqa is for `Function definition does not bind loop variable`
312
+ # it will be problematic if we save the created lambda function
313
+ # and use it later, because it will look up the name `b` in the
314
+ # enclosing scope, and the value of `b` will always be 256.
315
+ # it is fine here, because we only use the lambda function once.
316
+ runtime = do_bench(lambda: graphs[b][0] # noqa
317
+ (input_ids[:b], positions[:b])) # noqa
318
+ piecewise_cudagraph_time[b] = runtime
319
+ else:
320
+ runtime = do_bench(lambda: graphs[b][0].replay()) # noqa
321
+ eager_runtime = do_bench(
322
+ lambda: model(input_ids[:b], positions[:b])) # noqa
323
+ full_cudagraph_time[b] = runtime
324
+ eager_time[b] = eager_runtime
325
+
326
+ # print in tabular format
327
+ print("batch size\teager mode\tfull cudagraph\tpiecewise cudagraph")
328
+ for b in cudagraph_sizes:
329
+ print((f"{b}\t{eager_time[b]:.3f}\t{full_cudagraph_time[b]:.3f}"
330
+ f"\t{piecewise_cudagraph_time[b]:.3f}"))
331
+
332
+
333
+if __name__ == "__main__":
334
+ benchmark()
tests/compile/test_full_graph.py
+1
-1
@@ -9,7 +9,7 @@ from .utils import TEST_MODELS, check_full_graph_support
9
@pytest.mark.parametrize("model_info", TEST_MODELS)
10
@pytest.mark.parametrize(
11
"optimization_level",
12
- [CompilationLevel.DYNAMO_ONCE, CompilationLevel.INDUCTOR])
12
+ [CompilationLevel.DYNAMO_ONCE, CompilationLevel.PIECEWISE])
13
@fork_new_process_for_each_test
14
def test_full_graph(model_info, optimization_level):
15
model = model_info[0]
tests/compile/utils.py
+10
-8
@@ -9,17 +9,19 @@ from vllm.platforms import current_platform
9
10
TEST_MODELS = [
11
("facebook/opt-125m", {}),
12
- ("nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", {
13
- "dtype": torch.float16,
14
- "quantization": "compressed-tensors"
15
- }),
12
+ # TODO: add fake implementation for compressed-tensors
13
+ # ("nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", {
14
+ # "dtype": torch.float16,
15
+ # "quantization": "compressed-tensors"
16
+ # }),
17
("neuralmagic/Meta-Llama-3-8B-Instruct-FP8", {
18
"dtype": torch.float16,
19
"quantization": "fp8"
20
}),
20
- ("nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dyn-Per-Token-2048-Samples", {
21
- "quantization": "compressed-tensors"
22
- }),
21
+ # TODO: add fake implementation for compressed-tensors
22
+ # ("nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dyn-Per-Token-2048-Samples", {
23
+ # "quantization": "compressed-tensors"
24
+ # }),
25
("meta-llama/Meta-Llama-3-8B", {}),
26
]
27
@@ -73,7 +75,7 @@ def check_full_graph_support(model,
75
# much memory.
76
quantization = model_kwargs.get("quantization")
77
if ((quantization == "fp8" or model == "meta-llama/Meta-Llama-3-8B")
76
- and optimization_level >= CompilationLevel.INDUCTOR):
78
+ and optimization_level >= CompilationLevel.PIECEWISE):
79
return
80
81
prompts = [
vllm/compilation/backends.py
+300
-84
@@ -1,13 +1,16 @@
1
import copy
2
+import dataclasses
3
import operator
3
-from typing import Callable, Dict, List, Optional, Tuple, Union
4
+from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
5
6
import torch
7
import torch.fx as fx
8
9
from vllm.logger import init_logger
10
+from vllm.utils import weak_ref_tensors
11
10
-from .compile_context import get_compile_context
12
+from .config import CompilationConfig
13
+from .counter import compilation_counter
14
from .levels import CompilationLevel
15
16
logger = init_logger(__name__)
@@ -157,113 +160,326 @@ def fix_functionalization(graph: fx.Graph):
160
# print(graph.python_code(root_module="self", verbose=True).src, file=f)
161
162
160
-def wrap_inductor(graph, example_inputs, additional_inductor_config):
163
+def wrap_inductor(graph,
164
+ example_inputs,
165
+ additional_inductor_config,
166
+ do_logging=False,
167
+ runtime_shape: Optional[int] = None,
168
+ use_inductor: bool = True):
169
+ if not use_inductor:
170
+ return graph
171
+
172
+ compilation_counter.num_inductor_compilations += 1
173
+
174
+ if do_logging:
175
+ if runtime_shape is None:
176
+ logger.info("Compiling a graph for general shape")
177
+ else:
178
+ logger.info("Compiling a graph for shape %s", runtime_shape)
179
+
180
from torch._inductor import config
181
current_config = config.shallow_copy_dict()
182
from torch._inductor.compile_fx import compile_fx
183
184
if additional_inductor_config is not None:
185
current_config.update(additional_inductor_config)
167
- if current_config['post_grad_custom_post_pass'] is not None:
168
- logger.warning(
169
- "post_grad_custom_post_pass is already set in the config. "
170
- "Overwriting it with the fix_functionalization")
171
- current_config['post_grad_custom_post_pass'] = fix_functionalization
186
+
187
+ # inductor can inplace modify the graph, so we need to copy it
188
+ # see https://github.com/pytorch/pytorch/issues/138980
189
+ graph = copy.deepcopy(graph)
190
return compile_fx(graph, example_inputs, config_patches=current_config)
191
192
175
-def vllm_backend(
193
+@dataclasses.dataclass
194
+class SplitItem:
195
+ submod_name: str
196
+ is_splitting_graph: bool
197
+ graph: fx.GraphModule
198
+
199
+
200
+def split_graph(graph: fx.GraphModule,
201
+ ops: List[str]) -> Tuple[fx.GraphModule, List[SplitItem]]:
202
+ # split graph by ops
203
+ subgraph_id = 0
204
+ node_to_subgraph_id = {}
205
+ split_op_graphs = []
206
+ for node in graph.graph.nodes:
207
+ if node.op in ("output", "placeholder"):
208
+ continue
209
+ if node.op == 'call_function' and str(node.target) in ops:
210
+ subgraph_id += 1
211
+ node_to_subgraph_id[node] = subgraph_id
212
+ split_op_graphs.append(subgraph_id)
213
+ subgraph_id += 1
214
+ else:
215
+ node_to_subgraph_id[node] = subgraph_id
216
+
217
+ # `keep_original_order` is important!
218
+ # otherwise pytorch might reorder the nodes and
219
+ # the semantics of the graph will change when we
220
+ # have mutations in the graph
221
+ split_gm = torch.fx.passes.split_module.split_module(
222
graph,
177
- example_inputs,
178
- additional_inductor_config: Optional[Dict] = None) -> Callable:
179
-
180
- context = get_compile_context()
181
- context = copy.deepcopy(context) if context is not None else []
182
- sizes_to_specialize: List[int] = context
223
+ None,
224
+ lambda node: node_to_subgraph_id[node],
225
+ keep_original_order=True)
226
184
- # flags for all the seen shapes, whether we need to specialize
185
- runtime_shapes_to_compile_flags: Dict[Tuple[int, ...], bool] = {}
227
+ outputs = []
228
187
- # if we need to specialize, the compiled graph for that shape
188
- runtime_shapes_to_compiled_graph: Dict[Tuple[int, ...], Callable] = {}
229
+ # sort the names to make sure the order is deterministic
230
+ names = [name for (name, module) in split_gm.named_modules()]
231
+ names.sort()
232
190
- # this is the first compilation, we will compile a graph with
191
- # dynamic shape, as the caller will mark first dimension as dynamic
192
- logger.info("Compiling a graph for general shapes")
193
- graph_for_symbolic_shape = wrap_inductor(graph, example_inputs,
194
- additional_inductor_config)
233
+ for name in names:
234
+ if "." in name or name == "":
235
+ # recursive child module or the root module
236
+ continue
237
196
- # TODO: Dynamo does not pass all dynamic shapes.
197
- # Need to investigate why. It works now because all the dynamic
198
- # shapes have the same value, and either of them can be used.
199
- sym_shape_indices = [
200
- i for i, x in enumerate(example_inputs) if isinstance(x, torch.SymInt)
201
- ]
238
+ module = getattr(split_gm, name)
239
203
- first_run = True
240
+ graph_id = int(name.replace("submod_", ""))
241
+ outputs.append(SplitItem(name, graph_id in split_op_graphs, module))
242
205
- # this is the function we return to Dynamo to run finally
206
- def compiled_graph_wrapper(*args):
243
+ return split_gm, outputs
244
208
- runtime_shapes: Tuple[int,
209
- ...] = tuple(args[i] for i in sym_shape_indices)
245
211
- nonlocal first_run
212
- nonlocal runtime_shapes_to_compile_flags
213
- nonlocal runtime_shapes_to_compiled_graph
246
+class VllmBackend:
247
+ """The compilation backend for `torch.compile` with VLLM.
248
+ It is used for compilation level of `CompilationLevel.PIECEWISE`,
249
+ where we customize the compilation.
250
215
- if first_run:
216
- # the first compilation is for profiling, we directly run it
217
- first_run = False
218
- return graph_for_symbolic_shape(*args)
219
-
220
- if runtime_shapes not in runtime_shapes_to_compile_flags:
221
- # we haven't seen this shape before
222
- # query if we need to specialize for this shape
223
- # we only specialize for the first dimension.
224
- # TODO: investigate if any model needs to specialize
225
- # beyond the first dimension
226
- runtime_shapes_to_compile_flags[runtime_shapes] = runtime_shapes[
227
- 0] in sizes_to_specialize
228
-
229
- if not runtime_shapes_to_compile_flags[runtime_shapes]:
230
- # we don't need to specialize for this shape
231
- return graph_for_symbolic_shape(*args)
251
+ The major work of this backend is to split the graph into
252
+ piecewise graphs, and pass them to the piecewise backend.
253
+ """
254
233
- if runtime_shapes not in runtime_shapes_to_compiled_graph:
234
- # we need to specialize for this shape, and we haven't compiled
235
- # compile the graph for this shape
236
- logger.info("Compiling a graph for shapes %s", runtime_shapes)
237
- runtime_shapes_to_compiled_graph[runtime_shapes] = wrap_inductor(
238
- graph, args, additional_inductor_config)
255
+ compilation_configs: CompilationConfig
256
+ graph_pool: Any
257
+ _called: bool = False
258
+ # the graph we compiled
259
+ graph: fx.GraphModule
260
+ # the stiching graph module for all the piecewise graphs
261
+ split_gm: fx.GraphModule
262
+ piecewise_graphs: List[SplitItem]
263
+ returned_callable: Callable
264
+
265
+ def __init__(self, ):
266
+ # every instance of VllmBackend has its own graph pool
267
+ self.graph_pool = torch.cuda.graph_pool_handle()
268
+
269
+ # `torch.compile` is JIT compiled, so we don't need to
270
+ # do anything here
271
+
272
+ def __call__(self, graph: fx.GraphModule, example_inputs) -> Callable:
273
+
274
+ compilation_counter.num_graphs_seen += 1
275
+
276
+ # we control the compilation process, each instance can only be
277
+ # called once
278
+ assert not self._called, "VllmBackend can only be called once"
279
+
280
+ self.graph = graph
281
+ # config is read now, because only here can
282
+ # we get the sizes to capture for cudagraph
283
+ # from compilation context
284
+ self.compilation_configs = CompilationConfig.select_and_init_config()
285
+
286
+ self.split_gm, self.piecewise_graphs = split_graph(
287
+ graph, self.compilation_configs.non_cudagraph_ops)
288
+
289
+ returned_callable: Callable # type: ignore
290
+
291
+ if len(self.piecewise_graphs) == 0:
292
+ compilation_counter.num_piecewise_graphs_seen += 1
293
+ compilation_counter.num_piecewise_capturable_graphs_seen += 1
294
+ returned_callable = PiecewiseBackend(graph,
295
+ self.compilation_configs,
296
+ self.graph_pool,
297
+ is_first_graph=True)
298
+ else:
299
+ from torch._dynamo.utils import lazy_format_graph_code
300
+ logger.debug(
301
+ "%s", lazy_format_graph_code("stiching module", self.split_gm))
302
+
303
+ is_first_graph = True
304
+
305
+ for item in self.piecewise_graphs:
306
+ compilation_counter.num_piecewise_graphs_seen += 1
307
+ compilation_counter.num_piecewise_capturable_graphs_seen += not item.is_splitting_graph # noqa
308
+ if not item.is_splitting_graph:
309
+ # cannot setattr to a module, so we need to set
310
+ # the attribute in the __dict__
311
+ self.split_gm.__dict__[
312
+ item.submod_name] = PiecewiseBackend(
313
+ item.graph, self.compilation_configs,
314
+ self.graph_pool, is_first_graph)
315
+ is_first_graph = False
316
+ returned_callable = self.split_gm
317
+
318
+ self.returned_callable = returned_callable
319
+ # trigger the first compilation
320
+ # code borrowed from https://github.com/pytorch/pytorch/blob/4e3e08b71171fa34172b2362ff668553fac75f27/torch/_dynamo/backends/distributed.py#L206 # noqa
321
+ # to turn the inputs into fake tensors
322
+ import torch._guards
323
+ from torch._guards import detect_fake_mode
324
+ fake_mode = detect_fake_mode(example_inputs)
325
+ fake_args = []
326
+ for arg in example_inputs:
327
+ if isinstance(arg, torch.Tensor) and not isinstance(
328
+ arg, torch._subclasses.FakeTensor):
329
+ fake_args.append(
330
+ torch._dynamo.utils.to_fake_tensor(arg, fake_mode))
331
+ else:
332
+ fake_args.append(arg)
333
+ self.returned_callable(*fake_args)
334
+
335
+ self._called = True
336
+
337
+ return self.returned_callable
338
+
339
+
340
+@dataclasses.dataclass
341
+class ConcreteSizeEntry:
342
+ runtime_shape: int
343
+ need_to_compile: bool # the size is in compile_sizes
344
+ use_cudagraph: bool # the size is in capture_sizes
345
+
346
+ compiled: bool = False
347
+ runnable: Callable = None # type: ignore
348
+ num_finished_warmup: int = 0
349
+ cudagraph: Optional[torch.cuda.CUDAGraph] = None
350
+ output: Optional[Any] = None
351
+
352
+
353
+class PiecewiseBackend:
354
+
355
+ def __init__(self,
356
+ graph: fx.GraphModule,
357
+ compilation_configs: CompilationConfig,
358
+ graph_pool: Any,
359
+ is_first_graph: bool = False):
360
+ """
361
+ The backend for piecewise compilation.
362
+ It mainly handles the compilation and cudagraph capturing.
363
+
364
+ We will compile `self.graph` once for the general shape,
365
+ and then compile for different shapes specified in
366
+ `compilation_configs.compile_sizes`.
367
+
368
+ Independently, we will capture cudagraph for different shapes.
369
+
370
+ If a shape needs both compilation and cudagraph, we will
371
+ compile it first, and then capture cudagraph.
372
+ """
373
+ self.graph = graph
374
+ self.compilation_configs = compilation_configs
375
+ self.graph_pool = graph_pool
376
+ self.is_first_graph = is_first_graph
377
+
378
+ self.compile_sizes: Set[int] = set(
379
+ self.compilation_configs.compile_sizes)
380
+ self.capture_sizes: Set[int] = set(
381
+ self.compilation_configs.capture_sizes
382
+ ) if self.compilation_configs.use_cudagraph else set()
383
+
384
+ self.compile_finished = False
385
+ self.first_run_finished = False
386
+
387
+ self.compiled_graph_for_general_shape: Callable = None # type: ignore
388
+
389
+ self.sym_shape_indices: List[int] = []
390
+
391
+ # the entries for different shapes that we need to either
392
+ # compile or capture cudagraph
393
+ self.concrete_size_entries: Dict[int, ConcreteSizeEntry] = {}
394
+ for shape in self.compile_sizes.union(self.capture_sizes):
395
+ self.concrete_size_entries[shape] = ConcreteSizeEntry(
396
+ runtime_shape=shape,
397
+ need_to_compile=shape in self.compile_sizes,
398
+ use_cudagraph=shape in self.capture_sizes,
399
+ )
400
+
401
+ def __call__(self, *args) -> Any:
402
+
403
+ if not self.compile_finished:
404
+ self.compile_finished = True
405
+
406
+ # this is the first compilation, we will compile a graph with
407
+ # dynamic shape, as the caller will mark first dimension as dynamic
408
+
409
+ self.sym_shape_indices = [
410
+ i for i, x in enumerate(args) if isinstance(x, torch.SymInt)
411
+ ]
412
+
413
+ self.compiled_graph_for_general_shape = wrap_inductor(
414
+ self.graph,
415
+ args,
416
+ self.compilation_configs.inductor_compile_config,
417
+ runtime_shape=None,
418
+ do_logging=self.is_first_graph,
419
+ use_inductor=self.compilation_configs.use_inductor)
420
+
421
+ return self.graph(*args)
422
+
423
+ if not self.first_run_finished:
424
+ self.first_run_finished = True
425
+ return self.compiled_graph_for_general_shape(*args)
426
+
427
+ runtime_shape = args[self.sym_shape_indices[0]]
428
+ if runtime_shape not in self.concrete_size_entries:
429
+ # we don't need to do anything for this shape
430
+ return self.compiled_graph_for_general_shape(*args)
431
+
432
+ entry = self.concrete_size_entries[runtime_shape]
433
240
- return runtime_shapes_to_compiled_graph[runtime_shapes](*args)
434
+ if entry.runnable is None:
435
+ entry.runnable = self.compiled_graph_for_general_shape
436
242
- return compiled_graph_wrapper
437
+ if entry.need_to_compile and not entry.compiled:
438
+ entry.compiled = True
439
+ # args are real arguments
440
+ entry.runnable = wrap_inductor(
441
+ self.graph,
442
+ args,
443
+ self.compilation_configs.inductor_compile_config,
444
+ runtime_shape=runtime_shape,
445
+ do_logging=self.is_first_graph,
446
+ use_inductor=self.compilation_configs.use_inductor)
447
+
448
+ if not entry.use_cudagraph:
449
+ return entry.runnable(*args)
450
+
451
+ if entry.cudagraph is None:
452
+ if entry.num_finished_warmup < self.compilation_configs.cudagraph_num_of_warmups: # noqa
453
+ entry.num_finished_warmup += 1
454
+ if self.is_first_graph:
455
+ logger.debug(
456
+ "Warming up %s/%s for shape %s",
457
+ entry.num_finished_warmup,
458
+ self.compilation_configs.cudagraph_num_of_warmups,
459
+ runtime_shape)
460
+ return entry.runnable(*args)
461
+
462
+ if self.is_first_graph:
463
+ logger.info("Capturing a cudagraph for shape %s",
464
+ runtime_shape)
465
+
466
+ cudagraph = torch.cuda.CUDAGraph()
467
+ with torch.cuda.graph(cudagraph, pool=self.graph_pool):
468
+ entry.output = weak_ref_tensors(entry.runnable(*args))
469
+
470
+ compilation_counter.num_cudagraph_caputured += 1
471
+
472
+ entry.cudagraph = cudagraph
473
+ return entry.output
474
+
475
+ entry.cudagraph.replay()
476
+ return entry.output
477
478
479
def select_default_backend(level: int) -> Union[str, Callable]:
480
if level in [CompilationLevel.DYNAMO_AS_IS, CompilationLevel.DYNAMO_ONCE]:
481
backend_str = "eager"
482
return backend_str
249
- assert level in [
250
- CompilationLevel.INDUCTOR, CompilationLevel.INDUCTOR_MAX_AUTOTUNE
251
- ], f"Invalid level {level}"
252
-
253
- from vllm.compilation.backends import vllm_backend
254
- from vllm.plugins import get_inductor_additional_configs
255
- additional_configs = get_inductor_additional_configs()
256
-
257
- if level == CompilationLevel.INDUCTOR_MAX_AUTOTUNE:
258
- if "max_autotune" in additional_configs and not additional_configs[
259
- "max_autotune"]:
260
- logger.warning(
261
- "max_autotune is disabled, but is overridden by level %s",
262
- CompilationLevel.INDUCTOR_MAX_AUTOTUNE)
263
- additional_configs['max_autotune'] = True
264
-
265
- from functools import partial
266
- backend = partial(vllm_backend,
267
- additional_inductor_config=additional_configs)
268
-
269
- return backend
483
+ assert level == CompilationLevel.PIECEWISE
484
+
485
+ return VllmBackend()
vllm/compilation/config.py
new
+154
@@ -0,0 +1,154 @@
1
+import copy
2
+from typing import Any, Dict, List, Optional
3
+
4
+from pydantic import BaseModel, Field, PrivateAttr
5
+
6
+import vllm.envs as envs
7
+from vllm.logger import init_logger
8
+
9
+from .compile_context import get_compile_context
10
+
11
+logger = init_logger(__name__)
12
+
13
+
14
+class CompilationConfig(BaseModel):
15
+ """
16
+ Configuration for compilation.
17
+ It has two parts:
18
+ - CudaGraph capture:
19
+ - use_cudagraph: whether to use cudagraph inside compilation.
20
+ - False: cudagraph inside compilation is not used.
21
+ - True: cudagraph inside compilation is used. It requires
22
+ that all input buffers have fixed addresses.
23
+ Note that this is orthogonal to the cudagraph capture out
24
+ side of compilation.
25
+ TODO: move outside cudagraph logic into compilation.
26
+ torch.compile will handle cudagraph capture logic in the future.
27
+ - cudagraph_capture_sizes: sizes to capture cudagraph.
28
+ - None: capture sizes are inferred from compilation context.
29
+ - List[int]: capture sizes are specified.
30
+ - cudagraph_num_of_warmups: number of warmup runs for cudagraph.
31
+ It means the first several runs will be treated as warmup runs.
32
+ Only after that, the execution will be recorded, and the recorded
33
+ cudagraph will be used for subsequent runs.
34
+ - Inductor compilation:
35
+ - use_inductor: whether to use inductor compilation.
36
+ - False: inductor compilation is not used. graph runs in eager.
37
+ - True: inductor compilation is used. one graph for symbolic shape
38
+ is compiled. In addition, compile for different sizes specified
39
+ in inductor_compile_sizes, using configurations
40
+ in inductor_compile_config.
41
+ - inductor_compile_sizes: sizes to compile for inductor.
42
+ - inductor_specialize_for_cudagraph_no_more_than: an optional integer
43
+ to specialize inductor for cudagraph sizes no more than the
44
+ specified size. It is useful when we want to specialize inductor
45
+ with a subset of cudagraph sizes.
46
+ - inductor_compile_config: additional configurations for inductor.
47
+ - None: use default configurations.
48
+ - inductor_passes: additional passes for inductor. It is a dictionary
49
+ from pass name to pass function qualified name. We use function
50
+ name because the config uses json format. If we pass the config
51
+ from Python, functions can also be passed directly via Python object
52
+ constructor, e.g. `CompilationConfig(inductor_passes={"a": func})`
53
+
54
+ Why we have different sizes for cudagraph and inductor:
55
+ - cudagraph: a cudagraph captured for a specific size can only be used
56
+ for the same size. We need to capture all the sizes we want to use.
57
+ - inductor: a graph compiled by inductor for a general shape can be used
58
+ for different sizes. Inductor can also compile for specific sizes,
59
+ where it can have more information to optimize the graph with fully
60
+ static shapes. However, we find the general shape compilation is
61
+ sufficient for most cases. It might be beneficial to compile for
62
+ certain small batchsizes, where inductor is good at optimizing.
63
+ """
64
+ use_inductor: bool = True
65
+ inductor_specialize_for_cudagraph_no_more_than: Optional[int] = None
66
+ inductor_compile_sizes: Optional[List[int]] = Field(default_factory=dict)
67
+ inductor_compile_config: Dict = Field(default_factory=dict)
68
+ inductor_passes: Dict[str, str] = Field(default_factory=dict)
69
+
70
+ use_cudagraph: bool = False
71
+ non_cudagraph_ops: List[str] = Field(default_factory=list)
72
+ cudagraph_num_of_warmups: int = 0
73
+ cudagraph_capture_sizes: Optional[List[int]] = None
74
+
75
+ # not configurable, computed after init
76
+ compile_sizes: List[int] = PrivateAttr
77
+ capture_sizes: List[int] = PrivateAttr
78
+
79
+ def model_post_init(self, __context: Any) -> None:
80
+ for k, v in self.inductor_passes.items():
81
+ if not isinstance(v, str):
82
+ assert callable(v), (
83
+ f"pass {k} should be a function or a qualified name")
84
+ self.inductor_passes[k] = v
85
+ continue
86
+
87
+ # resolve function from qualified name
88
+ names = v.split(".")
89
+ module = ".".join(names[:-1])
90
+ func_name = names[-1]
91
+ func = __import__(module).__dict__[func_name]
92
+ self.inductor_compile_config[k] = func
93
+
94
+ from vllm.compilation.backends import fix_functionalization
95
+ from vllm.utils import combine_fx_passes
96
+ if "post_grad_custom_post_pass" in self.inductor_compile_config:
97
+ self.inductor_compile_config[
98
+ "post_grad_custom_post_pass"] = combine_fx_passes(
99
+ fix_functionalization,
100
+ self.inductor_compile_config["post_grad_custom_post_pass"],
101
+ )
102
+ else:
103
+ self.inductor_compile_config[
104
+ "post_grad_custom_post_pass"] = fix_functionalization
105
+
106
+ def init_during_runtime(self):
107
+ """To complete the initialization of config,
108
+ we need to know the compile context, which is only available
109
+ during the first run of the model.
110
+ """
111
+ context = get_compile_context()
112
+ context = copy.deepcopy(context) if context is not None else []
113
+ sizes_to_specialize: List[int] = context
114
+ if self.cudagraph_capture_sizes is None:
115
+ self.capture_sizes = sizes_to_specialize
116
+ else:
117
+ self.capture_sizes = self.cudagraph_capture_sizes
118
+ logger.info(("cudagraph sizes specified by model runner"
119
+ " %s is overridden by config %s"),
120
+ sizes_to_specialize, self.cudagraph_capture_sizes)
121
+ if self.inductor_specialize_for_cudagraph_no_more_than is not None:
122
+ assert self.inductor_compile_sizes is None, (
123
+ "inductor_compile_sizes should be None when "
124
+ "inductor_specialize_for_cudagraph_no_more_than is not None")
125
+ self.compile_sizes = [
126
+ x for x in self.capture_sizes
127
+ if x <= self.inductor_specialize_for_cudagraph_no_more_than
128
+ ]
129
+ else:
130
+ assert self.inductor_compile_sizes is not None, (
131
+ "inductor_compile_sizes should not be None when "
132
+ "inductor_specialize_for_cudagraph_no_more_than is None")
133
+ self.compile_sizes = self.inductor_compile_sizes
134
+
135
+ @staticmethod
136
+ def select_and_init_config() -> "CompilationConfig":
137
+ """The order of selecting config is:
138
+ 1. Use the config specified in environment variable.
139
+ 2. Use the config specified in plugins.
140
+ 3. Use the default config.
141
+ """
142
+ config_path = envs.VLLM_TORCH_COMPILE_CONFIG
143
+ if config_path is not None:
144
+ with open(config_path) as json_file:
145
+ config = CompilationConfig.model_validate_json(
146
+ json_file.read())
147
+ else:
148
+ from vllm.plugins import get_compilation_config
149
+ predefined_config = get_compilation_config()
150
+ config = predefined_config if predefined_config is not None else (
151
+ CompilationConfig())
152
+
153
+ config.init_during_runtime()
154
+ return config
vllm/compilation/counter.py
new
+30
@@ -0,0 +1,30 @@
1
+import copy
2
+import dataclasses
3
+from contextlib import contextmanager
4
+
5
+
6
+@dataclasses.dataclass
7
+class CompilationCounter:
8
+ num_graphs_seen: int = 0
9
+ # including the splitting ops
10
+ num_piecewise_graphs_seen: int = 0
11
+ # not including the splitting ops
12
+ num_piecewise_capturable_graphs_seen: int = 0
13
+ num_inductor_compilations: int = 0
14
+ num_cudagraph_caputured: int = 0
15
+
16
+ def clone(self) -> "CompilationCounter":
17
+ return copy.deepcopy(self)
18
+
19
+ @contextmanager
20
+ def expect(self, **kwargs):
21
+ old = self.clone()
22
+ yield
23
+ for k, v in kwargs.items():
24
+ assert getattr(self, k) - getattr(old, k) == v, (
25
+ f"{k} not as expected, before it is {getattr(old, k)}"
26
+ f", after it is {getattr(self, k)}, "
27
+ f"expected diff is {v}")
28
+
29
+
30
+compilation_counter = CompilationCounter()
vllm/compilation/decorators.py
+9
-1
@@ -121,7 +121,10 @@ def _support_torch_compile(cls: type,
121
# take care of method resolution order
122
# make sure super().__init__ is called on the base class
123
# other than TorchCompileWrapperWithCustomDispatcher
124
- cls.__bases__ = cls.__bases__ + (TorchCompileWrapperWithCustomDispatcher, )
124
+ if TorchCompileWrapperWithCustomDispatcher not in cls.__bases__:
125
+ # support decorating multiple times
126
+ cls.__bases__ = cls.__bases__ + (
127
+ TorchCompileWrapperWithCustomDispatcher, )
128
129
old_init = cls.__init__ # type: ignore
130
@@ -160,6 +163,11 @@ def _support_torch_compile(cls: type,
163
# compiled function and let torch.compile handle the dispatching,
164
# with the overhead of guard evaluation and recompilation.
165
if len(self.compiled_codes) < 1 or not self.use_custom_dispatcher:
166
+ # it seems Dynamo reuse the compilation across instances,
167
+ # while we need to make sure the compiled code is not reused.
168
+ # we need to control all the compilation of the model.
169
+ torch._dynamo.eval_frame.remove_from_cache(
170
+ self.original_code_object)
171
return self.compiled_callable(*args, **kwargs)
172
173
# usually, capturing the model once is enough, and then we can
vllm/compilation/levels.py
+1
-2
@@ -5,5 +5,4 @@ class CompilationLevel:
5
NO_COMPILATION = 0
6
DYNAMO_AS_IS = 1
7
DYNAMO_ONCE = 2
8
- INDUCTOR = 3
9
- INDUCTOR_MAX_AUTOTUNE = 4
8
+ PIECEWISE = 3
vllm/envs.py
+5
@@ -209,6 +209,11 @@ environment_variables: Dict[str, Callable[[], Any]] = {
209
os.environ.get("VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE", "1") != "0"),
210
"VLLM_TORCH_COMPILE_LEVEL":
211
lambda: int(os.environ.get("VLLM_TORCH_COMPILE_LEVEL", "0")),
212
+
213
+ # Path to the config file for torch compile
214
+ "VLLM_TORCH_COMPILE_CONFIG":
215
+ lambda: os.environ.get("VLLM_TORCH_COMPILE_CONFIG", None),
216
+
217
# Fine-grained control over which custom ops to enable/disable.
218
# Use 'all' to enable all, 'none' to disable all.
219
# Also specify a list of custom op names to enable (prefixed with a '+'),
vllm/model_executor/custom_op.py
+2
-2
@@ -100,7 +100,7 @@ class CustomOp(nn.Module):
100
101
return (CustomOp.default_on() or enabled) and not disabled
102
103
- # On by default if VLLM_TORCH_COMPILE_LEVEL < CompilationLevel.INDUCTOR
103
+ # On by default if VLLM_TORCH_COMPILE_LEVEL < CompilationLevel.PIECEWISE
104
# Specifying 'all' or 'none' in VLLM_CUSTOM_OPS takes precedence.
105
@staticmethod
106
@lru_cache()
@@ -108,7 +108,7 @@ class CustomOp(nn.Module):
108
count_none = envs.VLLM_CUSTOM_OPS.count("none")
109
count_all = envs.VLLM_CUSTOM_OPS.count("all")
110
assert count_none + count_all <= 1, "Can only specify 'none' or 'all'"
111
- return envs.VLLM_TORCH_COMPILE_LEVEL < CompilationLevel.INDUCTOR and \
111
+ return envs.VLLM_TORCH_COMPILE_LEVEL < CompilationLevel.PIECEWISE and \
112
not count_none > 0 or count_all > 0
113
114
# Dictionary of all custom ops (classes, indexed by registered name).
vllm/platforms/tpu.py
+1
-1
@@ -11,7 +11,7 @@ from .interface import Platform, PlatformEnum
11
if "VLLM_TORCH_COMPILE_LEVEL" not in os.environ:
12
os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(CompilationLevel.DYNAMO_ONCE)
13
14
-assert envs.VLLM_TORCH_COMPILE_LEVEL < CompilationLevel.INDUCTOR,\
14
+assert envs.VLLM_TORCH_COMPILE_LEVEL < CompilationLevel.PIECEWISE,\
15
"TPU does not support Inductor."
16
17
set_torch_compile_backend("openxla")
vllm/plugins/__init__.py
+8
-7
@@ -1,7 +1,8 @@
1
import logging
2
-from typing import Callable, Dict, Optional, Union
2
+from typing import Callable, Optional, Union
3
4
import vllm.envs as envs
5
+from vllm.compilation.config import CompilationConfig
6
7
logger = logging.getLogger(__name__)
8
@@ -44,13 +45,13 @@ def get_torch_compile_backend() -> Optional[Union[Callable, str]]:
45
return _torch_compile_backend
46
47
47
-_inductor_additional_configs: Dict = {}
48
+_compilation_config: Optional[CompilationConfig] = None
49
50
50
-def set_inductor_additional_configs(configs: Dict):
51
- global _inductor_additional_configs
52
- _inductor_additional_configs = configs
51
+def set_compilation_config(config: Optional[CompilationConfig]):
52
+ global _compilation_config
53
+ _compilation_config = config
54
55
55
-def get_inductor_additional_configs() -> Dict:
56
- return _inductor_additional_configs
56
+def get_compilation_config() -> Optional[CompilationConfig]:
57
+ return _compilation_config
vllm/utils.py
+25
@@ -1479,6 +1479,15 @@ class LazyDict(Mapping, Generic[T]):
1479
return len(self._factory)
1480
1481
1482
+def combine_fx_passes(passes: List[Callable]) -> Callable:
1483
+
1484
+ def combined_fx(graph) -> None:
1485
+ for fx in passes:
1486
+ fx(graph)
1487
+
1488
+ return combined_fx
1489
+
1490
+
1491
def weak_ref_tensor(tensor: torch.Tensor) -> torch.Tensor:
1492
"""
1493
Create a weak reference to a tensor.
@@ -1486,3 +1495,19 @@ def weak_ref_tensor(tensor: torch.Tensor) -> torch.Tensor:
1495
but will not keep the original tensor alive.
1496
"""
1497
return torch.ops._C.weak_ref_tensor(tensor)
1498
+
1499
+
1500
+def weak_ref_tensors(
1501
+ tensors: Union[torch.Tensor, List[torch.Tensor], Tuple[torch.Tensor]]
1502
+) -> Union[torch.Tensor, List[torch.Tensor], Tuple[torch.Tensor]]:
1503
+ """
1504
+ Convenience function to create weak references to tensors,
1505
+ for single tensor, list of tensors or tuple of tensors.
1506
+ """
1507
+ if isinstance(tensors, torch.Tensor):
1508
+ return weak_ref_tensor(tensors)
1509
+ if isinstance(tensors, list):
1510
+ return [weak_ref_tensor(t) for t in tensors]
1511
+ if isinstance(tensors, tuple):
1512
+ return tuple(weak_ref_tensor(t) for t in tensors)
1513
+ raise ValueError("Invalid type for tensors")