@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ff5ed6e1bc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[torch.compile] rework compile control with piecewise cudagraph (#9715)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Oct 29, 2024 at 23:03 UTC ff5ed6e1bcbd112a26f8eb43b6bfdbc5ec73726e
17 files changed +983 -106
.buildkite/test-pipeline.yaml
+3
@@ -229,6 +229,9 @@ steps:
229 - tests/compile
230 commands:
231 - pytest -v -s compile/test_basic_correctness.py
232 + # these tests need to be separated, cannot combine
233 + - pytest -v -s compile/piecewise/test_simple.py
234 + - pytest -v -s compile/piecewise/test_toy_llama.py
235
236 - label: "PyTorch Fullgraph Test" # 18min
237 source_file_dependencies:
tests/compile/piecewise/__init__.py
tests/compile/piecewise/piecewise_compilation_config.json new
+4
@@ -0,0 +1,4 @@
1 +{
2 + "use_cudagraph": true,
3 + "non_cudagraph_ops": ["silly.attention"]
4 +}
\ No newline at end of file
tests/compile/piecewise/test_simple.py new
+96
@@ -0,0 +1,96 @@
1 +"""
2 +Test the piecewise compilation with a simple model so that we
3 +can exactly calculate the expected output and side effects.
4 +"""
5 +import os
6 +
7 +import torch
8 +from torch import nn
9 +
10 +from vllm.compilation.compile_context import set_compile_context
11 +from vllm.compilation.counter import compilation_counter
12 +from vllm.compilation.decorators import support_torch_compile
13 +from vllm.compilation.levels import CompilationLevel
14 +
15 +os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(CompilationLevel.PIECEWISE)
16 +
17 +global_counter = 0
18 +
19 +
20 +@torch.library.custom_op("silly::attention", mutates_args=["out"])
21 +def silly_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,
22 + out: torch.Tensor) -> None:
23 + global global_counter
24 + global_counter += 1
25 + print(f"{global_counter=}")
26 + out.copy_(q)
27 + out[0] += 1
28 +
29 +
30 +@silly_attention.register_fake
31 +def _(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,
32 + out: torch.Tensor) -> None:
33 + return
34 +
35 +
36 +@support_torch_compile
37 +class SillyModel(nn.Module):
38 +
39 + def __init__(self) -> None:
40 + super().__init__()
41 +
42 + def forward(self, x: torch.Tensor) -> torch.Tensor:
43 + """
44 + Overall effect:
45 + x += 1
46 + x[0] += 2
47 + global_counter += 2
48 + """
49 + x = x + 1
50 + x = x + 2
51 + out = torch.empty_like(x)
52 + torch.ops.silly.attention(x, x, x, out)
53 + x = out
54 + x = x - 2
55 + x = x - 1
56 + out = torch.empty_like(x)
57 + torch.ops.silly.attention(x, x, x, out)
58 + x = out
59 + x = x + 1
60 + return x
61 +
62 +
63 +def test_simple_piecewise_compile():
64 +
65 + model = SillyModel()
66 +
67 + directory = os.path.dirname(__file__)
68 + config = os.path.join(directory, "piecewise_compilation_config.json")
69 + os.environ["VLLM_TORCH_COMPILE_CONFIG"] = config
70 +
71 + input_buffer = torch.randn(100).cuda()
72 +
73 + with compilation_counter.expect(
74 + num_graphs_seen=1, # one graph for the model
75 + num_piecewise_graphs_seen=5, # 2 * num_layers + 1
76 + num_piecewise_capturable_graphs_seen=3, # 1 + num_layers
77 + num_inductor_compilations=3, # num_piecewise_capturable_graphs_seen
78 + num_cudagraph_caputured=
79 + 6, # num_cudagraph_sizes * num_piecewise_capturable_graphs_seen
80 + ):
81 +
82 + with set_compile_context([1, 2]):
83 + model(input_buffer)
84 +
85 + model(input_buffer[:2])
86 + model(input_buffer[:1])
87 +
88 + input_buffer[:2].zero_()
89 + global global_counter
90 + global_counter = 0
91 + output = model(input_buffer[:2])
92 + assert global_counter == 2
93 + assert torch.allclose(output.cpu(), torch.tensor([3., 1.]))
94 +
95 + # clean up to avoid side effects for other tests
96 + del os.environ["VLLM_TORCH_COMPILE_CONFIG"]
tests/compile/piecewise/test_toy_llama.py new
+334
@@ -0,0 +1,334 @@
1 +"""
2 +Test the piecewise compilation with a simple model, comparing the output
3 +with and without the piecewise compilation.
4 +"""
5 +import os
6 +from dataclasses import dataclass
7 +from typing import Optional, Tuple
8 +
9 +import torch
10 +from torch import nn
11 +
12 +from vllm.compilation.compile_context import set_compile_context
13 +from vllm.compilation.config import CompilationConfig
14 +from vllm.compilation.counter import compilation_counter
15 +from vllm.compilation.decorators import support_torch_compile
16 +from vllm.compilation.levels import CompilationLevel
17 +from vllm.plugins import set_compilation_config
18 +
19 +
20 +@torch.library.custom_op("silly::attention", mutates_args=["out"])
21 +def silly_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,
22 + out: torch.Tensor) -> None:
23 + out.copy_(q)
24 + out += k
25 + out += v
26 +
27 +
28 +@silly_attention.register_fake
29 +def _(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,
30 + out: torch.Tensor) -> None:
31 + return
32 +
33 +
34 +@dataclass
35 +class LlamaConfig:
36 + hidden_size: int = 128
37 + mlp_size: int = 256
38 + vocab_size: int = 128
39 + num_layers: int = 2
40 +
41 +
42 +class LlamaMLP(nn.Module):
43 +
44 + def __init__(self, config: LlamaConfig) -> None:
45 + super().__init__()
46 + self.gate_up_projection = nn.Linear(
47 + in_features=config.hidden_size,
48 + out_features=config.mlp_size * 2,
49 + bias=False,
50 + )
51 + self.down_projection = nn.Linear(
52 + in_features=config.mlp_size,
53 + out_features=config.hidden_size,
54 + bias=False,
55 + )
56 +
57 + self.gate_up_projection.weight.data.fill_(0.0)
58 + self.down_projection.weight.data.fill_(0.0)
59 +
60 + def forward(self, x):
61 + x = self.gate_up_projection(x)
62 + x = x[:, :x.size(1) // 2] * torch.nn.functional.relu(
63 + x[:, x.size(1) // 2:])
64 + x = self.down_projection(x)
65 + return x
66 +
67 +
68 +class LlamaAttention(nn.Module):
69 +
70 + def __init__(self, config: LlamaConfig) -> None:
71 + super().__init__()
72 + self.qkv_projection = nn.Linear(
73 + in_features=config.hidden_size,
74 + out_features=config.hidden_size * 3,
75 + )
76 +
77 + self.output_projection = nn.Linear(
78 + in_features=config.hidden_size,
79 + out_features=config.hidden_size,
80 + )
81 +
82 + self.qkv_projection.weight.data.fill_(0.0)
83 + self.output_projection.weight.data.fill_(0.0)
84 +
85 + def forward(
86 + self,
87 + positions: torch.Tensor,
88 + hidden_states: torch.Tensor,
89 + ) -> torch.Tensor:
90 + qkv = self.qkv_projection(hidden_states)
91 + hidden_size = qkv.size(-1) // 3
92 + q, k, v = qkv.split([hidden_size, hidden_size, hidden_size], dim=-1)
93 +
94 + q = q + positions.unsqueeze(1)
95 + k = k + positions.unsqueeze(1)
96 +
97 + attn_output = torch.empty_like(q)
98 + torch.ops.silly.attention(q, k, v, attn_output)
99 +
100 + output = self.output_projection(attn_output)
101 + return output
102 +
103 +
104 +class LlamaDecoderLayer(nn.Module):
105 +
106 + def __init__(self, config: LlamaConfig) -> None:
107 + super().__init__()
108 + self.self_attention = LlamaAttention(config)
109 + self.mlp = LlamaMLP(config)
110 +
111 + def forward(
112 + self,
113 + positions: torch.Tensor,
114 + hidden_states: torch.Tensor,
115 + residual: Optional[torch.Tensor],
116 + ) -> Tuple[torch.Tensor, torch.Tensor]:
117 + if residual is None:
118 + residual = hidden_states
119 + hidden_states = hidden_states / 2
120 + else:
121 + hidden_states = hidden_states + residual
122 + residual = hidden_states
123 + hidden_states = hidden_states / 2
124 +
125 + hidden_states = self.self_attention(positions=positions,
126 + hidden_states=hidden_states)
127 +
128 + hidden_states = hidden_states + residual
129 + residual = hidden_states
130 + hidden_states = hidden_states / 2
131 + hidden_states = self.mlp(hidden_states)
132 +
133 + return hidden_states, residual
134 +
135 +
136 +class LlamaModel(nn.Module):
137 +
138 + def __init__(self, config: LlamaConfig) -> None:
139 + super().__init__()
140 + self.embedding_tokens = nn.Embedding(
141 + num_embeddings=config.vocab_size,
142 + embedding_dim=config.hidden_size,
143 + )
144 + self.layers = nn.ModuleList(
145 + [LlamaDecoderLayer(config) for _ in range(config.num_layers)])
146 +
147 + self.embedding_tokens.weight.data.fill_(0.0)
148 +
149 + def forward(
150 + self,
151 + input_ids: Optional[torch.Tensor],
152 + positions: torch.Tensor,
153 + ) -> torch.Tensor:
154 + hidden_states = self.embedding_tokens(input_ids)
155 + residual = None
156 + for layer in self.layers:
157 + hidden_states, residual = layer(positions, hidden_states, residual)
158 + return hidden_states
159 +
160 +
161 +@torch.inference_mode
162 +def run_model(llama_config,
163 + use_compile: bool,
164 + split_attn: bool = False) -> torch.Tensor:
165 +
166 + if use_compile:
167 + os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(
168 + CompilationLevel.PIECEWISE)
169 +
170 + if split_attn:
171 + set_compilation_config(
172 + CompilationConfig(
173 + use_cudagraph=True,
174 + non_cudagraph_ops=["silly.attention"],
175 + ))
176 + else:
177 + set_compilation_config(CompilationConfig(use_cudagraph=True, ))
178 + else:
179 + os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(
180 + CompilationLevel.NO_COMPILATION)
181 + set_compilation_config(None)
182 +
183 + cls = LlamaModel
184 + if use_compile:
185 + cls = support_torch_compile(LlamaModel)
186 + model = cls(llama_config).eval().cuda()
187 +
188 + B = 16 # max batch size
189 + input_ids = torch.randint(0, llama_config.vocab_size, (B, )).cuda()
190 + positions = torch.arange(B).cuda()
191 +
192 + with set_compile_context([1, 2]):
193 + model(input_ids, positions)
194 + model(input_ids[:2], positions[:2])
195 + model(input_ids[:1], positions[:1])
196 +
197 + input_ids[:2].zero_()
198 + output = model(input_ids[:2], positions[:2])
199 +
200 + # manual cleanup
201 + del os.environ["VLLM_TORCH_COMPILE_LEVEL"]
202 + set_compilation_config(None)
203 +
204 + return output.cpu()
205 +
206 +
207 +def test_toy_llama():
208 + # compare output with and without piecewise compilation
209 +
210 + llama_config = LlamaConfig(hidden_size=128,
211 + mlp_size=256,
212 + vocab_size=128,
213 + num_layers=2)
214 +
215 + outputs = []
216 + with compilation_counter.expect(
217 + num_graphs_seen=0,
218 + num_piecewise_graphs_seen=0,
219 + num_piecewise_capturable_graphs_seen=0,
220 + num_inductor_compilations=0,
221 + num_cudagraph_caputured=0,
222 + ):
223 + outputs.append(run_model(llama_config, use_compile=False))
224 + with compilation_counter.expect(
225 + num_graphs_seen=1, # one graph for the model
226 + num_piecewise_graphs_seen=1,
227 + num_piecewise_capturable_graphs_seen=1,
228 + num_inductor_compilations=1, # num_piecewise_capturable_graphs_seen
229 + num_cudagraph_caputured=
230 + 2, # num_cudagraph_sizes * num_piecewise_capturable_graphs_seen
231 + ):
232 + outputs.append(run_model(llama_config, use_compile=True))
233 +
234 + with compilation_counter.expect(
235 + num_graphs_seen=1, # one graph for the model
236 + num_piecewise_graphs_seen=2 * llama_config.num_layers +
237 + 1, # 2 * num_layers + 1
238 + num_piecewise_capturable_graphs_seen=1 +
239 + llama_config.num_layers, # 1 + num_layers
240 + num_inductor_compilations=1 +
241 + llama_config.num_layers, # num_piecewise_capturable_graphs_seen
242 + num_cudagraph_caputured=2 *
243 + (1 + llama_config.num_layers
244 + ), # num_cudagraph_sizes * num_piecewise_capturable_graphs_seen
245 + ):
246 + outputs.append(
247 + run_model(llama_config, use_compile=True, split_attn=True))
248 +
249 + for i in range(1, len(outputs)):
250 + assert torch.allclose(outputs[0], outputs[i])
251 +
252 +
253 +@torch.inference_mode
254 +def benchmark():
255 + os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(CompilationLevel.PIECEWISE)
256 + from triton.testing import do_bench
257 + cls = support_torch_compile(LlamaModel)
258 +
259 + # similar to llama 3.1-8B
260 + llama_config = LlamaConfig(hidden_size=4096,
261 + mlp_size=14336,
262 + vocab_size=128 * 1024,
263 + num_layers=32)
264 +
265 + # a tiny model to measure the overhead
266 + # of piecewise cudagraph
267 + llama_config = LlamaConfig(hidden_size=40,
268 + mlp_size=80,
269 + vocab_size=128,
270 + num_layers=2)
271 +
272 + cudagraph_sizes = [1, 2, 4] + [i * 8 for i in range(1, 33)]
273 +
274 + eager_time = {}
275 + full_cudagraph_time = {}
276 + piecewise_cudagraph_time = {}
277 +
278 + pool = torch.cuda.graph_pool_handle()
279 +
280 + for piecewise in [False, True]:
281 + if piecewise:
282 + set_compilation_config(
283 + CompilationConfig(
284 + use_cudagraph=True,
285 + non_cudagraph_ops=["silly.attention"],
286 + ))
287 + else:
288 + set_compilation_config(None)
289 +
290 + model = cls(llama_config).eval().cuda().to(torch.bfloat16)
291 +
292 + B = 256 # max batch size
293 + input_ids = torch.randint(0, llama_config.vocab_size, (B, )).cuda()
294 + positions = torch.arange(B).cuda().to(torch.bfloat16)
295 +
296 + graphs = {}
297 +
298 + with set_compile_context(cudagraph_sizes):
299 + model(input_ids, positions)
300 + for b in cudagraph_sizes[::-1]:
301 + if not piecewise:
302 + graph = torch.cuda.CUDAGraph()
303 + with torch.cuda.graph(graph, pool=pool):
304 + output = model(input_ids[:b], positions[:b])
305 + graphs[b] = (graph, output)
306 + else:
307 + output = model(input_ids[:b], positions[:b])
308 + graphs[b] = (model, output)
309 + for b in cudagraph_sizes:
310 + if piecewise:
311 + # noqa is for `Function definition does not bind loop variable`
312 + # it will be problematic if we save the created lambda function
313 + # and use it later, because it will look up the name `b` in the
314 + # enclosing scope, and the value of `b` will always be 256.
315 + # it is fine here, because we only use the lambda function once.
316 + runtime = do_bench(lambda: graphs[b][0] # noqa
317 + (input_ids[:b], positions[:b])) # noqa
318 + piecewise_cudagraph_time[b] = runtime
319 + else:
320 + runtime = do_bench(lambda: graphs[b][0].replay()) # noqa
321 + eager_runtime = do_bench(
322 + lambda: model(input_ids[:b], positions[:b])) # noqa
323 + full_cudagraph_time[b] = runtime
324 + eager_time[b] = eager_runtime
325 +
326 + # print in tabular format
327 + print("batch size\teager mode\tfull cudagraph\tpiecewise cudagraph")
328 + for b in cudagraph_sizes:
329 + print((f"{b}\t{eager_time[b]:.3f}\t{full_cudagraph_time[b]:.3f}"
330 + f"\t{piecewise_cudagraph_time[b]:.3f}"))
331 +
332 +
333 +if __name__ == "__main__":
334 + benchmark()
tests/compile/test_full_graph.py
+1 -1
@@ -9,7 +9,7 @@ from .utils import TEST_MODELS, check_full_graph_support
9 @pytest.mark.parametrize("model_info", TEST_MODELS)
10 @pytest.mark.parametrize(
11 "optimization_level",
12 - [CompilationLevel.DYNAMO_ONCE, CompilationLevel.INDUCTOR])
12 + [CompilationLevel.DYNAMO_ONCE, CompilationLevel.PIECEWISE])
13 @fork_new_process_for_each_test
14 def test_full_graph(model_info, optimization_level):
15 model = model_info[0]
tests/compile/utils.py
+10 -8
@@ -9,17 +9,19 @@ from vllm.platforms import current_platform
9
10 TEST_MODELS = [
11 ("facebook/opt-125m", {}),
12 - ("nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", {
13 - "dtype": torch.float16,
14 - "quantization": "compressed-tensors"
15 - }),
12 + # TODO: add fake implementation for compressed-tensors
13 + # ("nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", {
14 + # "dtype": torch.float16,
15 + # "quantization": "compressed-tensors"
16 + # }),
17 ("neuralmagic/Meta-Llama-3-8B-Instruct-FP8", {
18 "dtype": torch.float16,
19 "quantization": "fp8"
20 }),
20 - ("nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dyn-Per-Token-2048-Samples", {
21 - "quantization": "compressed-tensors"
22 - }),
21 + # TODO: add fake implementation for compressed-tensors
22 + # ("nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dyn-Per-Token-2048-Samples", {
23 + # "quantization": "compressed-tensors"
24 + # }),
25 ("meta-llama/Meta-Llama-3-8B", {}),
26 ]
27
@@ -73,7 +75,7 @@ def check_full_graph_support(model,
75 # much memory.
76 quantization = model_kwargs.get("quantization")
77 if ((quantization == "fp8" or model == "meta-llama/Meta-Llama-3-8B")
76 - and optimization_level >= CompilationLevel.INDUCTOR):
78 + and optimization_level >= CompilationLevel.PIECEWISE):
79 return
80
81 prompts = [
vllm/compilation/backends.py
+300 -84
@@ -1,13 +1,16 @@
1 import copy
2 +import dataclasses
3 import operator
3 -from typing import Callable, Dict, List, Optional, Tuple, Union
4 +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
5
6 import torch
7 import torch.fx as fx
8
9 from vllm.logger import init_logger
10 +from vllm.utils import weak_ref_tensors
11
10 -from .compile_context import get_compile_context
12 +from .config import CompilationConfig
13 +from .counter import compilation_counter
14 from .levels import CompilationLevel
15
16 logger = init_logger(__name__)
@@ -157,113 +160,326 @@ def fix_functionalization(graph: fx.Graph):
160 # print(graph.python_code(root_module="self", verbose=True).src, file=f)
161
162
160 -def wrap_inductor(graph, example_inputs, additional_inductor_config):
163 +def wrap_inductor(graph,
164 + example_inputs,
165 + additional_inductor_config,
166 + do_logging=False,
167 + runtime_shape: Optional[int] = None,
168 + use_inductor: bool = True):
169 + if not use_inductor:
170 + return graph
171 +
172 + compilation_counter.num_inductor_compilations += 1
173 +
174 + if do_logging:
175 + if runtime_shape is None:
176 + logger.info("Compiling a graph for general shape")
177 + else:
178 + logger.info("Compiling a graph for shape %s", runtime_shape)
179 +
180 from torch._inductor import config
181 current_config = config.shallow_copy_dict()
182 from torch._inductor.compile_fx import compile_fx
183
184 if additional_inductor_config is not None:
185 current_config.update(additional_inductor_config)
167 - if current_config['post_grad_custom_post_pass'] is not None:
168 - logger.warning(
169 - "post_grad_custom_post_pass is already set in the config. "
170 - "Overwriting it with the fix_functionalization")
171 - current_config['post_grad_custom_post_pass'] = fix_functionalization
186 +
187 + # inductor can inplace modify the graph, so we need to copy it
188 + # see https://github.com/pytorch/pytorch/issues/138980
189 + graph = copy.deepcopy(graph)
190 return compile_fx(graph, example_inputs, config_patches=current_config)
191
192
175 -def vllm_backend(
193 +@dataclasses.dataclass
194 +class SplitItem:
195 + submod_name: str
196 + is_splitting_graph: bool
197 + graph: fx.GraphModule
198 +
199 +
200 +def split_graph(graph: fx.GraphModule,
201 + ops: List[str]) -> Tuple[fx.GraphModule, List[SplitItem]]:
202 + # split graph by ops
203 + subgraph_id = 0
204 + node_to_subgraph_id = {}
205 + split_op_graphs = []
206 + for node in graph.graph.nodes:
207 + if node.op in ("output", "placeholder"):
208 + continue
209 + if node.op == 'call_function' and str(node.target) in ops:
210 + subgraph_id += 1
211 + node_to_subgraph_id[node] = subgraph_id
212 + split_op_graphs.append(subgraph_id)
213 + subgraph_id += 1
214 + else:
215 + node_to_subgraph_id[node] = subgraph_id
216 +
217 + # `keep_original_order` is important!
218 + # otherwise pytorch might reorder the nodes and
219 + # the semantics of the graph will change when we
220 + # have mutations in the graph
221 + split_gm = torch.fx.passes.split_module.split_module(
222 graph,
177 - example_inputs,
178 - additional_inductor_config: Optional[Dict] = None) -> Callable:
179 -
180 - context = get_compile_context()
181 - context = copy.deepcopy(context) if context is not None else []
182 - sizes_to_specialize: List[int] = context
223 + None,
224 + lambda node: node_to_subgraph_id[node],
225 + keep_original_order=True)
226
184 - # flags for all the seen shapes, whether we need to specialize
185 - runtime_shapes_to_compile_flags: Dict[Tuple[int, ...], bool] = {}
227 + outputs = []
228
187 - # if we need to specialize, the compiled graph for that shape
188 - runtime_shapes_to_compiled_graph: Dict[Tuple[int, ...], Callable] = {}
229 + # sort the names to make sure the order is deterministic
230 + names = [name for (name, module) in split_gm.named_modules()]
231 + names.sort()
232
190 - # this is the first compilation, we will compile a graph with
191 - # dynamic shape, as the caller will mark first dimension as dynamic
192 - logger.info("Compiling a graph for general shapes")
193 - graph_for_symbolic_shape = wrap_inductor(graph, example_inputs,
194 - additional_inductor_config)
233 + for name in names:
234 + if "." in name or name == "":
235 + # recursive child module or the root module
236 + continue
237
196 - # TODO: Dynamo does not pass all dynamic shapes.
197 - # Need to investigate why. It works now because all the dynamic
198 - # shapes have the same value, and either of them can be used.
199 - sym_shape_indices = [
200 - i for i, x in enumerate(example_inputs) if isinstance(x, torch.SymInt)
201 - ]
238 + module = getattr(split_gm, name)
239
203 - first_run = True
240 + graph_id = int(name.replace("submod_", ""))
241 + outputs.append(SplitItem(name, graph_id in split_op_graphs, module))
242
205 - # this is the function we return to Dynamo to run finally
206 - def compiled_graph_wrapper(*args):
243 + return split_gm, outputs
244
208 - runtime_shapes: Tuple[int,
209 - ...] = tuple(args[i] for i in sym_shape_indices)
245
211 - nonlocal first_run
212 - nonlocal runtime_shapes_to_compile_flags
213 - nonlocal runtime_shapes_to_compiled_graph
246 +class VllmBackend:
247 + """The compilation backend for `torch.compile` with VLLM.
248 + It is used for compilation level of `CompilationLevel.PIECEWISE`,
249 + where we customize the compilation.
250
215 - if first_run:
216 - # the first compilation is for profiling, we directly run it
217 - first_run = False
218 - return graph_for_symbolic_shape(*args)
219 -
220 - if runtime_shapes not in runtime_shapes_to_compile_flags:
221 - # we haven't seen this shape before
222 - # query if we need to specialize for this shape
223 - # we only specialize for the first dimension.
224 - # TODO: investigate if any model needs to specialize
225 - # beyond the first dimension
226 - runtime_shapes_to_compile_flags[runtime_shapes] = runtime_shapes[
227 - 0] in sizes_to_specialize
228 -
229 - if not runtime_shapes_to_compile_flags[runtime_shapes]:
230 - # we don't need to specialize for this shape
231 - return graph_for_symbolic_shape(*args)
251 + The major work of this backend is to split the graph into
252 + piecewise graphs, and pass them to the piecewise backend.
253 + """
254
233 - if runtime_shapes not in runtime_shapes_to_compiled_graph:
234 - # we need to specialize for this shape, and we haven't compiled
235 - # compile the graph for this shape
236 - logger.info("Compiling a graph for shapes %s", runtime_shapes)
237 - runtime_shapes_to_compiled_graph[runtime_shapes] = wrap_inductor(
238 - graph, args, additional_inductor_config)
255 + compilation_configs: CompilationConfig
256 + graph_pool: Any
257 + _called: bool = False
258 + # the graph we compiled
259 + graph: fx.GraphModule
260 + # the stiching graph module for all the piecewise graphs
261 + split_gm: fx.GraphModule
262 + piecewise_graphs: List[SplitItem]
263 + returned_callable: Callable
264 +
265 + def __init__(self, ):
266 + # every instance of VllmBackend has its own graph pool
267 + self.graph_pool = torch.cuda.graph_pool_handle()
268 +
269 + # `torch.compile` is JIT compiled, so we don't need to
270 + # do anything here
271 +
272 + def __call__(self, graph: fx.GraphModule, example_inputs) -> Callable:
273 +
274 + compilation_counter.num_graphs_seen += 1
275 +
276 + # we control the compilation process, each instance can only be
277 + # called once
278 + assert not self._called, "VllmBackend can only be called once"
279 +
280 + self.graph = graph
281 + # config is read now, because only here can
282 + # we get the sizes to capture for cudagraph
283 + # from compilation context
284 + self.compilation_configs = CompilationConfig.select_and_init_config()
285 +
286 + self.split_gm, self.piecewise_graphs = split_graph(
287 + graph, self.compilation_configs.non_cudagraph_ops)
288 +
289 + returned_callable: Callable # type: ignore
290 +
291 + if len(self.piecewise_graphs) == 0:
292 + compilation_counter.num_piecewise_graphs_seen += 1
293 + compilation_counter.num_piecewise_capturable_graphs_seen += 1
294 + returned_callable = PiecewiseBackend(graph,
295 + self.compilation_configs,
296 + self.graph_pool,
297 + is_first_graph=True)
298 + else:
299 + from torch._dynamo.utils import lazy_format_graph_code
300 + logger.debug(
301 + "%s", lazy_format_graph_code("stiching module", self.split_gm))
302 +
303 + is_first_graph = True
304 +
305 + for item in self.piecewise_graphs:
306 + compilation_counter.num_piecewise_graphs_seen += 1
307 + compilation_counter.num_piecewise_capturable_graphs_seen += not item.is_splitting_graph # noqa
308 + if not item.is_splitting_graph:
309 + # cannot setattr to a module, so we need to set
310 + # the attribute in the __dict__
311 + self.split_gm.__dict__[
312 + item.submod_name] = PiecewiseBackend(
313 + item.graph, self.compilation_configs,
314 + self.graph_pool, is_first_graph)
315 + is_first_graph = False
316 + returned_callable = self.split_gm
317 +
318 + self.returned_callable = returned_callable
319 + # trigger the first compilation
320 + # code borrowed from https://github.com/pytorch/pytorch/blob/4e3e08b71171fa34172b2362ff668553fac75f27/torch/_dynamo/backends/distributed.py#L206 # noqa
321 + # to turn the inputs into fake tensors
322 + import torch._guards
323 + from torch._guards import detect_fake_mode
324 + fake_mode = detect_fake_mode(example_inputs)
325 + fake_args = []
326 + for arg in example_inputs:
327 + if isinstance(arg, torch.Tensor) and not isinstance(
328 + arg, torch._subclasses.FakeTensor):
329 + fake_args.append(
330 + torch._dynamo.utils.to_fake_tensor(arg, fake_mode))
331 + else:
332 + fake_args.append(arg)
333 + self.returned_callable(*fake_args)
334 +
335 + self._called = True
336 +
337 + return self.returned_callable
338 +
339 +
340 +@dataclasses.dataclass
341 +class ConcreteSizeEntry:
342 + runtime_shape: int
343 + need_to_compile: bool # the size is in compile_sizes
344 + use_cudagraph: bool # the size is in capture_sizes
345 +
346 + compiled: bool = False
347 + runnable: Callable = None # type: ignore
348 + num_finished_warmup: int = 0
349 + cudagraph: Optional[torch.cuda.CUDAGraph] = None
350 + output: Optional[Any] = None
351 +
352 +
353 +class PiecewiseBackend:
354 +
355 + def __init__(self,
356 + graph: fx.GraphModule,
357 + compilation_configs: CompilationConfig,
358 + graph_pool: Any,
359 + is_first_graph: bool = False):
360 + """
361 + The backend for piecewise compilation.
362 + It mainly handles the compilation and cudagraph capturing.
363 +
364 + We will compile `self.graph` once for the general shape,
365 + and then compile for different shapes specified in
366 + `compilation_configs.compile_sizes`.
367 +
368 + Independently, we will capture cudagraph for different shapes.
369 +
370 + If a shape needs both compilation and cudagraph, we will
371 + compile it first, and then capture cudagraph.
372 + """
373 + self.graph = graph
374 + self.compilation_configs = compilation_configs
375 + self.graph_pool = graph_pool
376 + self.is_first_graph = is_first_graph
377 +
378 + self.compile_sizes: Set[int] = set(
379 + self.compilation_configs.compile_sizes)
380 + self.capture_sizes: Set[int] = set(
381 + self.compilation_configs.capture_sizes
382 + ) if self.compilation_configs.use_cudagraph else set()
383 +
384 + self.compile_finished = False
385 + self.first_run_finished = False
386 +
387 + self.compiled_graph_for_general_shape: Callable = None # type: ignore
388 +
389 + self.sym_shape_indices: List[int] = []
390 +
391 + # the entries for different shapes that we need to either
392 + # compile or capture cudagraph
393 + self.concrete_size_entries: Dict[int, ConcreteSizeEntry] = {}
394 + for shape in self.compile_sizes.union(self.capture_sizes):
395 + self.concrete_size_entries[shape] = ConcreteSizeEntry(
396 + runtime_shape=shape,
397 + need_to_compile=shape in self.compile_sizes,
398 + use_cudagraph=shape in self.capture_sizes,
399 + )
400 +
401 + def __call__(self, *args) -> Any:
402 +
403 + if not self.compile_finished:
404 + self.compile_finished = True
405 +
406 + # this is the first compilation, we will compile a graph with
407 + # dynamic shape, as the caller will mark first dimension as dynamic
408 +
409 + self.sym_shape_indices = [
410 + i for i, x in enumerate(args) if isinstance(x, torch.SymInt)
411 + ]
412 +
413 + self.compiled_graph_for_general_shape = wrap_inductor(
414 + self.graph,
415 + args,
416 + self.compilation_configs.inductor_compile_config,
417 + runtime_shape=None,
418 + do_logging=self.is_first_graph,
419 + use_inductor=self.compilation_configs.use_inductor)
420 +
421 + return self.graph(*args)
422 +
423 + if not self.first_run_finished:
424 + self.first_run_finished = True
425 + return self.compiled_graph_for_general_shape(*args)
426 +
427 + runtime_shape = args[self.sym_shape_indices[0]]
428 + if runtime_shape not in self.concrete_size_entries:
429 + # we don't need to do anything for this shape
430 + return self.compiled_graph_for_general_shape(*args)
431 +
432 + entry = self.concrete_size_entries[runtime_shape]
433
240 - return runtime_shapes_to_compiled_graph[runtime_shapes](*args)
434 + if entry.runnable is None:
435 + entry.runnable = self.compiled_graph_for_general_shape
436
242 - return compiled_graph_wrapper
437 + if entry.need_to_compile and not entry.compiled:
438 + entry.compiled = True
439 + # args are real arguments
440 + entry.runnable = wrap_inductor(
441 + self.graph,
442 + args,
443 + self.compilation_configs.inductor_compile_config,
444 + runtime_shape=runtime_shape,
445 + do_logging=self.is_first_graph,
446 + use_inductor=self.compilation_configs.use_inductor)
447 +
448 + if not entry.use_cudagraph:
449 + return entry.runnable(*args)
450 +
451 + if entry.cudagraph is None:
452 + if entry.num_finished_warmup < self.compilation_configs.cudagraph_num_of_warmups: # noqa
453 + entry.num_finished_warmup += 1
454 + if self.is_first_graph:
455 + logger.debug(
456 + "Warming up %s/%s for shape %s",
457 + entry.num_finished_warmup,
458 + self.compilation_configs.cudagraph_num_of_warmups,
459 + runtime_shape)
460 + return entry.runnable(*args)
461 +
462 + if self.is_first_graph:
463 + logger.info("Capturing a cudagraph for shape %s",
464 + runtime_shape)
465 +
466 + cudagraph = torch.cuda.CUDAGraph()
467 + with torch.cuda.graph(cudagraph, pool=self.graph_pool):
468 + entry.output = weak_ref_tensors(entry.runnable(*args))
469 +
470 + compilation_counter.num_cudagraph_caputured += 1
471 +
472 + entry.cudagraph = cudagraph
473 + return entry.output
474 +
475 + entry.cudagraph.replay()
476 + return entry.output
477
478
479 def select_default_backend(level: int) -> Union[str, Callable]:
480 if level in [CompilationLevel.DYNAMO_AS_IS, CompilationLevel.DYNAMO_ONCE]:
481 backend_str = "eager"
482 return backend_str
249 - assert level in [
250 - CompilationLevel.INDUCTOR, CompilationLevel.INDUCTOR_MAX_AUTOTUNE
251 - ], f"Invalid level {level}"
252 -
253 - from vllm.compilation.backends import vllm_backend
254 - from vllm.plugins import get_inductor_additional_configs
255 - additional_configs = get_inductor_additional_configs()
256 -
257 - if level == CompilationLevel.INDUCTOR_MAX_AUTOTUNE:
258 - if "max_autotune" in additional_configs and not additional_configs[
259 - "max_autotune"]:
260 - logger.warning(
261 - "max_autotune is disabled, but is overridden by level %s",
262 - CompilationLevel.INDUCTOR_MAX_AUTOTUNE)
263 - additional_configs['max_autotune'] = True
264 -
265 - from functools import partial
266 - backend = partial(vllm_backend,
267 - additional_inductor_config=additional_configs)
268 -
269 - return backend
483 + assert level == CompilationLevel.PIECEWISE
484 +
485 + return VllmBackend()
vllm/compilation/config.py new
+154
@@ -0,0 +1,154 @@
1 +import copy
2 +from typing import Any, Dict, List, Optional
3 +
4 +from pydantic import BaseModel, Field, PrivateAttr
5 +
6 +import vllm.envs as envs
7 +from vllm.logger import init_logger
8 +
9 +from .compile_context import get_compile_context
10 +
11 +logger = init_logger(__name__)
12 +
13 +
14 +class CompilationConfig(BaseModel):
15 + """
16 + Configuration for compilation.
17 + It has two parts:
18 + - CudaGraph capture:
19 + - use_cudagraph: whether to use cudagraph inside compilation.
20 + - False: cudagraph inside compilation is not used.
21 + - True: cudagraph inside compilation is used. It requires
22 + that all input buffers have fixed addresses.
23 + Note that this is orthogonal to the cudagraph capture out
24 + side of compilation.
25 + TODO: move outside cudagraph logic into compilation.
26 + torch.compile will handle cudagraph capture logic in the future.
27 + - cudagraph_capture_sizes: sizes to capture cudagraph.
28 + - None: capture sizes are inferred from compilation context.
29 + - List[int]: capture sizes are specified.
30 + - cudagraph_num_of_warmups: number of warmup runs for cudagraph.
31 + It means the first several runs will be treated as warmup runs.
32 + Only after that, the execution will be recorded, and the recorded
33 + cudagraph will be used for subsequent runs.
34 + - Inductor compilation:
35 + - use_inductor: whether to use inductor compilation.
36 + - False: inductor compilation is not used. graph runs in eager.
37 + - True: inductor compilation is used. one graph for symbolic shape
38 + is compiled. In addition, compile for different sizes specified
39 + in inductor_compile_sizes, using configurations
40 + in inductor_compile_config.
41 + - inductor_compile_sizes: sizes to compile for inductor.
42 + - inductor_specialize_for_cudagraph_no_more_than: an optional integer
43 + to specialize inductor for cudagraph sizes no more than the
44 + specified size. It is useful when we want to specialize inductor
45 + with a subset of cudagraph sizes.
46 + - inductor_compile_config: additional configurations for inductor.
47 + - None: use default configurations.
48 + - inductor_passes: additional passes for inductor. It is a dictionary
49 + from pass name to pass function qualified name. We use function
50 + name because the config uses json format. If we pass the config
51 + from Python, functions can also be passed directly via Python object
52 + constructor, e.g. `CompilationConfig(inductor_passes={"a": func})`
53 +
54 + Why we have different sizes for cudagraph and inductor:
55 + - cudagraph: a cudagraph captured for a specific size can only be used
56 + for the same size. We need to capture all the sizes we want to use.
57 + - inductor: a graph compiled by inductor for a general shape can be used
58 + for different sizes. Inductor can also compile for specific sizes,
59 + where it can have more information to optimize the graph with fully
60 + static shapes. However, we find the general shape compilation is
61 + sufficient for most cases. It might be beneficial to compile for
62 + certain small batchsizes, where inductor is good at optimizing.
63 + """
64 + use_inductor: bool = True
65 + inductor_specialize_for_cudagraph_no_more_than: Optional[int] = None
66 + inductor_compile_sizes: Optional[List[int]] = Field(default_factory=dict)
67 + inductor_compile_config: Dict = Field(default_factory=dict)
68 + inductor_passes: Dict[str, str] = Field(default_factory=dict)
69 +
70 + use_cudagraph: bool = False
71 + non_cudagraph_ops: List[str] = Field(default_factory=list)
72 + cudagraph_num_of_warmups: int = 0
73 + cudagraph_capture_sizes: Optional[List[int]] = None
74 +
75 + # not configurable, computed after init
76 + compile_sizes: List[int] = PrivateAttr
77 + capture_sizes: List[int] = PrivateAttr
78 +
79 + def model_post_init(self, __context: Any) -> None:
80 + for k, v in self.inductor_passes.items():
81 + if not isinstance(v, str):
82 + assert callable(v), (
83 + f"pass {k} should be a function or a qualified name")
84 + self.inductor_passes[k] = v
85 + continue
86 +
87 + # resolve function from qualified name
88 + names = v.split(".")
89 + module = ".".join(names[:-1])
90 + func_name = names[-1]
91 + func = __import__(module).__dict__[func_name]
92 + self.inductor_compile_config[k] = func
93 +
94 + from vllm.compilation.backends import fix_functionalization
95 + from vllm.utils import combine_fx_passes
96 + if "post_grad_custom_post_pass" in self.inductor_compile_config:
97 + self.inductor_compile_config[
98 + "post_grad_custom_post_pass"] = combine_fx_passes(
99 + fix_functionalization,
100 + self.inductor_compile_config["post_grad_custom_post_pass"],
101 + )
102 + else:
103 + self.inductor_compile_config[
104 + "post_grad_custom_post_pass"] = fix_functionalization
105 +
106 + def init_during_runtime(self):
107 + """To complete the initialization of config,
108 + we need to know the compile context, which is only available
109 + during the first run of the model.
110 + """
111 + context = get_compile_context()
112 + context = copy.deepcopy(context) if context is not None else []
113 + sizes_to_specialize: List[int] = context
114 + if self.cudagraph_capture_sizes is None:
115 + self.capture_sizes = sizes_to_specialize
116 + else:
117 + self.capture_sizes = self.cudagraph_capture_sizes
118 + logger.info(("cudagraph sizes specified by model runner"
119 + " %s is overridden by config %s"),
120 + sizes_to_specialize, self.cudagraph_capture_sizes)
121 + if self.inductor_specialize_for_cudagraph_no_more_than is not None:
122 + assert self.inductor_compile_sizes is None, (
123 + "inductor_compile_sizes should be None when "
124 + "inductor_specialize_for_cudagraph_no_more_than is not None")
125 + self.compile_sizes = [
126 + x for x in self.capture_sizes
127 + if x <= self.inductor_specialize_for_cudagraph_no_more_than
128 + ]
129 + else:
130 + assert self.inductor_compile_sizes is not None, (
131 + "inductor_compile_sizes should not be None when "
132 + "inductor_specialize_for_cudagraph_no_more_than is None")
133 + self.compile_sizes = self.inductor_compile_sizes
134 +
135 + @staticmethod
136 + def select_and_init_config() -> "CompilationConfig":
137 + """The order of selecting config is:
138 + 1. Use the config specified in environment variable.
139 + 2. Use the config specified in plugins.
140 + 3. Use the default config.
141 + """
142 + config_path = envs.VLLM_TORCH_COMPILE_CONFIG
143 + if config_path is not None:
144 + with open(config_path) as json_file:
145 + config = CompilationConfig.model_validate_json(
146 + json_file.read())
147 + else:
148 + from vllm.plugins import get_compilation_config
149 + predefined_config = get_compilation_config()
150 + config = predefined_config if predefined_config is not None else (
151 + CompilationConfig())
152 +
153 + config.init_during_runtime()
154 + return config
vllm/compilation/counter.py new
+30
@@ -0,0 +1,30 @@
1 +import copy
2 +import dataclasses
3 +from contextlib import contextmanager
4 +
5 +
6 +@dataclasses.dataclass
7 +class CompilationCounter:
8 + num_graphs_seen: int = 0
9 + # including the splitting ops
10 + num_piecewise_graphs_seen: int = 0
11 + # not including the splitting ops
12 + num_piecewise_capturable_graphs_seen: int = 0
13 + num_inductor_compilations: int = 0
14 + num_cudagraph_caputured: int = 0
15 +
16 + def clone(self) -> "CompilationCounter":
17 + return copy.deepcopy(self)
18 +
19 + @contextmanager
20 + def expect(self, **kwargs):
21 + old = self.clone()
22 + yield
23 + for k, v in kwargs.items():
24 + assert getattr(self, k) - getattr(old, k) == v, (
25 + f"{k} not as expected, before it is {getattr(old, k)}"
26 + f", after it is {getattr(self, k)}, "
27 + f"expected diff is {v}")
28 +
29 +
30 +compilation_counter = CompilationCounter()
vllm/compilation/decorators.py
+9 -1
@@ -121,7 +121,10 @@ def _support_torch_compile(cls: type,
121 # take care of method resolution order
122 # make sure super().__init__ is called on the base class
123 # other than TorchCompileWrapperWithCustomDispatcher
124 - cls.__bases__ = cls.__bases__ + (TorchCompileWrapperWithCustomDispatcher, )
124 + if TorchCompileWrapperWithCustomDispatcher not in cls.__bases__:
125 + # support decorating multiple times
126 + cls.__bases__ = cls.__bases__ + (
127 + TorchCompileWrapperWithCustomDispatcher, )
128
129 old_init = cls.__init__ # type: ignore
130
@@ -160,6 +163,11 @@ def _support_torch_compile(cls: type,
163 # compiled function and let torch.compile handle the dispatching,
164 # with the overhead of guard evaluation and recompilation.
165 if len(self.compiled_codes) < 1 or not self.use_custom_dispatcher:
166 + # it seems Dynamo reuse the compilation across instances,
167 + # while we need to make sure the compiled code is not reused.
168 + # we need to control all the compilation of the model.
169 + torch._dynamo.eval_frame.remove_from_cache(
170 + self.original_code_object)
171 return self.compiled_callable(*args, **kwargs)
172
173 # usually, capturing the model once is enough, and then we can
vllm/compilation/levels.py
+1 -2
@@ -5,5 +5,4 @@ class CompilationLevel:
5 NO_COMPILATION = 0
6 DYNAMO_AS_IS = 1
7 DYNAMO_ONCE = 2
8 - INDUCTOR = 3
9 - INDUCTOR_MAX_AUTOTUNE = 4
8 + PIECEWISE = 3
vllm/envs.py
+5
@@ -209,6 +209,11 @@ environment_variables: Dict[str, Callable[[], Any]] = {
209 os.environ.get("VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE", "1") != "0"),
210 "VLLM_TORCH_COMPILE_LEVEL":
211 lambda: int(os.environ.get("VLLM_TORCH_COMPILE_LEVEL", "0")),
212 +
213 + # Path to the config file for torch compile
214 + "VLLM_TORCH_COMPILE_CONFIG":
215 + lambda: os.environ.get("VLLM_TORCH_COMPILE_CONFIG", None),
216 +
217 # Fine-grained control over which custom ops to enable/disable.
218 # Use 'all' to enable all, 'none' to disable all.
219 # Also specify a list of custom op names to enable (prefixed with a '+'),
vllm/model_executor/custom_op.py
+2 -2
@@ -100,7 +100,7 @@ class CustomOp(nn.Module):
100
101 return (CustomOp.default_on() or enabled) and not disabled
102
103 - # On by default if VLLM_TORCH_COMPILE_LEVEL < CompilationLevel.INDUCTOR
103 + # On by default if VLLM_TORCH_COMPILE_LEVEL < CompilationLevel.PIECEWISE
104 # Specifying 'all' or 'none' in VLLM_CUSTOM_OPS takes precedence.
105 @staticmethod
106 @lru_cache()
@@ -108,7 +108,7 @@ class CustomOp(nn.Module):
108 count_none = envs.VLLM_CUSTOM_OPS.count("none")
109 count_all = envs.VLLM_CUSTOM_OPS.count("all")
110 assert count_none + count_all <= 1, "Can only specify 'none' or 'all'"
111 - return envs.VLLM_TORCH_COMPILE_LEVEL < CompilationLevel.INDUCTOR and \
111 + return envs.VLLM_TORCH_COMPILE_LEVEL < CompilationLevel.PIECEWISE and \
112 not count_none > 0 or count_all > 0
113
114 # Dictionary of all custom ops (classes, indexed by registered name).
vllm/platforms/tpu.py
+1 -1
@@ -11,7 +11,7 @@ from .interface import Platform, PlatformEnum
11 if "VLLM_TORCH_COMPILE_LEVEL" not in os.environ:
12 os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(CompilationLevel.DYNAMO_ONCE)
13
14 -assert envs.VLLM_TORCH_COMPILE_LEVEL < CompilationLevel.INDUCTOR,\
14 +assert envs.VLLM_TORCH_COMPILE_LEVEL < CompilationLevel.PIECEWISE,\
15 "TPU does not support Inductor."
16
17 set_torch_compile_backend("openxla")
vllm/plugins/__init__.py
+8 -7
@@ -1,7 +1,8 @@
1 import logging
2 -from typing import Callable, Dict, Optional, Union
2 +from typing import Callable, Optional, Union
3
4 import vllm.envs as envs
5 +from vllm.compilation.config import CompilationConfig
6
7 logger = logging.getLogger(__name__)
8
@@ -44,13 +45,13 @@ def get_torch_compile_backend() -> Optional[Union[Callable, str]]:
45 return _torch_compile_backend
46
47
47 -_inductor_additional_configs: Dict = {}
48 +_compilation_config: Optional[CompilationConfig] = None
49
50
50 -def set_inductor_additional_configs(configs: Dict):
51 - global _inductor_additional_configs
52 - _inductor_additional_configs = configs
51 +def set_compilation_config(config: Optional[CompilationConfig]):
52 + global _compilation_config
53 + _compilation_config = config
54
55
55 -def get_inductor_additional_configs() -> Dict:
56 - return _inductor_additional_configs
56 +def get_compilation_config() -> Optional[CompilationConfig]:
57 + return _compilation_config
vllm/utils.py
+25
@@ -1479,6 +1479,15 @@ class LazyDict(Mapping, Generic[T]):
1479 return len(self._factory)
1480
1481
1482 +def combine_fx_passes(passes: List[Callable]) -> Callable:
1483 +
1484 + def combined_fx(graph) -> None:
1485 + for fx in passes:
1486 + fx(graph)
1487 +
1488 + return combined_fx
1489 +
1490 +
1491 def weak_ref_tensor(tensor: torch.Tensor) -> torch.Tensor:
1492 """
1493 Create a weak reference to a tensor.
@@ -1486,3 +1495,19 @@ def weak_ref_tensor(tensor: torch.Tensor) -> torch.Tensor:
1495 but will not keep the original tensor alive.
1496 """
1497 return torch.ops._C.weak_ref_tensor(tensor)
1498 +
1499 +
1500 +def weak_ref_tensors(
1501 + tensors: Union[torch.Tensor, List[torch.Tensor], Tuple[torch.Tensor]]
1502 +) -> Union[torch.Tensor, List[torch.Tensor], Tuple[torch.Tensor]]:
1503 + """
1504 + Convenience function to create weak references to tensors,
1505 + for single tensor, list of tensors or tuple of tensors.
1506 + """
1507 + if isinstance(tensors, torch.Tensor):
1508 + return weak_ref_tensor(tensors)
1509 + if isinstance(tensors, list):
1510 + return [weak_ref_tensor(t) for t in tensors]
1511 + if isinstance(tensors, tuple):
1512 + return tuple(weak_ref_tensor(t) for t in tensors)
1513 + raise ValueError("Invalid type for tensors")