@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ca9844b340
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[bugfix] fix weak ref in piecewise cudagraph and tractable test (#10048)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 5, 2024 at 14:49 UTC ca9844b340f45f23f8d30fdce23777d215ad987c
2 files changed +168 -25
tests/compile/piecewise/test_toy_llama.py
+101 -10
@@ -1,6 +1,10 @@
1 """
2 Test the piecewise compilation with a simple model, comparing the output
3 with and without the piecewise compilation.
4 +
5 +This is a tractable model, the weights and computation are specially designed
6 +if the config `tractable_init` is set to True. Otherwise, the weights are
7 +initialized randomly with a fixed seed.
8 """
9 import os
10 from dataclasses import dataclass
@@ -49,6 +53,12 @@ class LlamaConfig:
53 mlp_size: int = 256
54 vocab_size: int = 128
55 num_layers: int = 2
56 + init_value: float = 1.0
57 + tractable_init: bool = False
58 + random_seed: int = 0
59 +
60 + def __post_init__(self):
61 + assert self.mlp_size >= self.hidden_size
62
63
64 class LlamaMLP(nn.Module):
@@ -66,10 +76,23 @@ class LlamaMLP(nn.Module):
76 bias=False,
77 )
78
69 - self.gate_up_projection.weight.data.fill_(0.0)
70 - self.down_projection.weight.data.fill_(0.0)
79 + if config.tractable_init:
80 + nn.init.eye_(self.gate_up_projection.weight.data[:config.mlp_size])
81 + nn.init.eye_(self.gate_up_projection.weight.data[config.mlp_size:])
82 + nn.init.eye_(self.down_projection.weight.data)
83 + else:
84 + nn.init.xavier_normal_(self.gate_up_projection.weight.data,
85 + generator=torch.Generator().manual_seed(
86 + config.random_seed),
87 + gain=0.001)
88 + nn.init.xavier_normal_(self.down_projection.weight.data,
89 + generator=torch.Generator().manual_seed(
90 + config.random_seed),
91 + gain=0.001)
92
93 def forward(self, x):
94 + # for tractable_init and positive input, this is
95 + # essentially an elementwise-square
96 x = self.gate_up_projection(x)
97 x = x[:, :x.size(1) // 2] * torch.nn.functional.relu(
98 x[:, x.size(1) // 2:])
@@ -84,21 +107,39 @@ class LlamaAttention(nn.Module):
107 self.qkv_projection = nn.Linear(
108 in_features=config.hidden_size,
109 out_features=config.hidden_size * 3,
110 + bias=False,
111 )
112
113 self.output_projection = nn.Linear(
114 in_features=config.hidden_size,
115 out_features=config.hidden_size,
116 + bias=False,
117 )
118
94 - self.qkv_projection.weight.data.fill_(0.0)
95 - self.output_projection.weight.data.fill_(0.0)
119 + if config.tractable_init:
120 + nn.init.eye_(self.qkv_projection.weight.data[:config.hidden_size])
121 + nn.init.eye_(self.qkv_projection.weight.data[config.hidden_size:2 *
122 + config.hidden_size])
123 + nn.init.eye_(self.qkv_projection.weight.data[2 *
124 + config.hidden_size:])
125 + nn.init.eye_(self.output_projection.weight.data)
126 + else:
127 + nn.init.xavier_normal_(self.qkv_projection.weight.data,
128 + generator=torch.Generator().manual_seed(
129 + config.random_seed),
130 + gain=0.001)
131 + nn.init.xavier_normal_(self.output_projection.weight.data,
132 + generator=torch.Generator().manual_seed(
133 + config.random_seed),
134 + gain=0.001)
135
136 def forward(
137 self,
138 positions: torch.Tensor,
139 hidden_states: torch.Tensor,
140 ) -> torch.Tensor:
141 + # for tractable_init, this is:
142 + # output = (hidden_states * 3 + positions * 2)
143 qkv = self.qkv_projection(hidden_states)
144 hidden_size = qkv.size(-1) // 3
145 q, k, v = qkv.split([hidden_size, hidden_size, hidden_size], dim=-1)
@@ -126,20 +167,29 @@ class LlamaDecoderLayer(nn.Module):
167 hidden_states: torch.Tensor,
168 residual: Optional[torch.Tensor],
169 ) -> Tuple[torch.Tensor, torch.Tensor]:
170 + """
171 + For tractable computation:
172 + - if residual is None, the outputs are:
173 + - residual = (hidden_states + 1) * 3 + positions * 2 + hidden_states = hidden_states * 4 + positions * 2 + 3
174 + - hidden_states = (residual + 1) ** 2
175 + - if residual is not None, the outputs are:
176 + - residual = (hidden_states + residual + 1) * 3 + positions * 2 + hidden_states + residual = (hidden_states + residual) * 4 + positions * 2 + 3
177 + - hidden_states = (residual + 1) ** 2
178 + """ # noqa
179 if residual is None:
180 residual = hidden_states
131 - hidden_states = hidden_states / 2
181 + hidden_states = hidden_states + 1
182 else:
183 hidden_states = hidden_states + residual
184 residual = hidden_states
135 - hidden_states = hidden_states / 2
185 + hidden_states = hidden_states + 1
186
187 hidden_states = self.self_attention(positions=positions,
188 hidden_states=hidden_states)
189
190 hidden_states = hidden_states + residual
191 residual = hidden_states
142 - hidden_states = hidden_states / 2
192 + hidden_states = hidden_states + 1
193 hidden_states = self.mlp(hidden_states)
194
195 return hidden_states, residual
@@ -156,7 +206,8 @@ class LlamaModel(nn.Module):
206 self.layers = nn.ModuleList(
207 [LlamaDecoderLayer(config) for _ in range(config.num_layers)])
208
159 - self.embedding_tokens.weight.data.fill_(0.0)
209 + # this is the initial value of the hidden states
210 + self.embedding_tokens.weight.data.fill_(config.init_value)
211
212 def forward(
213 self,
@@ -170,6 +221,28 @@ class LlamaModel(nn.Module):
221 return hidden_states
222
223
224 +def tractable_computation(input_ids: torch.Tensor,
225 + positions: torch.Tensor,
226 + config: LlamaConfig,
227 + init_value: float = 1.0) -> torch.Tensor:
228 + hidden_states = torch.ones(input_ids.size(0),
229 + config.hidden_size,
230 + device=input_ids.device,
231 + dtype=input_ids.dtype) * init_value
232 +
233 + # first layer
234 + residual = hidden_states * 4 + positions.unsqueeze(1) * 2 + 3
235 + hidden_states = (residual + 1)**2
236 +
237 + # following layers
238 + for _ in range(config.num_layers - 1):
239 + hidden_states = hidden_states + residual
240 + residual = hidden_states * 4 + positions.unsqueeze(1) * 2 + 3
241 + hidden_states = (residual + 1)**2
242 +
243 + return hidden_states
244 +
245 +
246 @torch.inference_mode
247 def run_model(llama_config,
248 use_compile: bool,
@@ -213,7 +286,15 @@ def run_model(llama_config,
286 del os.environ["VLLM_TORCH_COMPILE_LEVEL"]
287 set_compilation_config(None)
288
216 - return output.cpu()
289 + output = output.cpu()
290 +
291 + if llama_config.tractable_init:
292 + expected_output = tractable_computation(input_ids[:2], positions[:2],
293 + llama_config).cpu()
294 +
295 + assert torch.allclose(output, expected_output)
296 + else:
297 + return output.cpu()
298
299
300 def test_toy_llama():
@@ -222,7 +303,13 @@ def test_toy_llama():
303 llama_config = LlamaConfig(hidden_size=128,
304 mlp_size=256,
305 vocab_size=128,
225 - num_layers=2)
306 + num_layers=12)
307 +
308 + tractable_config = LlamaConfig(hidden_size=128,
309 + mlp_size=256,
310 + vocab_size=128,
311 + num_layers=2,
312 + tractable_init=True)
313
314 outputs = []
315 with compilation_counter.expect(
@@ -233,6 +320,8 @@ def test_toy_llama():
320 num_cudagraph_caputured=0,
321 ):
322 outputs.append(run_model(llama_config, use_compile=False))
323 + run_model(tractable_config, use_compile=False)
324 +
325 with compilation_counter.expect(
326 num_graphs_seen=1, # one graph for the model
327 num_piecewise_graphs_seen=1,
@@ -242,6 +331,7 @@ def test_toy_llama():
331 2, # num_cudagraph_sizes * num_piecewise_capturable_graphs_seen
332 ):
333 outputs.append(run_model(llama_config, use_compile=True))
334 + run_model(tractable_config, use_compile=True)
335
336 with compilation_counter.expect(
337 num_graphs_seen=1, # one graph for the model
@@ -257,6 +347,7 @@ def test_toy_llama():
347 ):
348 outputs.append(
349 run_model(llama_config, use_compile=True, split_attn=True))
350 + run_model(tractable_config, use_compile=True, split_attn=True)
351
352 for i in range(1, len(outputs)):
353 assert torch.allclose(outputs[0], outputs[i])
vllm/compilation/backends.py
+67 -15
@@ -6,6 +6,7 @@ from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
6 import torch
7 import torch.fx as fx
8
9 +import vllm.envs as envs
10 from vllm.logger import init_logger
11 from vllm.utils import weak_ref_tensors
12
@@ -193,6 +194,7 @@ def wrap_inductor(graph,
194 @dataclasses.dataclass
195 class SplitItem:
196 submod_name: str
197 + graph_id: int
198 is_splitting_graph: bool
199 graph: fx.GraphModule
200
@@ -226,9 +228,7 @@ def split_graph(graph: fx.GraphModule,
228
229 outputs = []
230
229 - # sort the names to make sure the order is deterministic
231 names = [name for (name, module) in split_gm.named_modules()]
231 - names.sort()
232
233 for name in names:
234 if "." in name or name == "":
@@ -238,7 +238,11 @@ def split_graph(graph: fx.GraphModule,
238 module = getattr(split_gm, name)
239
240 graph_id = int(name.replace("submod_", ""))
241 - outputs.append(SplitItem(name, graph_id in split_op_graphs, module))
241 + outputs.append(
242 + SplitItem(name, graph_id, (graph_id in split_op_graphs), module))
243 +
244 + # sort by intetger graph_id, rather than string name
245 + outputs.sort(key=lambda x: x.graph_id)
246
247 return split_gm, outputs
248
@@ -252,6 +256,11 @@ class PiecewiseCompileInterpreter(torch.fx.Interpreter):
256 It runs the given graph with fake inputs, and compile some
257 submodules specified by `compile_submod_names` with the given
258 compilation configs.
259 +
260 + NOTE: the order in `compile_submod_names` matters, because
261 + it will be used to determine the order of the compiled piecewise
262 + graphs. The first graph will handle logging, and the last graph
263 + has some special cudagraph output handling.
264 """
265
266 def __init__(self, module: torch.fx.GraphModule,
@@ -263,7 +272,6 @@ class PiecewiseCompileInterpreter(torch.fx.Interpreter):
272 self.compile_submod_names = compile_submod_names
273 self.compilation_configs = compilation_configs
274 self.graph_pool = graph_pool
266 - self.have_seen_first_graph = False
275
276 def run(self, *args):
277 fake_args = [
@@ -279,6 +287,7 @@ class PiecewiseCompileInterpreter(torch.fx.Interpreter):
287 output = super().call_module(target, args, kwargs)
288
289 if target in self.compile_submod_names:
290 + index = self.compile_submod_names.index(target)
291 submod = self.fetch_attr(target)
292 sym_shape_indices = [
293 i for i, x in enumerate(args) if isinstance(x, torch.SymInt)
@@ -288,15 +297,14 @@ class PiecewiseCompileInterpreter(torch.fx.Interpreter):
297 args,
298 self.compilation_configs.inductor_compile_config,
299 runtime_shape=None,
291 - do_logging=not self.have_seen_first_graph,
300 + do_logging=index == 0,
301 use_inductor=self.compilation_configs.use_inductor)
302
303 self.module.__dict__[target] = PiecewiseBackend(
295 - submod, self.compilation_configs, self.graph_pool,
296 - not self.have_seen_first_graph, sym_shape_indices,
304 + submod, self.compilation_configs, self.graph_pool, index,
305 + len(self.compile_submod_names), sym_shape_indices,
306 compiled_graph_for_general_shape)
307
299 - self.have_seen_first_graph = True
308 compilation_counter.num_piecewise_capturable_graphs_seen += 1
309
310 return output
@@ -352,8 +360,9 @@ class VllmBackend:
360 graph, self.compilation_configs.non_cudagraph_ops)
361
362 from torch._dynamo.utils import lazy_format_graph_code
355 - logger.debug("%s",
356 - lazy_format_graph_code("stiching module", self.split_gm))
363 + logger.debug("%s", lazy_format_graph_code("before split", self.graph))
364 + logger.debug("%s", lazy_format_graph_code("after split",
365 + self.split_gm))
366
367 compilation_counter.num_piecewise_graphs_seen += len(
368 self.piecewise_graphs)
@@ -385,12 +394,17 @@ class ConcreteSizeEntry:
394 cudagraph: Optional[torch.cuda.CUDAGraph] = None
395 output: Optional[Any] = None
396
397 + # for cudagraph debugging, track the input addresses
398 + # during capture, and check if they are the same during replay
399 + input_addresses: Optional[List[int]] = None
400 +
401
402 class PiecewiseBackend:
403
404 def __init__(self, graph: fx.GraphModule,
405 compilation_configs: CompilationConfig, graph_pool: Any,
393 - is_first_graph: bool, sym_shape_indices: List[int],
406 + piecewise_compile_index: int, total_piecewise_compiles: int,
407 + sym_shape_indices: List[int],
408 compiled_graph_for_general_shape: Callable):
409 """
410 The backend for piecewise compilation.
@@ -408,7 +422,12 @@ class PiecewiseBackend:
422 self.graph = graph
423 self.compilation_configs = compilation_configs
424 self.graph_pool = graph_pool
411 - self.is_first_graph = is_first_graph
425 + self.piecewise_compile_index = piecewise_compile_index
426 + self.total_piecewise_compiles = total_piecewise_compiles
427 +
428 + self.is_first_graph = piecewise_compile_index == 0
429 + self.is_last_graph = (
430 + piecewise_compile_index == total_piecewise_compiles - 1)
431
432 self.compile_sizes: Set[int] = set(
433 self.compilation_configs.compile_sizes)
@@ -422,6 +441,8 @@ class PiecewiseBackend:
441
442 self.sym_shape_indices = sym_shape_indices
443
444 + self.is_debugging_mode = envs.VLLM_LOGGING_LEVEL == "DEBUG"
445 +
446 # the entries for different shapes that we need to either
447 # compile or capture cudagraph
448 self.concrete_size_entries: Dict[int, ConcreteSizeEntry] = {}
@@ -476,14 +497,45 @@ class PiecewiseBackend:
497 logger.info("Capturing a cudagraph for shape %s",
498 runtime_shape)
499
500 + input_addresses = [
501 + x.data_ptr() for x in args if isinstance(x, torch.Tensor)
502 + ]
503 + entry.input_addresses = input_addresses
504 cudagraph = torch.cuda.CUDAGraph()
505 +
506 + # mind-exploding: carefully manage the reference and memory.
507 with torch.cuda.graph(cudagraph, pool=self.graph_pool):
481 - entry.output = weak_ref_tensors(entry.runnable(*args))
508 + # `output` is managed by pytorch's cudagraph pool
509 + output = entry.runnable(*args)
510 + if self.is_last_graph:
511 + # by converting it to weak ref,
512 + # the original `output` will immediately be released
513 + # to save memory. It is only safe to do this for
514 + # the last graph, because the output of the last graph
515 + # will not be used by any other cuda graph.
516 + output = weak_ref_tensors(output)
517 +
518 + # here we always use weak ref for the output
519 + # to save memory
520 + entry.output = weak_ref_tensors(output)
521 + entry.cudagraph = cudagraph
522
523 compilation_counter.num_cudagraph_caputured += 1
524
485 - entry.cudagraph = cudagraph
486 - return entry.output
525 + # important: we need to return the output, rather than
526 + # the weak ref of the output, so that pytorch can correctly
527 + # manage the memory during cuda graph capture
528 + return output
529 +
530 + if self.is_debugging_mode:
531 + # check if the input addresses are the same
532 + new_input_addresses = [
533 + x.data_ptr() for x in args if isinstance(x, torch.Tensor)
534 + ]
535 + assert new_input_addresses == entry.input_addresses, (
536 + "Input addresses for cudagraphs are different during replay."
537 + f" Expected {entry.input_addresses}, got {new_input_addresses}"
538 + )
539
540 entry.cudagraph.replay()
541 return entry.output