@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 330e82d34a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[v1][torch.compile] support managing cudagraph buffer (#10203)

Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

youkaichao committed Nov 11, 2024 at 11:10 UTC 330e82d34a36ccee3f2f80fded3e7cc0d67718d6
4 files changed +59 -8
tests/compile/piecewise/piecewise_compilation_config.json
+2 -1
@@ -1,4 +1,5 @@
1 {
2 "use_cudagraph": true,
3 - "non_cudagraph_ops": ["silly.attention"]
3 + "non_cudagraph_ops": ["silly.attention"],
4 + "cudagraph_copy_inputs": true
5 }
\ No newline at end of file
tests/compile/piecewise/test_simple.py
+6 -6
@@ -80,7 +80,7 @@ def test_simple_piecewise_compile():
80 config = os.path.join(directory, "piecewise_compilation_config.json")
81 os.environ["VLLM_TORCH_COMPILE_CONFIG"] = config
82
83 - input_buffer = torch.randn(100).cuda()
83 + inputs = torch.randn(100).cuda()
84
85 with compilation_counter.expect(
86 num_graphs_seen=1, # one graph for the model
@@ -92,15 +92,15 @@ def test_simple_piecewise_compile():
92 ):
93
94 with set_compile_context([1, 2]):
95 - model(input_buffer)
95 + model(inputs)
96
97 - model(input_buffer[:2])
98 - model(input_buffer[:1])
97 + model(torch.randn(2).cuda())
98 + model(torch.randn(1).cuda())
99
100 - input_buffer[:2].zero_()
100 + input = torch.zeros(2).cuda()
101 global global_counter
102 global_counter = 0
103 - output = model(input_buffer[:2])
103 + output = model(input)
104 assert global_counter == 2
105 assert torch.allclose(output.cpu(), torch.tensor([3., 1.]))
106
vllm/compilation/backends.py
+45 -1
@@ -389,6 +389,8 @@ class VllmBackend:
389 returned_callable: Callable
390 # Inductor passes to run on the graph pre-defunctionalization
391 post_grad_passes: Sequence[Callable]
392 + sym_tensor_indices: List[int]
393 + input_buffers: List[torch.Tensor]
394
395 def __init__(self, post_grad_passes: Sequence[Callable] = ()):
396 global global_graph_pool
@@ -401,6 +403,9 @@ class VllmBackend:
403 self.graph_pool = global_graph_pool
404 self.post_grad_passes = post_grad_passes
405
406 + self.sym_tensor_indices = []
407 + self.input_buffers = []
408 +
409 # `torch.compile` is JIT compiled, so we don't need to
410 # do anything here
411
@@ -461,7 +466,46 @@ class VllmBackend:
466
467 self._called = True
468
464 - return self.split_gm
469 + if not self.compilation_configs.use_cudagraph or \
470 + not self.compilation_configs.cudagraph_copy_inputs:
471 + return self.split_gm
472 +
473 + # if we need to copy input buffers for cudagraph
474 + from torch._guards import detect_fake_mode
475 + fake_mode = detect_fake_mode()
476 + fake_args = [
477 + fake_mode.from_tensor(t) if isinstance(t, torch.Tensor) else t
478 + for t in example_inputs
479 + ]
480 +
481 + # index of tensors that have symbolic shapes (batch size)
482 + self.sym_tensor_indices = [
483 + i for i, x in enumerate(fake_args)
484 + if isinstance(x, torch._subclasses.fake_tensor.FakeTensor)
485 + ]
486 +
487 + # compiler managed cudagraph input buffers
488 + # we assume the first run with symbolic shapes
489 + # has the maximum size among all the tensors
490 + self.input_buffers = [
491 + example_inputs[x].clone() for x in self.sym_tensor_indices
492 + ]
493 +
494 + def copy_and_call(*args):
495 + list_args = list(args)
496 + for i, index in enumerate(self.sym_tensor_indices):
497 + runtime_tensor = list_args[index]
498 + runtime_shape = runtime_tensor.shape[0]
499 + static_tensor = self.input_buffers[i][:runtime_shape]
500 +
501 + # copy the tensor to the static buffer
502 + static_tensor.copy_(runtime_tensor)
503 +
504 + # replace the tensor in the list_args to the static buffer
505 + list_args[index] = static_tensor
506 + return self.split_gm(*list_args)
507 +
508 + return copy_and_call
509
510
511 @dataclasses.dataclass
vllm/compilation/config.py
+6
@@ -32,6 +32,11 @@ class CompilationConfig(BaseModel):
32 It means the first several runs will be treated as warmup runs.
33 Only after that, the execution will be recorded, and the recorded
34 cudagraph will be used for subsequent runs.
35 + - cudagraph_copy_inputs: whether to copy input tensors for
36 + cudagraph. If the caller can guarantee that the same input buffers
37 + are always used, it can set this to False. Otherwise, it should
38 + set this to True, and the compiler will copy the input to an
39 + internally managed buffer. Default is False.
40 - Inductor compilation:
41 - use_inductor: whether to use inductor compilation.
42 - False: inductor compilation is not used. graph runs in eager.
@@ -78,6 +83,7 @@ class CompilationConfig(BaseModel):
83 non_cudagraph_ops: List[str] = Field(default_factory=list)
84 cudagraph_num_of_warmups: int = 0
85 cudagraph_capture_sizes: Optional[List[int]] = None
86 + cudagraph_copy_inputs: bool = False
87
88 dump_graph_stages: List[str] = Field(default_factory=list)
89 dump_graph_dir: Path = Field(default=Path("."))