A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[v1][torch.compile] support managing cudagraph buffer (#10203)
Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
youkaichao committed
Nov 11, 2024 at 11:10 UTC
330e82d34a36ccee3f2f80fded3e7cc0d67718d6
4 files changed
+59
-8
tests/compile/piecewise/piecewise_compilation_config.json
+2
-1
@@ -1,4 +1,5 @@
1
{
2
"use_cudagraph": true,
3
- "non_cudagraph_ops": ["silly.attention"]
3
+ "non_cudagraph_ops": ["silly.attention"],
4
+ "cudagraph_copy_inputs": true
5
}
\ No newline at end of file
tests/compile/piecewise/test_simple.py
+6
-6
@@ -80,7 +80,7 @@ def test_simple_piecewise_compile():
80
config = os.path.join(directory, "piecewise_compilation_config.json")
81
os.environ["VLLM_TORCH_COMPILE_CONFIG"] = config
82
83
- input_buffer = torch.randn(100).cuda()
83
+ inputs = torch.randn(100).cuda()
84
85
with compilation_counter.expect(
86
num_graphs_seen=1, # one graph for the model
@@ -92,15 +92,15 @@ def test_simple_piecewise_compile():
92
):
93
94
with set_compile_context([1, 2]):
95
- model(input_buffer)
95
+ model(inputs)
96
97
- model(input_buffer[:2])
98
- model(input_buffer[:1])
97
+ model(torch.randn(2).cuda())
98
+ model(torch.randn(1).cuda())
99
100
- input_buffer[:2].zero_()
100
+ input = torch.zeros(2).cuda()
101
global global_counter
102
global_counter = 0
103
- output = model(input_buffer[:2])
103
+ output = model(input)
104
assert global_counter == 2
105
assert torch.allclose(output.cpu(), torch.tensor([3., 1.]))
106
vllm/compilation/backends.py
+45
-1
@@ -389,6 +389,8 @@ class VllmBackend:
389
returned_callable: Callable
390
# Inductor passes to run on the graph pre-defunctionalization
391
post_grad_passes: Sequence[Callable]
392
+ sym_tensor_indices: List[int]
393
+ input_buffers: List[torch.Tensor]
394
395
def __init__(self, post_grad_passes: Sequence[Callable] = ()):
396
global global_graph_pool
@@ -401,6 +403,9 @@ class VllmBackend:
403
self.graph_pool = global_graph_pool
404
self.post_grad_passes = post_grad_passes
405
406
+ self.sym_tensor_indices = []
407
+ self.input_buffers = []
408
+
409
# `torch.compile` is JIT compiled, so we don't need to
410
# do anything here
411
@@ -461,7 +466,46 @@ class VllmBackend:
466
467
self._called = True
468
464
- return self.split_gm
469
+ if not self.compilation_configs.use_cudagraph or \
470
+ not self.compilation_configs.cudagraph_copy_inputs:
471
+ return self.split_gm
472
+
473
+ # if we need to copy input buffers for cudagraph
474
+ from torch._guards import detect_fake_mode
475
+ fake_mode = detect_fake_mode()
476
+ fake_args = [
477
+ fake_mode.from_tensor(t) if isinstance(t, torch.Tensor) else t
478
+ for t in example_inputs
479
+ ]
480
+
481
+ # index of tensors that have symbolic shapes (batch size)
482
+ self.sym_tensor_indices = [
483
+ i for i, x in enumerate(fake_args)
484
+ if isinstance(x, torch._subclasses.fake_tensor.FakeTensor)
485
+ ]
486
+
487
+ # compiler managed cudagraph input buffers
488
+ # we assume the first run with symbolic shapes
489
+ # has the maximum size among all the tensors
490
+ self.input_buffers = [
491
+ example_inputs[x].clone() for x in self.sym_tensor_indices
492
+ ]
493
+
494
+ def copy_and_call(*args):
495
+ list_args = list(args)
496
+ for i, index in enumerate(self.sym_tensor_indices):
497
+ runtime_tensor = list_args[index]
498
+ runtime_shape = runtime_tensor.shape[0]
499
+ static_tensor = self.input_buffers[i][:runtime_shape]
500
+
501
+ # copy the tensor to the static buffer
502
+ static_tensor.copy_(runtime_tensor)
503
+
504
+ # replace the tensor in the list_args to the static buffer
505
+ list_args[index] = static_tensor
506
+ return self.split_gm(*list_args)
507
+
508
+ return copy_and_call
509
510
511
@dataclasses.dataclass
vllm/compilation/config.py
+6
@@ -32,6 +32,11 @@ class CompilationConfig(BaseModel):
32
It means the first several runs will be treated as warmup runs.
33
Only after that, the execution will be recorded, and the recorded
34
cudagraph will be used for subsequent runs.
35
+ - cudagraph_copy_inputs: whether to copy input tensors for
36
+ cudagraph. If the caller can guarantee that the same input buffers
37
+ are always used, it can set this to False. Otherwise, it should
38
+ set this to True, and the compiler will copy the input to an
39
+ internally managed buffer. Default is False.
40
- Inductor compilation:
41
- use_inductor: whether to use inductor compilation.
42
- False: inductor compilation is not used. graph runs in eager.
@@ -78,6 +83,7 @@ class CompilationConfig(BaseModel):
83
non_cudagraph_ops: List[str] = Field(default_factory=list)
84
cudagraph_num_of_warmups: int = 0
85
cudagraph_capture_sizes: Optional[List[int]] = None
86
+ cudagraph_copy_inputs: bool = False
87
88
dump_graph_stages: List[str] = Field(default_factory=list)
89
dump_graph_dir: Path = Field(default=Path("."))