@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0373e1837e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Core][CUDA Graph] add output buffer for cudagraph (#5074)

[Core][CUDA Graph] add output buffer for cudagraph to reduce memory footprint (#5074)

youkaichao committed Jun 8, 2024 at 19:14 UTC 0373e1837e1a85c595fa9fc67c775bc6cbe105a2
1 file changed +20 -4
vllm/worker/model_runner.py
+20 -4
@@ -1,3 +1,4 @@
1 +import gc
2 import time
3 import warnings
4 from collections import defaultdict
@@ -894,6 +895,10 @@ class ModelRunner:
895 seq_lens = torch.ones(max_batch_size, dtype=torch.int32).cuda()
896 block_tables = torch.from_numpy(self.graph_block_tables).cuda()
897
898 + # Prepare buffer for outputs. These will be reused for all batch sizes.
899 + # It will be filled after the first graph capture.
900 + hidden_states: Optional[torch.Tensor] = None
901 +
902 graph_batch_size = _get_graph_batch_size(
903 self.scheduler_config.max_num_seqs)
904 batch_size_capture_list = [
@@ -930,9 +935,11 @@ class ModelRunner:
935 self.set_active_loras(set(), lora_mapping)
936
937 graph_runner = CUDAGraphRunner(self.model)
933 - graph_runner.capture(
938 + hidden_states = graph_runner.capture(
939 input_tokens[:batch_size],
940 input_positions[:batch_size],
941 + hidden_states[:batch_size]
942 + if hidden_states is not None else None,
943 kv_caches,
944 attn_metadata,
945 memory_pool=self.graph_memory_pool,
@@ -969,12 +976,13 @@ class CUDAGraphRunner:
976 self,
977 input_ids: torch.Tensor,
978 positions: torch.Tensor,
979 + hidden_states: Optional[torch.Tensor],
980 kv_caches: List[torch.Tensor],
981 attn_metadata: AttentionMetadata,
982 memory_pool: Optional[Tuple[int, int]],
983 stream: torch.cuda.Stream,
984 **kwargs,
977 - ) -> None:
985 + ) -> torch.Tensor:
986 assert self._graph is None
987 # Run the model a few times without capturing the graph.
988 # This is to make sure that the captured graph does not include the
@@ -993,13 +1001,21 @@ class CUDAGraphRunner:
1001 # Capture the graph.
1002 self._graph = torch.cuda.CUDAGraph()
1003 with torch.cuda.graph(self._graph, pool=memory_pool, stream=stream):
996 - hidden_states = self.model(
1004 + output_hidden_states = self.model(
1005 input_ids,
1006 positions,
1007 kv_caches,
1008 attn_metadata,
1009 **kwargs,
1010 )
1011 + if hidden_states is not None:
1012 + hidden_states.copy_(output_hidden_states)
1013 + else:
1014 + hidden_states = output_hidden_states
1015 + del output_hidden_states
1016 + # make sure `output_hidden_states` is deleted
1017 + # in the graph's memory pool
1018 + gc.collect()
1019 torch.cuda.synchronize()
1020
1021 # Save the input and output buffers.
@@ -1012,7 +1028,7 @@ class CUDAGraphRunner:
1028 "block_tables": attn_metadata.decode_metadata.block_tables,
1029 }
1030 self.output_buffers = {"hidden_states": hidden_states}
1015 - return
1031 + return hidden_states
1032
1033 def forward(
1034 self,