A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Core][CUDA Graph] add output buffer for cudagraph (#5074)
[Core][CUDA Graph] add output buffer for cudagraph to reduce memory footprint (#5074)
youkaichao committed
Jun 8, 2024 at 19:14 UTC
0373e1837e1a85c595fa9fc67c775bc6cbe105a2
1 file changed
+20
-4
vllm/worker/model_runner.py
+20
-4
@@ -1,3 +1,4 @@
1
+import gc
2
import time
3
import warnings
4
from collections import defaultdict
@@ -894,6 +895,10 @@ class ModelRunner:
895
seq_lens = torch.ones(max_batch_size, dtype=torch.int32).cuda()
896
block_tables = torch.from_numpy(self.graph_block_tables).cuda()
897
898
+ # Prepare buffer for outputs. These will be reused for all batch sizes.
899
+ # It will be filled after the first graph capture.
900
+ hidden_states: Optional[torch.Tensor] = None
901
+
902
graph_batch_size = _get_graph_batch_size(
903
self.scheduler_config.max_num_seqs)
904
batch_size_capture_list = [
@@ -930,9 +935,11 @@ class ModelRunner:
935
self.set_active_loras(set(), lora_mapping)
936
937
graph_runner = CUDAGraphRunner(self.model)
933
- graph_runner.capture(
938
+ hidden_states = graph_runner.capture(
939
input_tokens[:batch_size],
940
input_positions[:batch_size],
941
+ hidden_states[:batch_size]
942
+ if hidden_states is not None else None,
943
kv_caches,
944
attn_metadata,
945
memory_pool=self.graph_memory_pool,
@@ -969,12 +976,13 @@ class CUDAGraphRunner:
976
self,
977
input_ids: torch.Tensor,
978
positions: torch.Tensor,
979
+ hidden_states: Optional[torch.Tensor],
980
kv_caches: List[torch.Tensor],
981
attn_metadata: AttentionMetadata,
982
memory_pool: Optional[Tuple[int, int]],
983
stream: torch.cuda.Stream,
984
**kwargs,
977
- ) -> None:
985
+ ) -> torch.Tensor:
986
assert self._graph is None
987
# Run the model a few times without capturing the graph.
988
# This is to make sure that the captured graph does not include the
@@ -993,13 +1001,21 @@ class CUDAGraphRunner:
1001
# Capture the graph.
1002
self._graph = torch.cuda.CUDAGraph()
1003
with torch.cuda.graph(self._graph, pool=memory_pool, stream=stream):
996
- hidden_states = self.model(
1004
+ output_hidden_states = self.model(
1005
input_ids,
1006
positions,
1007
kv_caches,
1008
attn_metadata,
1009
**kwargs,
1010
)
1011
+ if hidden_states is not None:
1012
+ hidden_states.copy_(output_hidden_states)
1013
+ else:
1014
+ hidden_states = output_hidden_states
1015
+ del output_hidden_states
1016
+ # make sure `output_hidden_states` is deleted
1017
+ # in the graph's memory pool
1018
+ gc.collect()
1019
torch.cuda.synchronize()
1020
1021
# Save the input and output buffers.
@@ -1012,7 +1028,7 @@ class CUDAGraphRunner:
1028
"block_tables": attn_metadata.decode_metadata.block_tables,
1029
}
1030
self.output_buffers = {"hidden_states": hidden_states}
1015
- return
1031
+ return hidden_states
1032
1033
def forward(
1034
self,