A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
extend cuda graph size for H200 (#7894)
Co-authored-by: youkaichao <youkaichao@126.com>
kushanam committed
Aug 29, 2024 at 12:15 UTC
c334b1898b68812af73a6d491010d929ffdb9862
1 file changed
+31
-7
vllm/worker/model_runner.py
+31
-7
@@ -60,10 +60,14 @@ logger = init_logger(__name__)
60
61
LORA_WARMUP_RANK = 8
62
_BATCH_SIZE_ALIGNMENT = 8
63
-# Capture graphs for token size 1, 2, 4, 8, 16, 24, 32, 40, ..., 256.
63
+# all the token sizes that **can** be captured by cudagraph.
64
+# they can be arbitrarily large.
65
+# currently it includes: 1, 2, 4, 8, 16, 24, 32, 40, ..., 8192.
66
+# the actual sizes to capture will be determined by the model,
67
+# depending on the model's max_num_seqs.
68
# NOTE: _get_graph_batch_size needs to be updated if this list is changed.
69
_BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [
66
- _BATCH_SIZE_ALIGNMENT * i for i in range(1, 33)
70
+ _BATCH_SIZE_ALIGNMENT * i for i in range(1, 1025)
71
]
72
_NUM_WARMUP_ITERS = 2
73
@@ -660,7 +664,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
664
def _use_captured_graph(self, batch_size: int,
665
max_decode_seq_len: int) -> bool:
666
return (self.decode_only and not self.runner.model_config.enforce_eager
663
- and batch_size <= _BATCH_SIZES_TO_CAPTURE[-1]
667
+ and batch_size <= self.runner.max_batchsize_to_capture
668
and max_decode_seq_len <= self.runner.max_seq_len_to_capture)
669
670
def build(self) -> ModelInputForGPU:
@@ -846,6 +850,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
850
self.sliding_window = model_config.get_sliding_window()
851
self.block_size = cache_config.block_size
852
self.max_seq_len_to_capture = self.model_config.max_seq_len_to_capture
853
+ self.max_batchsize_to_capture = _get_max_graph_batch_size(
854
+ self.scheduler_config.max_num_seqs)
855
856
self.graph_runners: List[Dict[int, CUDAGraphRunner]] = [
857
{} for _ in range(self.parallel_config.pipeline_parallel_size)
@@ -863,7 +869,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
869
# The shape of the cached block table will be
870
# (max batch size to capture, max context len to capture / block size).
871
self.graph_block_tables = np.zeros(
866
- (max(_BATCH_SIZES_TO_CAPTURE), self.get_max_block_per_batch()),
872
+ (self.max_batchsize_to_capture, self.get_max_block_per_batch()),
873
dtype=np.int32)
874
num_attn_heads = self.model_config.get_num_attention_heads(
875
self.parallel_config)
@@ -1218,7 +1224,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1224
start_time = time.perf_counter()
1225
1226
# Prepare dummy inputs. These will be reused for all batch sizes.
1221
- max_batch_size = max(_BATCH_SIZES_TO_CAPTURE)
1227
+ max_batch_size = self.max_batchsize_to_capture
1228
input_tokens = torch.zeros(max_batch_size, dtype=torch.long).cuda()
1229
input_positions = torch.zeros(max_batch_size, dtype=torch.long).cuda()
1230
@@ -1246,8 +1252,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1252
None
1253
] * self.parallel_config.pipeline_parallel_size
1254
1249
- graph_batch_size = _get_graph_batch_size(
1250
- self.scheduler_config.max_num_seqs)
1255
+ graph_batch_size = self.max_batchsize_to_capture
1256
batch_size_capture_list = [
1257
bs for bs in _BATCH_SIZES_TO_CAPTURE if bs <= graph_batch_size
1258
]
@@ -1673,3 +1678,22 @@ def _get_graph_batch_size(batch_size: int) -> int:
1678
else:
1679
return ((batch_size + _BATCH_SIZE_ALIGNMENT - 1) //
1680
_BATCH_SIZE_ALIGNMENT * _BATCH_SIZE_ALIGNMENT)
1681
+
1682
+
1683
+def _get_max_graph_batch_size(max_num_seqs: int) -> int:
1684
+ """
1685
+ max_num_seqs: Maximum number of sequences in a batch.
1686
+ _BATCH_SIZES_TO_CAPTURE: all the sizes that we want to capture.
1687
+
1688
+ pad the max_num_seqs if necessary by calling _get_graph_batch_size,
1689
+ which will deal with some edge cases like 1, 2, 4.
1690
+
1691
+ if the padded size is in _BATCH_SIZES_TO_CAPTURE, return the padded size.
1692
+ if not, it means the padded size is larger than the largest size in
1693
+ _BATCH_SIZES_TO_CAPTURE, return the largest size in _BATCH_SIZES_TO_CAPTURE.
1694
+ """
1695
+ padded_size = _get_graph_batch_size(max_num_seqs)
1696
+ if padded_size in _BATCH_SIZES_TO_CAPTURE:
1697
+ return padded_size
1698
+ assert padded_size > _BATCH_SIZES_TO_CAPTURE[-1]
1699
+ return _BATCH_SIZES_TO_CAPTURE[-1]