@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c334b1898b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

extend cuda graph size for H200 (#7894)

Co-authored-by: youkaichao <youkaichao@126.com>

kushanam committed Aug 29, 2024 at 12:15 UTC c334b1898b68812af73a6d491010d929ffdb9862
1 file changed +31 -7
vllm/worker/model_runner.py
+31 -7
@@ -60,10 +60,14 @@ logger = init_logger(__name__)
60
61 LORA_WARMUP_RANK = 8
62 _BATCH_SIZE_ALIGNMENT = 8
63 -# Capture graphs for token size 1, 2, 4, 8, 16, 24, 32, 40, ..., 256.
63 +# all the token sizes that **can** be captured by cudagraph.
64 +# they can be arbitrarily large.
65 +# currently it includes: 1, 2, 4, 8, 16, 24, 32, 40, ..., 8192.
66 +# the actual sizes to capture will be determined by the model,
67 +# depending on the model's max_num_seqs.
68 # NOTE: _get_graph_batch_size needs to be updated if this list is changed.
69 _BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [
66 - _BATCH_SIZE_ALIGNMENT * i for i in range(1, 33)
70 + _BATCH_SIZE_ALIGNMENT * i for i in range(1, 1025)
71 ]
72 _NUM_WARMUP_ITERS = 2
73
@@ -660,7 +664,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
664 def _use_captured_graph(self, batch_size: int,
665 max_decode_seq_len: int) -> bool:
666 return (self.decode_only and not self.runner.model_config.enforce_eager
663 - and batch_size <= _BATCH_SIZES_TO_CAPTURE[-1]
667 + and batch_size <= self.runner.max_batchsize_to_capture
668 and max_decode_seq_len <= self.runner.max_seq_len_to_capture)
669
670 def build(self) -> ModelInputForGPU:
@@ -846,6 +850,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
850 self.sliding_window = model_config.get_sliding_window()
851 self.block_size = cache_config.block_size
852 self.max_seq_len_to_capture = self.model_config.max_seq_len_to_capture
853 + self.max_batchsize_to_capture = _get_max_graph_batch_size(
854 + self.scheduler_config.max_num_seqs)
855
856 self.graph_runners: List[Dict[int, CUDAGraphRunner]] = [
857 {} for _ in range(self.parallel_config.pipeline_parallel_size)
@@ -863,7 +869,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
869 # The shape of the cached block table will be
870 # (max batch size to capture, max context len to capture / block size).
871 self.graph_block_tables = np.zeros(
866 - (max(_BATCH_SIZES_TO_CAPTURE), self.get_max_block_per_batch()),
872 + (self.max_batchsize_to_capture, self.get_max_block_per_batch()),
873 dtype=np.int32)
874 num_attn_heads = self.model_config.get_num_attention_heads(
875 self.parallel_config)
@@ -1218,7 +1224,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1224 start_time = time.perf_counter()
1225
1226 # Prepare dummy inputs. These will be reused for all batch sizes.
1221 - max_batch_size = max(_BATCH_SIZES_TO_CAPTURE)
1227 + max_batch_size = self.max_batchsize_to_capture
1228 input_tokens = torch.zeros(max_batch_size, dtype=torch.long).cuda()
1229 input_positions = torch.zeros(max_batch_size, dtype=torch.long).cuda()
1230
@@ -1246,8 +1252,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1252 None
1253 ] * self.parallel_config.pipeline_parallel_size
1254
1249 - graph_batch_size = _get_graph_batch_size(
1250 - self.scheduler_config.max_num_seqs)
1255 + graph_batch_size = self.max_batchsize_to_capture
1256 batch_size_capture_list = [
1257 bs for bs in _BATCH_SIZES_TO_CAPTURE if bs <= graph_batch_size
1258 ]
@@ -1673,3 +1678,22 @@ def _get_graph_batch_size(batch_size: int) -> int:
1678 else:
1679 return ((batch_size + _BATCH_SIZE_ALIGNMENT - 1) //
1680 _BATCH_SIZE_ALIGNMENT * _BATCH_SIZE_ALIGNMENT)
1681 +
1682 +
1683 +def _get_max_graph_batch_size(max_num_seqs: int) -> int:
1684 + """
1685 + max_num_seqs: Maximum number of sequences in a batch.
1686 + _BATCH_SIZES_TO_CAPTURE: all the sizes that we want to capture.
1687 +
1688 + pad the max_num_seqs if necessary by calling _get_graph_batch_size,
1689 + which will deal with some edge cases like 1, 2, 4.
1690 +
1691 + if the padded size is in _BATCH_SIZES_TO_CAPTURE, return the padded size.
1692 + if not, it means the padded size is larger than the largest size in
1693 + _BATCH_SIZES_TO_CAPTURE, return the largest size in _BATCH_SIZES_TO_CAPTURE.
1694 + """
1695 + padded_size = _get_graph_batch_size(max_num_seqs)
1696 + if padded_size in _BATCH_SIZES_TO_CAPTURE:
1697 + return padded_size
1698 + assert padded_size > _BATCH_SIZES_TO_CAPTURE[-1]
1699 + return _BATCH_SIZES_TO_CAPTURE[-1]