A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[core] clean up cudagraph batchsize padding logic (#10996)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Dec 12, 2024 at 22:57 UTC
be39e3cd18781c4571410323f3c767e67240eb51
11 files changed
+150
-104
tests/models/decoder_only/language/test_jamba.py
+3
-2
@@ -1,7 +1,7 @@
1
import pytest
2
3
from tests.utils import multi_gpu_test
4
-from vllm.config import VllmConfig
4
+from vllm.engine.arg_utils import EngineArgs
5
from vllm.sampling_params import SamplingParams
6
7
from ...utils import check_outputs_equal
@@ -189,7 +189,8 @@ def test_mamba_cache_cg_padding(
189
# This test is for verifying that mamba cache is padded to CG captured
190
# batch size. If it's not, a torch RuntimeError will be raised because
191
# tensor dimensions aren't compatible
192
- while len(example_prompts) == VllmConfig.get_graph_batch_size(
192
+ vllm_config = EngineArgs(model=model).create_engine_config()
193
+ while len(example_prompts) == vllm_config.pad_for_cudagraph(
194
len(example_prompts)):
195
example_prompts.append(example_prompts[0])
196
tests/models/decoder_only/language/test_mamba.py
+3
-2
@@ -5,7 +5,7 @@ Run `pytest tests/models/test_mamba.py`.
5
import pytest
6
from transformers import AutoModelForCausalLM, AutoTokenizer
7
8
-from vllm.config import VllmConfig
8
+from vllm.engine.arg_utils import EngineArgs
9
from vllm.sampling_params import SamplingParams
10
11
from ...utils import check_outputs_equal
@@ -200,7 +200,8 @@ def test_mamba_cache_cg_padding(
200
# This test is for verifying that mamba cache is padded to CG captured
201
# batch size. If it's not, a torch RuntimeError will be raised because
202
# tensor dimensions aren't compatible
203
- while len(example_prompts) == VllmConfig.get_graph_batch_size(
203
+ vllm_config = EngineArgs(model=model).create_engine_config()
204
+ while len(example_prompts) == vllm_config.pad_for_cudagraph(
205
len(example_prompts)):
206
example_prompts.append(example_prompts[0])
207
tests/worker/test_encoder_decoder_model_runner.py
+2
-2
@@ -4,7 +4,6 @@ from typing import List
4
import pytest
5
import torch
6
7
-from vllm.config import VllmConfig
7
from vllm.engine.arg_utils import EngineArgs
8
from vllm.platforms import current_platform
9
from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
@@ -548,7 +547,8 @@ def test_prepare_decode_cuda_graph(batch_size, multiple_seqs_per_seq_group):
547
# With CUDA Graph capture and replay enabled, the decoder and encoder
548
# input sequences will be padded. Create the expected padded tensors
549
# accordingly.
551
- graph_batch_size = VllmConfig.get_graph_batch_size(expanded_batch_size)
550
+ graph_batch_size = model_runner.vllm_config.pad_for_cudagraph(
551
+ expanded_batch_size)
552
cuda_graph_pad_size = graph_batch_size - expanded_batch_size
553
padded_seq_lens = seq_lens + list(itertools.repeat(1, cuda_graph_pad_size))
554
padded_encoder_seq_lens = encoder_seq_lens + list(
tests/worker/test_model_runner.py
+2
-2
@@ -3,7 +3,6 @@ from typing import List
3
import pytest
4
import torch
5
6
-from vllm.config import VllmConfig
6
from vllm.distributed.parallel_state import (ensure_model_parallel_initialized,
7
init_distributed_environment)
8
from vllm.engine.arg_utils import EngineArgs
@@ -177,7 +176,8 @@ def test_prepare_decode_cuda_graph(batch_size):
176
model_input.attn_metadata, model_input.attn_metadata.slot_mapping)
177
assert len(slot_mapping) == len(input_tokens)
178
180
- expected_bs = VllmConfig.get_graph_batch_size(len(seq_group_metadata_list))
179
+ expected_bs = model_runner.vllm_config.pad_for_cudagraph(
180
+ len(seq_group_metadata_list))
181
# Verify input metadata is correct for prompts.
182
device = model_runner.device
183
assert attn_metadata.num_prefills == 0
vllm/config.py
+105
-66
@@ -2354,6 +2354,12 @@ class CompilationConfig(BaseModel):
2354
# not configurable, computed after init
2355
compile_sizes: List[int] = PrivateAttr
2356
capture_sizes: List[int] = PrivateAttr
2357
+ max_capture_size: int = PrivateAttr
2358
+ # optimization:
2359
+ # Intuitively, bs_to_padded_graph_size should be Dict[int, int].
2360
+ # since we know all keys are in a range [0, max_capture_size],
2361
+ # we can optimize it to List[int] for better lookup performance.
2362
+ bs_to_padded_graph_size: List[int] = PrivateAttr
2363
2364
# keep track of enabled and disabled custom ops
2365
enabled_custom_ops: Counter[str] = PrivateAttr
@@ -2365,6 +2371,19 @@ class CompilationConfig(BaseModel):
2371
# Map from layer name to the attention cls
2372
static_forward_context: Dict[str, Any] = PrivateAttr
2373
2374
+ def __repr__(self) -> str:
2375
+ exclude = {
2376
+ "static_forward_context",
2377
+ "enabled_custom_ops",
2378
+ "disabled_custom_ops",
2379
+ "compilation_time",
2380
+ "bs_to_padded_graph_size",
2381
+ "pass_config",
2382
+ }
2383
+ return self.model_dump_json(exclude=exclude, exclude_unset=True)
2384
+
2385
+ __str__ = __repr__
2386
+
2387
@classmethod
2388
def from_cli(cls, cli_value: str) -> "CompilationConfig":
2389
"""Parse the CLI value for the compilation config."""
@@ -2450,18 +2469,22 @@ class CompilationConfig(BaseModel):
2469
2470
# sort to make sure cudagraph capture sizes are in descending order
2471
self.capture_sizes.sort(reverse=True)
2472
+ self.max_capture_size = self.capture_sizes[
2473
+ 0] if self.capture_sizes else 0
2474
2454
-
2455
-_BATCH_SIZE_ALIGNMENT = 8
2456
-# all the token sizes that **can** be captured by cudagraph.
2457
-# they can be arbitrarily large.
2458
-# currently it includes: 1, 2, 4, 8, 16, 24, 32, 40, ..., 8192.
2459
-# the actual sizes to capture will be determined by the model,
2460
-# depending on the model's max_num_seqs.
2461
-# NOTE: get_graph_batch_size needs to be updated if this list is changed.
2462
-_BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [
2463
- _BATCH_SIZE_ALIGNMENT * i for i in range(1, 1025)
2464
-]
2475
+ # pre-compute the mapping from batch size to padded graph size
2476
+ self.bs_to_padded_graph_size = [
2477
+ 0 for i in range(self.max_capture_size + 1)
2478
+ ]
2479
+ for end, start in zip(self.capture_sizes,
2480
+ self.capture_sizes[1:] + [0]):
2481
+ for bs in range(start, end):
2482
+ if bs == start:
2483
+ self.bs_to_padded_graph_size[bs] = start
2484
+ else:
2485
+ self.bs_to_padded_graph_size[bs] = end
2486
+ self.bs_to_padded_graph_size[
2487
+ self.max_capture_size] = self.max_capture_size
2488
2489
2490
@dataclass
@@ -2491,40 +2514,12 @@ class VllmConfig:
2514
init=True) # type: ignore
2515
instance_id: str = ""
2516
2494
- @staticmethod
2495
- def get_graph_batch_size(batch_size: int) -> int:
2496
- """Returns the padded batch size given actual batch size.
2497
-
2498
- Batch sizes are 1, 2, 4, _BATCH_SIZE_ALIGNMENT,
2499
- 2*_BATCH_SIZE_ALIGNMENT, 3*_BATCH_SIZE_ALIGNMENT...
2500
- """
2501
- if batch_size <= 2:
2502
- return batch_size
2503
- elif batch_size <= 4:
2504
- return 4
2505
- else:
2506
- return ((batch_size + _BATCH_SIZE_ALIGNMENT - 1) //
2507
- _BATCH_SIZE_ALIGNMENT * _BATCH_SIZE_ALIGNMENT)
2508
-
2509
- @staticmethod
2510
- def get_max_graph_batch_size(max_num_seqs: int) -> int:
2511
- """
2512
- max_num_seqs: Maximum number of sequences in a batch.
2513
- _BATCH_SIZES_TO_CAPTURE: all the sizes that we want to capture.
2514
-
2515
- pad the max_num_seqs if necessary by calling get_graph_batch_size,
2516
- which will deal with some edge cases like 1, 2, 4.
2517
-
2518
- if the padded size is in _BATCH_SIZES_TO_CAPTURE, return the padded
2519
- size. if not, it means the padded size is larger than the largest size
2520
- in _BATCH_SIZES_TO_CAPTURE, return the largest size in
2521
- _BATCH_SIZES_TO_CAPTURE.
2522
- """
2523
- padded_size = VllmConfig.get_graph_batch_size(max_num_seqs)
2524
- if padded_size in _BATCH_SIZES_TO_CAPTURE:
2525
- return padded_size
2526
- assert padded_size > _BATCH_SIZES_TO_CAPTURE[-1]
2527
- return _BATCH_SIZES_TO_CAPTURE[-1]
2517
+ def pad_for_cudagraph(self, batch_size: int) -> int:
2518
+ # if batch_size > self.compilation_config.max_capture_size,
2519
+ # it should raise an IndexError.
2520
+ # the caller should make sure the batch_size is within the range,
2521
+ # i.e., batch_size <= self.compilation_config.max_capture_size
2522
+ return self.compilation_config.bs_to_padded_graph_size[batch_size]
2523
2524
@staticmethod
2525
def _get_quantization_config(
@@ -2618,27 +2613,7 @@ class VllmConfig:
2613
self.compilation_config.pass_config.enable_reshape = False
2614
self.compilation_config.level = CompilationLevel.PIECEWISE
2615
2621
- if not envs.VLLM_USE_V1:
2622
- max_batchsize_to_capture = 0
2623
- if self.scheduler_config is not None and \
2624
- self.model_config is not None and \
2625
- not self.model_config.enforce_eager:
2626
- max_batchsize_to_capture = \
2627
- self.get_max_graph_batch_size(
2628
- self.scheduler_config.max_num_seqs)
2629
- batch_size_capture_list = [
2630
- size for size in _BATCH_SIZES_TO_CAPTURE
2631
- if size <= max_batchsize_to_capture
2632
- ]
2633
- else:
2634
- batch_size_capture_list = []
2635
- if self.model_config is not None and \
2636
- not self.model_config.enforce_eager:
2637
- batch_size_capture_list = [1, 2, 4
2638
- ] + [i for i in range(8, 513, 8)]
2639
-
2640
- self.compilation_config.init_with_cudagraph_sizes(
2641
- batch_size_capture_list)
2616
+ self._set_cudagraph_sizes()
2617
2618
if self.cache_config is not None and \
2619
self.cache_config.cpu_offload_gb > 0 and \
@@ -2659,6 +2634,70 @@ class VllmConfig:
2634
if not self.instance_id:
2635
self.instance_id = random_uuid()[:5]
2636
2637
+ def _set_cudagraph_sizes(self):
2638
+ """
2639
+ cudagraph batchsize padding logic:
2640
+
2641
+ `[1, 2, 4] + [8 * i for i in range(1, 1025)]` is a list of all possible
2642
+ batch sizes that cudagraph will capture.
2643
+
2644
+ Depending on the engine's configuration of `max_num_seqs`, the
2645
+ candidate batch sizes to capture cudagraph will shrink to the subset
2646
+ which just cover the range of `[1, max_num_seqs]`. In the common case,
2647
+ `max_num_seqs` is 256, and the cudagraph batch sizes will be
2648
+ `[1, 2, 4, 8, 16, 24, 32, 40, ..., 256]`.
2649
+
2650
+ However, if users specify the cudagraph capture sizes through
2651
+ compilation config, we will use the specified sizes instead.
2652
+
2653
+ In the end, `vllm_config.compilation_config.capture_sizes` will be the
2654
+ final sizes to capture cudagraph (in descending order).
2655
+
2656
+ During runtime, if batchsize is larger than
2657
+ `vllm_config.compilation_config.capture_sizes`,
2658
+ no cudagraph will be used.
2659
+ If the batch size is no larger than
2660
+ `vllm_config.compilation_config.capture_sizes`,
2661
+ we can quickly find the padded graph size for a given batch size by
2662
+ looking up `vllm_config.compilation_config.bs_to_padded_graph_size`.
2663
+ """
2664
+
2665
+ # calculate the default `batch_size_capture_list`
2666
+ if not envs.VLLM_USE_V1:
2667
+ batch_size_capture_list = []
2668
+ max_batchsize_to_capture = 0
2669
+ if self.scheduler_config is not None and \
2670
+ self.model_config is not None and \
2671
+ not self.model_config.enforce_eager:
2672
+
2673
+ possible_sizes = [1, 2, 4] + [8 * i for i in range(1, 1025)]
2674
+ # find the minimum size that is larger than max_num_seqs,
2675
+ # which then becomes the max_batchsize_to_capture
2676
+ larger_sizes = [
2677
+ x for x in possible_sizes
2678
+ if x >= self.scheduler_config.max_num_seqs
2679
+ ]
2680
+ if larger_sizes:
2681
+ max_batchsize_to_capture = larger_sizes[0]
2682
+ else:
2683
+ max_batchsize_to_capture = possible_sizes[-1]
2684
+
2685
+ # filter out the sizes that are
2686
+ # larger than max_batchsize_to_capture
2687
+ batch_size_capture_list = [
2688
+ size for size in possible_sizes
2689
+ if size <= max_batchsize_to_capture
2690
+ ]
2691
+ else:
2692
+ batch_size_capture_list = []
2693
+ if self.model_config is not None and \
2694
+ not self.model_config.enforce_eager:
2695
+ batch_size_capture_list = [1, 2, 4
2696
+ ] + [i for i in range(8, 513, 8)]
2697
+
2698
+ self.compilation_config.init_with_cudagraph_sizes(
2699
+ batch_size_capture_list)
2700
+
2701
def __str__(self):
2702
return (
2703
f"model={self.model_config.model!r},"
vllm/model_executor/models/jamba.py
+14
-6
@@ -7,7 +7,7 @@ from transformers import JambaConfig
7
8
from vllm.attention.backends.abstract import AttentionMetadata
9
from vllm.attention.layer import Attention
10
-from vllm.config import _BATCH_SIZES_TO_CAPTURE, CacheConfig, VllmConfig
10
+from vllm.config import CacheConfig, VllmConfig
11
from vllm.distributed import get_tensor_model_parallel_world_size
12
from vllm.distributed.parallel_state import get_pp_group
13
from vllm.model_executor.layers.fused_moe import FusedMoE
@@ -420,6 +420,17 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA, SupportsPP,
420
421
self.make_empty_intermediate_tensors = (
422
self.model.make_empty_intermediate_tensors)
423
+ if self.scheduler_config is not None and \
424
+ not self.model_config.enforce_eager:
425
+ if self.scheduler_config.max_num_seqs > \
426
+ vllm_config.compilation_config.max_capture_size:
427
+ self.max_batch_size = \
428
+ vllm_config.compilation_config.max_capture_size
429
+ else:
430
+ self.max_batch_size = vllm_config.pad_for_cudagraph(
431
+ self.scheduler_config.max_num_seqs)
432
+ else:
433
+ self.max_batch_size = 8192 + 2
434
435
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
436
return self.model.get_input_embeddings(input_ids)
@@ -433,15 +444,12 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA, SupportsPP,
444
inputs_embeds: Optional[torch.Tensor] = None,
445
**kwargs):
446
if self.mamba_cache is None:
436
- max_batch_size = (VllmConfig.get_graph_batch_size(
437
- self.scheduler_config.max_num_seqs) if self.scheduler_config
438
- else max(_BATCH_SIZES_TO_CAPTURE) + 2)
447
448
num_mamba_layers = self.model_config.get_num_layers_by_block_type(
449
self.vllm_config.parallel_config, LayerBlockType.mamba)
450
self.mamba_cache = MambaCacheManager(
443
- self.lm_head.weight.dtype, num_mamba_layers, max_batch_size,
444
- *self._get_mamba_cache_shape())
451
+ self.lm_head.weight.dtype, num_mamba_layers,
452
+ self.max_batch_size, *self._get_mamba_cache_shape())
453
(
454
mamba_cache_tensors,
455
state_indices_tensor,
vllm/model_executor/models/mamba.py
+14
-7
@@ -6,7 +6,7 @@ from torch import nn
6
from transformers import MambaConfig
7
8
from vllm.attention.backends.abstract import AttentionMetadata
9
-from vllm.config import _BATCH_SIZES_TO_CAPTURE, CacheConfig, VllmConfig
9
+from vllm.config import CacheConfig, VllmConfig
10
from vllm.distributed import get_tensor_model_parallel_world_size
11
from vllm.distributed.parallel_state import get_pp_group
12
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -195,6 +195,17 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree, SupportsPP):
195
196
self.make_empty_intermediate_tensors = (
197
self.backbone.make_empty_intermediate_tensors)
198
+ if self.scheduler_config is not None and \
199
+ not self.model_config.enforce_eager:
200
+ if self.scheduler_config.max_num_seqs > \
201
+ vllm_config.compilation_config.max_capture_size:
202
+ self.max_batch_size = \
203
+ vllm_config.compilation_config.max_capture_size
204
+ else:
205
+ self.max_batch_size = vllm_config.pad_for_cudagraph(
206
+ self.scheduler_config.max_num_seqs)
207
+ else:
208
+ self.max_batch_size = 8192 + 2
209
210
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
211
return self.backbone.get_input_embeddings(input_ids)
@@ -208,15 +219,11 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree, SupportsPP):
219
inputs_embeds: Optional[torch.Tensor] = None,
220
**kwargs):
221
if self.mamba_cache is None:
211
- max_batch_size = (VllmConfig.get_graph_batch_size(
212
- self.scheduler_config.max_num_seqs) if self.scheduler_config
213
- else max(_BATCH_SIZES_TO_CAPTURE) + 2)
214
-
222
num_mamba_layers = self.model_config.get_num_layers_by_block_type(
223
self.vllm_config.parallel_config, LayerBlockType.mamba)
224
self.mamba_cache = MambaCacheManager(
218
- self.lm_head.weight.dtype, num_mamba_layers, max_batch_size,
219
- *self._get_mamba_cache_shape())
225
+ self.lm_head.weight.dtype, num_mamba_layers,
226
+ self.max_batch_size, *self._get_mamba_cache_shape())
227
228
(
229
mamba_cache_tensors,
vllm/v1/worker/gpu_model_runner.py
+2
-9
@@ -1,6 +1,6 @@
1
import gc
2
import time
3
-from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
3
+from typing import TYPE_CHECKING, Dict, List, Tuple
4
5
import numpy as np
6
import torch
@@ -459,7 +459,7 @@ class GPUModelRunner:
459
and num_scheduled_tokens <= self.cudagraph_batch_sizes[-1]):
460
# Use piecewise CUDA graphs.
461
# Add padding to the batch size.
462
- num_input_tokens = self._get_padded_batch_size(
462
+ num_input_tokens = self.vllm_config.pad_for_cudagraph(
463
num_scheduled_tokens)
464
else:
465
# Eager mode.
@@ -641,10 +641,3 @@ class GPUModelRunner:
641
torch.zeros(kv_cache_shape,
642
dtype=self.kv_cache_dtype,
643
device=self.device))
644
-
645
- def _get_padded_batch_size(self, batch_size: int) -> Optional[int]:
646
- # TODO: Optimize this?
647
- for size in self.cudagraph_batch_sizes:
648
- if batch_size <= size:
649
- return size
650
- return None
vllm/worker/enc_dec_model_runner.py
+1
-1
@@ -464,7 +464,7 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
464
# We will be using CUDA graph replay for this decode.
465
max_len_of_block_table = self.get_max_block_per_batch()
466
batch_size = len(encoder_seq_lens)
467
- graph_batch_size = self.vllm_config.get_graph_batch_size(
467
+ graph_batch_size = self.vllm_config.pad_for_cudagraph(
468
batch_size)
469
assert graph_batch_size >= batch_size
470
cuda_graph_pad_size = graph_batch_size - batch_size
vllm/worker/model_runner.py
+4
-3
@@ -802,7 +802,8 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
802
max_encoder_seq_len):
803
return -1
804
805
- graph_batch_size = VllmConfig.get_graph_batch_size(batch_size)
805
+ graph_batch_size = self.runner.vllm_config.pad_for_cudagraph(
806
+ batch_size)
807
assert graph_batch_size >= batch_size
808
return graph_batch_size - batch_size
809
@@ -1014,8 +1015,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1015
self.sliding_window = model_config.get_sliding_window()
1016
self.block_size = cache_config.block_size
1017
self.max_seq_len_to_capture = self.model_config.max_seq_len_to_capture
1017
- self.max_batchsize_to_capture = VllmConfig.get_max_graph_batch_size(
1018
- self.scheduler_config.max_num_seqs)
1018
+ self.max_batchsize_to_capture = \
1019
+ self.vllm_config.compilation_config.max_capture_size
1020
1021
self.graph_runners: List[Dict[int, CUDAGraphRunner]] = [
1022
{} for _ in range(self.parallel_config.pipeline_parallel_size)
vllm/worker/xpu_model_runner.py
-4
@@ -37,10 +37,6 @@ if TYPE_CHECKING:
37
logger = init_logger(__name__)
38
39
_PAD_SLOT_ID = -1
40
-_BATCH_SIZE_ALIGNMENT = 8
41
-_BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [
42
- _BATCH_SIZE_ALIGNMENT * i for i in range(1, 33)
43
-]
40
41
TModelInputForXPU = TypeVar('TModelInputForXPU', bound="ModelInputForXPU")
42