@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / be39e3cd18
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[core] clean up cudagraph batchsize padding logic (#10996)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Dec 12, 2024 at 22:57 UTC be39e3cd18781c4571410323f3c767e67240eb51
11 files changed +150 -104
tests/models/decoder_only/language/test_jamba.py
+3 -2
@@ -1,7 +1,7 @@
1 import pytest
2
3 from tests.utils import multi_gpu_test
4 -from vllm.config import VllmConfig
4 +from vllm.engine.arg_utils import EngineArgs
5 from vllm.sampling_params import SamplingParams
6
7 from ...utils import check_outputs_equal
@@ -189,7 +189,8 @@ def test_mamba_cache_cg_padding(
189 # This test is for verifying that mamba cache is padded to CG captured
190 # batch size. If it's not, a torch RuntimeError will be raised because
191 # tensor dimensions aren't compatible
192 - while len(example_prompts) == VllmConfig.get_graph_batch_size(
192 + vllm_config = EngineArgs(model=model).create_engine_config()
193 + while len(example_prompts) == vllm_config.pad_for_cudagraph(
194 len(example_prompts)):
195 example_prompts.append(example_prompts[0])
196
tests/models/decoder_only/language/test_mamba.py
+3 -2
@@ -5,7 +5,7 @@ Run `pytest tests/models/test_mamba.py`.
5 import pytest
6 from transformers import AutoModelForCausalLM, AutoTokenizer
7
8 -from vllm.config import VllmConfig
8 +from vllm.engine.arg_utils import EngineArgs
9 from vllm.sampling_params import SamplingParams
10
11 from ...utils import check_outputs_equal
@@ -200,7 +200,8 @@ def test_mamba_cache_cg_padding(
200 # This test is for verifying that mamba cache is padded to CG captured
201 # batch size. If it's not, a torch RuntimeError will be raised because
202 # tensor dimensions aren't compatible
203 - while len(example_prompts) == VllmConfig.get_graph_batch_size(
203 + vllm_config = EngineArgs(model=model).create_engine_config()
204 + while len(example_prompts) == vllm_config.pad_for_cudagraph(
205 len(example_prompts)):
206 example_prompts.append(example_prompts[0])
207
tests/worker/test_encoder_decoder_model_runner.py
+2 -2
@@ -4,7 +4,6 @@ from typing import List
4 import pytest
5 import torch
6
7 -from vllm.config import VllmConfig
7 from vllm.engine.arg_utils import EngineArgs
8 from vllm.platforms import current_platform
9 from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
@@ -548,7 +547,8 @@ def test_prepare_decode_cuda_graph(batch_size, multiple_seqs_per_seq_group):
547 # With CUDA Graph capture and replay enabled, the decoder and encoder
548 # input sequences will be padded. Create the expected padded tensors
549 # accordingly.
551 - graph_batch_size = VllmConfig.get_graph_batch_size(expanded_batch_size)
550 + graph_batch_size = model_runner.vllm_config.pad_for_cudagraph(
551 + expanded_batch_size)
552 cuda_graph_pad_size = graph_batch_size - expanded_batch_size
553 padded_seq_lens = seq_lens + list(itertools.repeat(1, cuda_graph_pad_size))
554 padded_encoder_seq_lens = encoder_seq_lens + list(
tests/worker/test_model_runner.py
+2 -2
@@ -3,7 +3,6 @@ from typing import List
3 import pytest
4 import torch
5
6 -from vllm.config import VllmConfig
6 from vllm.distributed.parallel_state import (ensure_model_parallel_initialized,
7 init_distributed_environment)
8 from vllm.engine.arg_utils import EngineArgs
@@ -177,7 +176,8 @@ def test_prepare_decode_cuda_graph(batch_size):
176 model_input.attn_metadata, model_input.attn_metadata.slot_mapping)
177 assert len(slot_mapping) == len(input_tokens)
178
180 - expected_bs = VllmConfig.get_graph_batch_size(len(seq_group_metadata_list))
179 + expected_bs = model_runner.vllm_config.pad_for_cudagraph(
180 + len(seq_group_metadata_list))
181 # Verify input metadata is correct for prompts.
182 device = model_runner.device
183 assert attn_metadata.num_prefills == 0
vllm/config.py
+105 -66
@@ -2354,6 +2354,12 @@ class CompilationConfig(BaseModel):
2354 # not configurable, computed after init
2355 compile_sizes: List[int] = PrivateAttr
2356 capture_sizes: List[int] = PrivateAttr
2357 + max_capture_size: int = PrivateAttr
2358 + # optimization:
2359 + # Intuitively, bs_to_padded_graph_size should be Dict[int, int].
2360 + # since we know all keys are in a range [0, max_capture_size],
2361 + # we can optimize it to List[int] for better lookup performance.
2362 + bs_to_padded_graph_size: List[int] = PrivateAttr
2363
2364 # keep track of enabled and disabled custom ops
2365 enabled_custom_ops: Counter[str] = PrivateAttr
@@ -2365,6 +2371,19 @@ class CompilationConfig(BaseModel):
2371 # Map from layer name to the attention cls
2372 static_forward_context: Dict[str, Any] = PrivateAttr
2373
2374 + def __repr__(self) -> str:
2375 + exclude = {
2376 + "static_forward_context",
2377 + "enabled_custom_ops",
2378 + "disabled_custom_ops",
2379 + "compilation_time",
2380 + "bs_to_padded_graph_size",
2381 + "pass_config",
2382 + }
2383 + return self.model_dump_json(exclude=exclude, exclude_unset=True)
2384 +
2385 + __str__ = __repr__
2386 +
2387 @classmethod
2388 def from_cli(cls, cli_value: str) -> "CompilationConfig":
2389 """Parse the CLI value for the compilation config."""
@@ -2450,18 +2469,22 @@ class CompilationConfig(BaseModel):
2469
2470 # sort to make sure cudagraph capture sizes are in descending order
2471 self.capture_sizes.sort(reverse=True)
2472 + self.max_capture_size = self.capture_sizes[
2473 + 0] if self.capture_sizes else 0
2474
2454 -
2455 -_BATCH_SIZE_ALIGNMENT = 8
2456 -# all the token sizes that **can** be captured by cudagraph.
2457 -# they can be arbitrarily large.
2458 -# currently it includes: 1, 2, 4, 8, 16, 24, 32, 40, ..., 8192.
2459 -# the actual sizes to capture will be determined by the model,
2460 -# depending on the model's max_num_seqs.
2461 -# NOTE: get_graph_batch_size needs to be updated if this list is changed.
2462 -_BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [
2463 - _BATCH_SIZE_ALIGNMENT * i for i in range(1, 1025)
2464 -]
2475 + # pre-compute the mapping from batch size to padded graph size
2476 + self.bs_to_padded_graph_size = [
2477 + 0 for i in range(self.max_capture_size + 1)
2478 + ]
2479 + for end, start in zip(self.capture_sizes,
2480 + self.capture_sizes[1:] + [0]):
2481 + for bs in range(start, end):
2482 + if bs == start:
2483 + self.bs_to_padded_graph_size[bs] = start
2484 + else:
2485 + self.bs_to_padded_graph_size[bs] = end
2486 + self.bs_to_padded_graph_size[
2487 + self.max_capture_size] = self.max_capture_size
2488
2489
2490 @dataclass
@@ -2491,40 +2514,12 @@ class VllmConfig:
2514 init=True) # type: ignore
2515 instance_id: str = ""
2516
2494 - @staticmethod
2495 - def get_graph_batch_size(batch_size: int) -> int:
2496 - """Returns the padded batch size given actual batch size.
2497 -
2498 - Batch sizes are 1, 2, 4, _BATCH_SIZE_ALIGNMENT,
2499 - 2*_BATCH_SIZE_ALIGNMENT, 3*_BATCH_SIZE_ALIGNMENT...
2500 - """
2501 - if batch_size <= 2:
2502 - return batch_size
2503 - elif batch_size <= 4:
2504 - return 4
2505 - else:
2506 - return ((batch_size + _BATCH_SIZE_ALIGNMENT - 1) //
2507 - _BATCH_SIZE_ALIGNMENT * _BATCH_SIZE_ALIGNMENT)
2508 -
2509 - @staticmethod
2510 - def get_max_graph_batch_size(max_num_seqs: int) -> int:
2511 - """
2512 - max_num_seqs: Maximum number of sequences in a batch.
2513 - _BATCH_SIZES_TO_CAPTURE: all the sizes that we want to capture.
2514 -
2515 - pad the max_num_seqs if necessary by calling get_graph_batch_size,
2516 - which will deal with some edge cases like 1, 2, 4.
2517 -
2518 - if the padded size is in _BATCH_SIZES_TO_CAPTURE, return the padded
2519 - size. if not, it means the padded size is larger than the largest size
2520 - in _BATCH_SIZES_TO_CAPTURE, return the largest size in
2521 - _BATCH_SIZES_TO_CAPTURE.
2522 - """
2523 - padded_size = VllmConfig.get_graph_batch_size(max_num_seqs)
2524 - if padded_size in _BATCH_SIZES_TO_CAPTURE:
2525 - return padded_size
2526 - assert padded_size > _BATCH_SIZES_TO_CAPTURE[-1]
2527 - return _BATCH_SIZES_TO_CAPTURE[-1]
2517 + def pad_for_cudagraph(self, batch_size: int) -> int:
2518 + # if batch_size > self.compilation_config.max_capture_size,
2519 + # it should raise an IndexError.
2520 + # the caller should make sure the batch_size is within the range,
2521 + # i.e., batch_size <= self.compilation_config.max_capture_size
2522 + return self.compilation_config.bs_to_padded_graph_size[batch_size]
2523
2524 @staticmethod
2525 def _get_quantization_config(
@@ -2618,27 +2613,7 @@ class VllmConfig:
2613 self.compilation_config.pass_config.enable_reshape = False
2614 self.compilation_config.level = CompilationLevel.PIECEWISE
2615
2621 - if not envs.VLLM_USE_V1:
2622 - max_batchsize_to_capture = 0
2623 - if self.scheduler_config is not None and \
2624 - self.model_config is not None and \
2625 - not self.model_config.enforce_eager:
2626 - max_batchsize_to_capture = \
2627 - self.get_max_graph_batch_size(
2628 - self.scheduler_config.max_num_seqs)
2629 - batch_size_capture_list = [
2630 - size for size in _BATCH_SIZES_TO_CAPTURE
2631 - if size <= max_batchsize_to_capture
2632 - ]
2633 - else:
2634 - batch_size_capture_list = []
2635 - if self.model_config is not None and \
2636 - not self.model_config.enforce_eager:
2637 - batch_size_capture_list = [1, 2, 4
2638 - ] + [i for i in range(8, 513, 8)]
2639 -
2640 - self.compilation_config.init_with_cudagraph_sizes(
2641 - batch_size_capture_list)
2616 + self._set_cudagraph_sizes()
2617
2618 if self.cache_config is not None and \
2619 self.cache_config.cpu_offload_gb > 0 and \
@@ -2659,6 +2634,70 @@ class VllmConfig:
2634 if not self.instance_id:
2635 self.instance_id = random_uuid()[:5]
2636
2637 + def _set_cudagraph_sizes(self):
2638 + """
2639 + cudagraph batchsize padding logic:
2640 +
2641 + `[1, 2, 4] + [8 * i for i in range(1, 1025)]` is a list of all possible
2642 + batch sizes that cudagraph will capture.
2643 +
2644 + Depending on the engine's configuration of `max_num_seqs`, the
2645 + candidate batch sizes to capture cudagraph will shrink to the subset
2646 + which just cover the range of `[1, max_num_seqs]`. In the common case,
2647 + `max_num_seqs` is 256, and the cudagraph batch sizes will be
2648 + `[1, 2, 4, 8, 16, 24, 32, 40, ..., 256]`.
2649 +
2650 + However, if users specify the cudagraph capture sizes through
2651 + compilation config, we will use the specified sizes instead.
2652 +
2653 + In the end, `vllm_config.compilation_config.capture_sizes` will be the
2654 + final sizes to capture cudagraph (in descending order).
2655 +
2656 + During runtime, if batchsize is larger than
2657 + `vllm_config.compilation_config.capture_sizes`,
2658 + no cudagraph will be used.
2659 + If the batch size is no larger than
2660 + `vllm_config.compilation_config.capture_sizes`,
2661 + we can quickly find the padded graph size for a given batch size by
2662 + looking up `vllm_config.compilation_config.bs_to_padded_graph_size`.
2663 + """
2664 +
2665 + # calculate the default `batch_size_capture_list`
2666 + if not envs.VLLM_USE_V1:
2667 + batch_size_capture_list = []
2668 + max_batchsize_to_capture = 0
2669 + if self.scheduler_config is not None and \
2670 + self.model_config is not None and \
2671 + not self.model_config.enforce_eager:
2672 +
2673 + possible_sizes = [1, 2, 4] + [8 * i for i in range(1, 1025)]
2674 + # find the minimum size that is larger than max_num_seqs,
2675 + # which then becomes the max_batchsize_to_capture
2676 + larger_sizes = [
2677 + x for x in possible_sizes
2678 + if x >= self.scheduler_config.max_num_seqs
2679 + ]
2680 + if larger_sizes:
2681 + max_batchsize_to_capture = larger_sizes[0]
2682 + else:
2683 + max_batchsize_to_capture = possible_sizes[-1]
2684 +
2685 + # filter out the sizes that are
2686 + # larger than max_batchsize_to_capture
2687 + batch_size_capture_list = [
2688 + size for size in possible_sizes
2689 + if size <= max_batchsize_to_capture
2690 + ]
2691 + else:
2692 + batch_size_capture_list = []
2693 + if self.model_config is not None and \
2694 + not self.model_config.enforce_eager:
2695 + batch_size_capture_list = [1, 2, 4
2696 + ] + [i for i in range(8, 513, 8)]
2697 +
2698 + self.compilation_config.init_with_cudagraph_sizes(
2699 + batch_size_capture_list)
2700 +
2701 def __str__(self):
2702 return (
2703 f"model={self.model_config.model!r},"
vllm/model_executor/models/jamba.py
+14 -6
@@ -7,7 +7,7 @@ from transformers import JambaConfig
7
8 from vllm.attention.backends.abstract import AttentionMetadata
9 from vllm.attention.layer import Attention
10 -from vllm.config import _BATCH_SIZES_TO_CAPTURE, CacheConfig, VllmConfig
10 +from vllm.config import CacheConfig, VllmConfig
11 from vllm.distributed import get_tensor_model_parallel_world_size
12 from vllm.distributed.parallel_state import get_pp_group
13 from vllm.model_executor.layers.fused_moe import FusedMoE
@@ -420,6 +420,17 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA, SupportsPP,
420
421 self.make_empty_intermediate_tensors = (
422 self.model.make_empty_intermediate_tensors)
423 + if self.scheduler_config is not None and \
424 + not self.model_config.enforce_eager:
425 + if self.scheduler_config.max_num_seqs > \
426 + vllm_config.compilation_config.max_capture_size:
427 + self.max_batch_size = \
428 + vllm_config.compilation_config.max_capture_size
429 + else:
430 + self.max_batch_size = vllm_config.pad_for_cudagraph(
431 + self.scheduler_config.max_num_seqs)
432 + else:
433 + self.max_batch_size = 8192 + 2
434
435 def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
436 return self.model.get_input_embeddings(input_ids)
@@ -433,15 +444,12 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA, SupportsPP,
444 inputs_embeds: Optional[torch.Tensor] = None,
445 **kwargs):
446 if self.mamba_cache is None:
436 - max_batch_size = (VllmConfig.get_graph_batch_size(
437 - self.scheduler_config.max_num_seqs) if self.scheduler_config
438 - else max(_BATCH_SIZES_TO_CAPTURE) + 2)
447
448 num_mamba_layers = self.model_config.get_num_layers_by_block_type(
449 self.vllm_config.parallel_config, LayerBlockType.mamba)
450 self.mamba_cache = MambaCacheManager(
443 - self.lm_head.weight.dtype, num_mamba_layers, max_batch_size,
444 - *self._get_mamba_cache_shape())
451 + self.lm_head.weight.dtype, num_mamba_layers,
452 + self.max_batch_size, *self._get_mamba_cache_shape())
453 (
454 mamba_cache_tensors,
455 state_indices_tensor,
vllm/model_executor/models/mamba.py
+14 -7
@@ -6,7 +6,7 @@ from torch import nn
6 from transformers import MambaConfig
7
8 from vllm.attention.backends.abstract import AttentionMetadata
9 -from vllm.config import _BATCH_SIZES_TO_CAPTURE, CacheConfig, VllmConfig
9 +from vllm.config import CacheConfig, VllmConfig
10 from vllm.distributed import get_tensor_model_parallel_world_size
11 from vllm.distributed.parallel_state import get_pp_group
12 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -195,6 +195,17 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree, SupportsPP):
195
196 self.make_empty_intermediate_tensors = (
197 self.backbone.make_empty_intermediate_tensors)
198 + if self.scheduler_config is not None and \
199 + not self.model_config.enforce_eager:
200 + if self.scheduler_config.max_num_seqs > \
201 + vllm_config.compilation_config.max_capture_size:
202 + self.max_batch_size = \
203 + vllm_config.compilation_config.max_capture_size
204 + else:
205 + self.max_batch_size = vllm_config.pad_for_cudagraph(
206 + self.scheduler_config.max_num_seqs)
207 + else:
208 + self.max_batch_size = 8192 + 2
209
210 def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
211 return self.backbone.get_input_embeddings(input_ids)
@@ -208,15 +219,11 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree, SupportsPP):
219 inputs_embeds: Optional[torch.Tensor] = None,
220 **kwargs):
221 if self.mamba_cache is None:
211 - max_batch_size = (VllmConfig.get_graph_batch_size(
212 - self.scheduler_config.max_num_seqs) if self.scheduler_config
213 - else max(_BATCH_SIZES_TO_CAPTURE) + 2)
214 -
222 num_mamba_layers = self.model_config.get_num_layers_by_block_type(
223 self.vllm_config.parallel_config, LayerBlockType.mamba)
224 self.mamba_cache = MambaCacheManager(
218 - self.lm_head.weight.dtype, num_mamba_layers, max_batch_size,
219 - *self._get_mamba_cache_shape())
225 + self.lm_head.weight.dtype, num_mamba_layers,
226 + self.max_batch_size, *self._get_mamba_cache_shape())
227
228 (
229 mamba_cache_tensors,
vllm/v1/worker/gpu_model_runner.py
+2 -9
@@ -1,6 +1,6 @@
1 import gc
2 import time
3 -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
3 +from typing import TYPE_CHECKING, Dict, List, Tuple
4
5 import numpy as np
6 import torch
@@ -459,7 +459,7 @@ class GPUModelRunner:
459 and num_scheduled_tokens <= self.cudagraph_batch_sizes[-1]):
460 # Use piecewise CUDA graphs.
461 # Add padding to the batch size.
462 - num_input_tokens = self._get_padded_batch_size(
462 + num_input_tokens = self.vllm_config.pad_for_cudagraph(
463 num_scheduled_tokens)
464 else:
465 # Eager mode.
@@ -641,10 +641,3 @@ class GPUModelRunner:
641 torch.zeros(kv_cache_shape,
642 dtype=self.kv_cache_dtype,
643 device=self.device))
644 -
645 - def _get_padded_batch_size(self, batch_size: int) -> Optional[int]:
646 - # TODO: Optimize this?
647 - for size in self.cudagraph_batch_sizes:
648 - if batch_size <= size:
649 - return size
650 - return None
vllm/worker/enc_dec_model_runner.py
+1 -1
@@ -464,7 +464,7 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
464 # We will be using CUDA graph replay for this decode.
465 max_len_of_block_table = self.get_max_block_per_batch()
466 batch_size = len(encoder_seq_lens)
467 - graph_batch_size = self.vllm_config.get_graph_batch_size(
467 + graph_batch_size = self.vllm_config.pad_for_cudagraph(
468 batch_size)
469 assert graph_batch_size >= batch_size
470 cuda_graph_pad_size = graph_batch_size - batch_size
vllm/worker/model_runner.py
+4 -3
@@ -802,7 +802,8 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
802 max_encoder_seq_len):
803 return -1
804
805 - graph_batch_size = VllmConfig.get_graph_batch_size(batch_size)
805 + graph_batch_size = self.runner.vllm_config.pad_for_cudagraph(
806 + batch_size)
807 assert graph_batch_size >= batch_size
808 return graph_batch_size - batch_size
809
@@ -1014,8 +1015,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1015 self.sliding_window = model_config.get_sliding_window()
1016 self.block_size = cache_config.block_size
1017 self.max_seq_len_to_capture = self.model_config.max_seq_len_to_capture
1017 - self.max_batchsize_to_capture = VllmConfig.get_max_graph_batch_size(
1018 - self.scheduler_config.max_num_seqs)
1018 + self.max_batchsize_to_capture = \
1019 + self.vllm_config.compilation_config.max_capture_size
1020
1021 self.graph_runners: List[Dict[int, CUDAGraphRunner]] = [
1022 {} for _ in range(self.parallel_config.pipeline_parallel_size)
vllm/worker/xpu_model_runner.py
-4
@@ -37,10 +37,6 @@ if TYPE_CHECKING:
37 logger = init_logger(__name__)
38
39 _PAD_SLOT_ID = -1
40 -_BATCH_SIZE_ALIGNMENT = 8
41 -_BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [
42 - _BATCH_SIZE_ALIGNMENT * i for i in range(1, 33)
43 -]
40
41 TModelInputForXPU = TypeVar('TModelInputForXPU', bound="ModelInputForXPU")
42