@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 70755e819e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[V1][Core] Autotune encoder cache budget (#11895)

Signed-off-by: Roger Wang <ywang@roblox.com>

Roger Wang committed Jan 15, 2025 at 11:29 UTC 70755e819e0ae5d963dab7d81321bdfaef6d955a
6 files changed +167 -50
vllm/config.py
+10 -5
@@ -1387,13 +1387,15 @@ class SchedulerConfig:
1387
1388 is_multimodal_model: bool = False
1389
1390 - # FIXME(woosuk & ywang96): Below are placeholder values. We need to
1391 - # calculate the actual values from the configurations.
1392 - # Multimodal encoder run compute budget, only used in V1
1393 - max_num_encoder_input_tokens = 16384
1390 + # NOTE: The following multimodal encoder budget will be initialized to
1391 + # max_num_batched_tokens and overridden in case max multimodal embedding
1392 + # size is larger.
1393 + # TODO (ywang96): Make these configurable.
1394 + # Multimodal encoder compute budget, only used in V1
1395 + max_num_encoder_input_tokens: int = field(default=None) # type: ignore
1396
1397 # Multimodal encoder cache size, only used in V1
1396 - encoder_cache_size = 16384
1398 + encoder_cache_size: int = field(default=None) # type: ignore
1399
1400 # Whether to perform preemption by swapping or
1401 # recomputation. If not specified, we determine the mode as follows:
@@ -1467,6 +1469,9 @@ class SchedulerConfig:
1469 _MULTIMODAL_MODEL_MAX_NUM_BATCHED_TOKENS,
1470 )
1471
1472 + self.max_num_encoder_input_tokens = self.max_num_batched_tokens
1473 + self.encoder_cache_size = self.max_num_batched_tokens
1474 +
1475 if self.enable_chunked_prefill:
1476 logger.info(
1477 "Chunked prefill is enabled with max_num_batched_tokens=%d.",
vllm/multimodal/registry.py
+24 -5
@@ -252,11 +252,8 @@ class MultiModalRegistry:
252 model_config: "ModelConfig",
253 ) -> Mapping[str, int]:
254 """
255 - Get the maximum number of tokens per data item from each modality
256 - for profiling the memory usage of a model.
257 -
258 - Note:
259 - This is currently directly used only in V1.
255 + Get the maximum number of tokens per data item from each modality based
256 + on underlying model configuration.
257 """
258 if self.has_processor(model_config):
259 tokenizer = cached_get_tokenizer(
@@ -272,6 +269,28 @@ class MultiModalRegistry:
269 for key, plugin in self._plugins.items()
270 }
271
272 + def get_max_tokens_per_item_by_nonzero_modality(
273 + self,
274 + model_config: "ModelConfig",
275 + ) -> Mapping[str, int]:
276 + """
277 + Get the maximum number of tokens per data item from each modality based
278 + on underlying model configuration, excluding modalities that user
279 + explicitly disabled via `limit_mm_per_prompt`.
280 +
281 + Note:
282 + This is currently directly used only in V1 for profiling the memory
283 + usage of a model.
284 + """
285 + limits_per_plugin = self._limits_by_model[model_config]
286 +
287 + return {
288 + key: max_tokens_per_mm_item
289 + for key, max_tokens_per_mm_item in
290 + self.get_max_tokens_per_item_by_modality(model_config).items()
291 + if limits_per_plugin[key] > 0
292 + }
293 +
294 def get_max_tokens_by_modality(
295 self,
296 model_config: "ModelConfig",
vllm/v1/core/encoder_cache_manager.py
+77 -1
@@ -1,7 +1,14 @@
1 -from typing import Dict, List, Set, Tuple
1 +from typing import TYPE_CHECKING, Dict, List, Set, Tuple
2
3 +from vllm.logger import init_logger
4 +from vllm.multimodal import MULTIMODAL_REGISTRY
5 from vllm.v1.request import Request
6
7 +if TYPE_CHECKING:
8 + from vllm.config import ModelConfig, SchedulerConfig
9 +
10 +logger = init_logger(__name__)
11 +
12
13 class EncoderCacheManager:
14
@@ -46,3 +53,72 @@ class EncoderCacheManager:
53 freed = self.freed
54 self.freed = []
55 return freed
56 +
57 +
58 +def compute_encoder_budget(
59 + model_config: "ModelConfig",
60 + scheduler_config: "SchedulerConfig",
61 +) -> Tuple[int, int]:
62 + """Compute the encoder cache budget based on the model and scheduler
63 + configurations.
64 +
65 + Args:
66 + model_config: Model configuration.
67 + scheduler_config: Scheduler configuration.
68 +
69 + Returns:
70 + - Compute budget for encoder execution, in unit of number of tokens
71 + in the input sequence.
72 + - Space budget for encoder cache size, in unit of number of tokens
73 + in the input sequence.
74 + """
75 +
76 + if not model_config.is_multimodal_model:
77 + return 0, 0
78 +
79 + # TODO: handle encoder-decoder models once we support them.
80 + (
81 + encoder_compute_budget,
82 + encoder_cache_size,
83 + ) = _compute_encoder_budget_multimodal(model_config, scheduler_config)
84 +
85 + return encoder_compute_budget, encoder_cache_size
86 +
87 +
88 +def _compute_encoder_budget_multimodal(
89 + model_config: "ModelConfig",
90 + scheduler_config: "SchedulerConfig",
91 +) -> Tuple[int, int]:
92 + """Compute the encoder cache budget based on the model and scheduler
93 + configurations for a multimodal model.
94 +
95 + Args:
96 + model_config: Model configuration.
97 + scheduler_config: Scheduler configuration.
98 +
99 + Returns:
100 + - Compute budget for encoder execution, in unit of number of tokens
101 + in the input sequence.
102 + - Space budget for encoder cache size, in unit of number of tokens
103 + in the input sequence.
104 + """
105 +
106 + max_tokens_by_modality_dict = MULTIMODAL_REGISTRY.get_max_tokens_per_item_by_nonzero_modality( # noqa: E501
107 + model_config)
108 +
109 + if not max_tokens_by_modality_dict:
110 + logger.warning(
111 + "All non-text modalities supported by the model have been "
112 + "explicitly disabled via limit_mm_per_prompt. Encoder cache will "
113 + "not be initialized.")
114 + return 0, 0
115 +
116 + _, max_tokens_per_mm_item = max(max_tokens_by_modality_dict.items(),
117 + key=lambda item: item[1])
118 +
119 + encoder_compute_budget = max(scheduler_config.max_num_encoder_input_tokens,
120 + max_tokens_per_mm_item)
121 + encoder_cache_size = max(scheduler_config.encoder_cache_size,
122 + max_tokens_per_mm_item)
123 +
124 + return encoder_compute_budget, encoder_cache_size
vllm/v1/core/scheduler.py
+18 -8
@@ -3,10 +3,11 @@ from dataclasses import dataclass
3 from typing import (TYPE_CHECKING, Deque, Dict, Iterable, List, Optional, Set,
4 Tuple, Union)
5
6 -from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
6 +from vllm.config import CacheConfig, LoRAConfig, ModelConfig, SchedulerConfig
7 from vllm.logger import init_logger
8 from vllm.sampling_params import SamplingParams
9 -from vllm.v1.core.encoder_cache_manager import EncoderCacheManager
9 +from vllm.v1.core.encoder_cache_manager import (EncoderCacheManager,
10 + compute_encoder_budget)
11 from vllm.v1.core.kv_cache_manager import KVCacheManager
12 from vllm.v1.engine import EngineCoreOutput, EngineCoreOutputs
13 from vllm.v1.metrics.stats import SchedulerStats
@@ -25,6 +26,7 @@ class Scheduler:
26 def __init__(
27 self,
28 scheduler_config: SchedulerConfig,
29 + model_config: ModelConfig,
30 cache_config: CacheConfig,
31 lora_config: Optional[LoRAConfig],
32 ) -> None:
@@ -69,16 +71,24 @@ class Scheduler:
71 self.running_reqs_data: Dict[str, RunningRequestData] = {}
72
73 # Encoder-related.
74 + # Calculate encoder cache size if applicable
75 + # NOTE: For now we use the same budget for both compute and space.
76 + # This can be changed when we make encoder cache for embedding caching
77 + # across requests.
78 + encoder_compute_budget, encoder_cache_size = compute_encoder_budget(
79 + model_config=model_config,
80 + scheduler_config=scheduler_config,
81 + )
82 +
83 # NOTE(woosuk): Here, "encoder" includes the vision encoder (and
84 # projector if needed). Currently, we assume that the encoder also
85 # has the Transformer architecture (e.g., ViT).
75 - self.max_num_encoder_input_tokens = self.scheduler_config.max_num_encoder_input_tokens #noqa: E501
76 - # NOTE(woosuk): For the models without encoder (e.g., text-only models),
77 - # the encoder cache will not be initialized and used, regardless of
78 - # the cache size. This is because the memory space for the encoder cache
79 - # is preallocated in the profiling run.
86 + self.max_num_encoder_input_tokens = encoder_compute_budget
87 + # NOTE: For the models without encoder (e.g., text-only models),
88 + # the encoder cache will not be initialized because cache size is 0
89 + # for these models.
90 self.encoder_cache_manager = EncoderCacheManager(
81 - cache_size=self.scheduler_config.encoder_cache_size)
91 + cache_size=encoder_cache_size)
92
93 def schedule(self) -> "SchedulerOutput":
94 # NOTE(woosuk) on the scheduling algorithm:
vllm/v1/engine/core.py
+6 -3
@@ -54,9 +54,12 @@ class EngineCore:
54 vllm_config.cache_config.num_cpu_blocks = num_cpu_blocks
55
56 # Setup scheduler.
57 - self.scheduler = Scheduler(vllm_config.scheduler_config,
58 - vllm_config.cache_config,
59 - vllm_config.lora_config)
57 + self.scheduler = Scheduler(
58 + scheduler_config=vllm_config.scheduler_config,
59 + model_config=vllm_config.model_config,
60 + cache_config=vllm_config.cache_config,
61 + lora_config=vllm_config.lora_config,
62 + )
63
64 self.mm_input_mapper_server = MMInputMapperServer(
65 vllm_config.model_config)
vllm/v1/worker/gpu_model_runner.py
+32 -28
@@ -20,6 +20,7 @@ from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler,
20 is_pin_memory_available)
21 from vllm.v1.attention.backends.flash_attn import (FlashAttentionBackend,
22 FlashAttentionMetadata)
23 +from vllm.v1.core.encoder_cache_manager import compute_encoder_budget
24 from vllm.v1.engine.mm_input_mapper import MMInputMapperClient
25 from vllm.v1.outputs import ModelRunnerOutput
26 from vllm.v1.sample.metadata import SamplingMetadata
@@ -88,8 +89,12 @@ class GPUModelRunner:
89 self.mm_input_mapper_profiling = MMInputMapperClient(self.model_config)
90 self.mm_input_mapper_profiling.use_cache = False
91
91 - self.max_num_encoder_input_tokens = self.scheduler_config.max_num_encoder_input_tokens # noqa: E501
92 - self.encoder_cache_size = self.scheduler_config.encoder_cache_size
92 + encoder_compute_budget, encoder_cache_size = compute_encoder_budget(
93 + model_config=model_config,
94 + scheduler_config=scheduler_config,
95 + )
96 + self.max_num_encoder_input_tokens = encoder_compute_budget
97 + self.encoder_cache_size = encoder_cache_size
98
99 # Lazy initialization
100 # self.model: nn.Module # Set after load_model
@@ -721,44 +726,30 @@ class GPUModelRunner:
726 ]
727
728 # Profile with multimodal encoder & encoder cache.
724 - if self.is_multimodal_model:
725 -
726 - # Create dummy batch of multimodal inputs.
727 - dummy_request_data = self.input_registry.dummy_data_for_profiling(
728 - model_config=self.model_config,
729 - seq_len=self.max_num_tokens,
730 - mm_registry=self.mm_registry,
731 - )
732 - dummy_mm_data = dummy_request_data.multi_modal_data
729 + # TODO: handle encoder-decoder models once we support them.
730 + if (self.is_multimodal_model and self.max_num_encoder_input_tokens > 0
731 + and self.encoder_cache_size > 0):
732
733 # NOTE: Currently model is profiled with a single non-text
734 # modality with the max possible input tokens even when
735 # it supports multiple.
737 - max_tokens_by_modality_dict = self.mm_registry.get_max_tokens_per_item_by_modality( # noqa: E501
736 + max_tokens_by_modality_dict = MULTIMODAL_REGISTRY.get_max_tokens_per_item_by_nonzero_modality( # noqa: E501
737 self.model_config)
739 -
738 dummy_data_modality, max_tokens_per_mm_item = max(
739 max_tokens_by_modality_dict.items(), key=lambda item: item[1])
740
741 # Check how many items of this modality can be supported by
744 - # the encoder cache budget.
745 - encoder_cache_budget = min(self.max_num_encoder_input_tokens,
746 - self.encoder_cache_size)
747 - max_num_mm_items_encoder_budget = encoder_cache_budget // \
748 - max_tokens_per_mm_item
749 -
750 - # TODO: Allow users to set encoder_cache_budget in case this
751 - # happens.
752 - assert max_num_mm_items_encoder_budget > 0, (
753 - f"Encoder cache budget={encoder_cache_budget} is too small to "
754 - f"support the maximum possible size of multimodal embeddings"
755 - f"={max_tokens_per_mm_item}.")
742 + # the encoder budget.
743 + encoder_budget = min(self.max_num_encoder_input_tokens,
744 + self.encoder_cache_size)
745 +
746 + max_num_mm_items_encoder_budget = cdiv(encoder_budget,
747 + max_tokens_per_mm_item)
748
749 # Check how many items of this modality can be supported by
750 # the decoder budget.
759 - max_mm_items_per_req = max(
760 - self.mm_registry.get_mm_limits_per_prompt(
761 - self.model_config).values())
751 + max_mm_items_per_req = self.mm_registry.get_mm_limits_per_prompt(
752 + self.model_config)[dummy_data_modality]
753
754 # NOTE: We do not consider max_num_batched_tokens on purpose
755 # because the multimodal embeddings can be generated in advance
@@ -769,6 +760,19 @@ class GPUModelRunner:
760 max_num_mm_items = min(max_num_mm_items_encoder_budget,
761 max_num_mm_items_decoder_budget)
762
763 + logger.info(
764 + "Encoder cache will be initialized with a budget of %s tokens,"
765 + " and profiled with %s %s items of the maximum feature size.",
766 + encoder_budget, max_num_mm_items, dummy_data_modality)
767 +
768 + # Create dummy batch of multimodal inputs.
769 + dummy_request_data = self.input_registry.dummy_data_for_profiling(
770 + model_config=self.model_config,
771 + seq_len=self.max_num_tokens,
772 + mm_registry=self.mm_registry,
773 + )
774 + dummy_mm_data = dummy_request_data.multi_modal_data
775 +
776 # Dummy data definition in V0 may contain multiple multimodal items
777 # (e.g, multiple images) for a single request, therefore here we
778 # always replicate first item by max_num_mm_items times since in V1