A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 05:00 UTC
[Bugfix] Early-reject requests with MM data longer than encode cache capacity (#33110)
Signed-off-by: YunzhuLu <lucia.yunzhu@gmail.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io>
YunzhuLu committed
Feb 1, 2026 at 00:41 UTC
27cb2f678f7567b14a13a2f7e085137dca1a4129
1 file changed
+31
vllm/v1/engine/input_processor.py
+31
@@ -35,6 +35,7 @@ from vllm.tokenizers import TokenizerLike
35
from vllm.tokenizers.mistral import MistralTokenizer
36
from vllm.utils import length_from_prompt_token_ids_or_embeds, random_uuid
37
from vllm.utils.torch_utils import set_default_torch_num_threads
38
+from vllm.v1.core.encoder_cache_manager import compute_mm_encoder_budget
39
from vllm.v1.engine import EngineCoreRequest
40
from vllm.v1.metrics.stats import MultiModalCacheStats
41
from vllm.v1.structured_output.backend_guidance import (
@@ -68,6 +69,17 @@ class InputProcessor:
69
70
self.mm_registry = mm_registry
71
self.mm_processor_cache = mm_registry.processor_cache_from_config(vllm_config)
72
+ self.mm_encoder_cache_size = None
73
+ if (
74
+ self.mm_registry.supports_multimodal_inputs(self.model_config)
75
+ and not self.model_config.skip_tokenizer_init
76
+ ):
77
+ max_tokens_by_modality = mm_registry.get_max_tokens_per_item_by_modality(
78
+ self.model_config
79
+ )
80
+ _, self.mm_encoder_cache_size = compute_mm_encoder_budget(
81
+ self.vllm_config.scheduler_config, max_tokens_by_modality
82
+ )
83
84
self.input_preprocessor = InputPreprocessor(
85
self.model_config,
@@ -743,6 +755,25 @@ class InputProcessor:
755
f"model length of {max_prompt_len}. {suggestion}"
756
)
757
758
+ if (
759
+ prompt_type == "decoder"
760
+ and prompt_inputs["type"] == "multimodal"
761
+ and self.mm_encoder_cache_size is not None
762
+ ):
763
+ decoder_mm_positions = prompt_inputs["mm_placeholders"]
764
+ for modality, mm_positions in decoder_mm_positions.items():
765
+ for mm_position in mm_positions:
766
+ embed_length = mm_position.get_num_embeds
767
+ if embed_length > self.mm_encoder_cache_size:
768
+ raise ValueError(
769
+ f"The {prompt_type} prompt contains a(n) {modality} item "
770
+ f"with length {embed_length}, which exceeds the "
771
+ f"pre-allocated encoder cache size "
772
+ f"{self.mm_encoder_cache_size}. Please reduce the input "
773
+ f"size or increase the encoder cache size "
774
+ f"by setting --limit-mm-per-prompt at startup."
775
+ )
776
+
777
def stat_mm_cache(self) -> MultiModalCacheStats | None:
778
return self.input_preprocessor.stat_mm_cache()
779