A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 06:30 UTC
[Refactor] Remove `get_encoder_dummy_data` (#32241)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 13, 2026 at 17:21 UTC
eb28e8068dd870283f1a4fe8251ed9254d6f477e
6 files changed
+21
-82
vllm/model_executor/models/nemotron_parse.py
+4
-4
@@ -605,6 +605,10 @@ class NemotronParseProcessingInfo(BaseProcessingInfo):
605
**kwargs,
606
)
607
608
+ @property
609
+ def skip_prompt_length_check(self) -> bool:
610
+ return True # Because the encoder prompt is padded
611
+
612
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
613
return {"image": 1}
614
@@ -657,10 +661,6 @@ class NemotronParseMultiModalProcessor(
661
) -> str | list[int]:
662
return [0]
663
660
- @property
661
- def pad_dummy_encoder_prompt(self) -> bool:
662
- return True
663
-
664
def _call_hf_processor(
665
self,
666
prompt: str,
vllm/model_executor/models/whisper.py
+4
-4
@@ -681,6 +681,10 @@ class WhisperProcessingInfo(BaseProcessingInfo):
681
def get_hf_config(self) -> WhisperConfig:
682
return self.ctx.get_hf_config(WhisperConfig)
683
684
+ @property
685
+ def skip_prompt_length_check(self) -> bool:
686
+ return True # Because the encoder prompt is padded
687
+
688
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
689
return {"audio": 1}
690
@@ -733,10 +737,6 @@ class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo
737
target_channels=self.info.get_target_channels(),
738
)
739
736
- @property
737
- def pad_dummy_encoder_prompt(self) -> bool:
738
- return True
739
-
740
def create_encoder_prompt(
741
self,
742
prompt: str | list[int],
vllm/multimodal/processing.py
+4
-4
@@ -1396,6 +1396,10 @@ class BaseProcessingInfo:
1396
"""
1397
return self.ctx.get_hf_processor(**kwargs)
1398
1399
+ @property
1400
+ def skip_prompt_length_check(self) -> bool:
1401
+ return False
1402
+
1403
@abstractmethod
1404
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
1405
"""
@@ -2403,10 +2407,6 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
2407
"""
2408
raise NotImplementedError
2409
2406
- @property
2407
- def pad_dummy_encoder_prompt(self) -> bool:
2408
- return False
2409
-
2410
def create_decoder_prompt(
2411
self,
2412
prompt: str | list[int],
vllm/multimodal/profiling.py
+1
-25
@@ -3,7 +3,7 @@
3
from abc import ABC, abstractmethod
4
from collections.abc import Mapping
5
from dataclasses import dataclass, field
6
-from typing import Generic, NamedTuple, TypeVar, cast
6
+from typing import Generic, NamedTuple, TypeVar
7
8
import numpy as np
9
import numpy.typing as npt
@@ -19,7 +19,6 @@ from vllm.logger import init_logger
19
20
from .inputs import (
21
MultiModalDataDict,
22
- MultiModalEncDecInputs,
22
MultiModalInputs,
23
MultiModalKwargsItems,
24
MultiModalPlaceholderDict,
@@ -27,7 +26,6 @@ from .inputs import (
26
from .processing import (
27
BaseMultiModalProcessor,
28
BaseProcessingInfo,
30
- EncDecMultiModalProcessor,
29
)
30
31
logger = init_logger(__name__)
@@ -282,28 +280,6 @@ class MultiModalProfiler(Generic[_I]):
280
for modality, placeholders in placeholders_by_modality.items()
281
}
282
285
- def get_encoder_dummy_data(
286
- self,
287
- seq_len: int,
288
- mm_counts: Mapping[str, int] | None = None,
289
- mm_options: Mapping[str, BaseDummyOptions] | None = None,
290
- ) -> DummyEncoderData:
291
- mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts, mm_options)
292
- mm_inputs = cast(MultiModalEncDecInputs, mm_inputs)
293
-
294
- # For encoder-decoder models, use encoder prompt token ids instead of
295
- # decoder prompt to construct dummy seq_data for encoder profiling.
296
- encoder_prompt_token_ids = mm_inputs["encoder_prompt_token_ids"]
297
-
298
- total_len = len(encoder_prompt_token_ids)
299
-
300
- processor = cast(EncDecMultiModalProcessor, self.processor)
301
- if processor.pad_dummy_encoder_prompt:
302
- num_tokens_to_pad = max(total_len, seq_len) - total_len
303
- encoder_prompt_token_ids.extend([0] * num_tokens_to_pad)
304
-
305
- return DummyEncoderData(encoder_prompt_token_ids)
306
-
283
def get_decoder_dummy_data(
284
self,
285
seq_len: int,
vllm/multimodal/registry.py
-38
@@ -18,7 +18,6 @@ from .processing import (
18
from .profiling import (
19
BaseDummyInputsBuilder,
20
DummyDecoderData,
21
- DummyEncoderData,
21
MultiModalProfiler,
22
)
23
@@ -317,43 +316,6 @@ class MultiModalRegistry:
316
317
return dummy_data
318
320
- def get_encoder_dummy_data(
321
- self,
322
- model_config: "ModelConfig",
323
- seq_len: int,
324
- mm_counts: Mapping[str, int] | None = None,
325
- *,
326
- cache: BaseMultiModalProcessorCache | None = None,
327
- observability_config: ObservabilityConfig | None = None,
328
- ) -> DummyEncoderData:
329
- """
330
- Create dummy data for profiling the memory usage of a model.
331
-
332
- The model is identified by `model_config`.
333
- """
334
- processor = self.create_processor(
335
- model_config, observability_config, cache=cache
336
- )
337
- profiler: MultiModalProfiler = MultiModalProfiler(processor)
338
-
339
- # Extract configurable options from multimodal config.
340
- # Only include modalities that use advanced option types so legacy
341
- # count-only behavior remains unchanged.
342
- mm_options = self._extract_mm_options(model_config)
343
-
344
- dummy_data = profiler.get_encoder_dummy_data(seq_len, mm_counts, mm_options)
345
-
346
- # Having more tokens is over-conservative but otherwise fine
347
- token_ids = dummy_data.prompt_token_ids
348
- if len(token_ids) < seq_len:
349
- logger.warning_once(
350
- "Expected at least %d dummy encoder tokens for profiling, but found %d tokens instead.", # noqa: E501
351
- seq_len,
352
- len(token_ids),
353
- )
354
-
355
- return dummy_data
356
-
319
def get_encdec_max_encoder_len(self, model_config: "ModelConfig") -> int:
320
"""
321
Get the maximum length of the encoder input for encoder-decoder models.
vllm/v1/engine/input_processor.py
+8
-7
@@ -17,7 +17,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
17
from vllm.multimodal.cache import processor_cache_from_config
18
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalUUIDDict
19
from vllm.multimodal.parse import MultiModalDataParser
20
-from vllm.multimodal.processing import EncDecMultiModalProcessor, set_request_id
20
+from vllm.multimodal.processing import set_request_id
21
from vllm.multimodal.utils import argsort_mm_positions
22
from vllm.pooling_params import PoolingParams
23
from vllm.sampling_params import _SAMPLING_EPS, SamplingParams
@@ -655,17 +655,18 @@ class InputProcessor:
655
656
max_prompt_len = self.model_config.max_model_len
657
if prompt_len > max_prompt_len:
658
- if prompt_type == "encoder" and model_config.is_multimodal_model:
658
+ if model_config.is_multimodal_model:
659
mm_registry = self.input_preprocessor.mm_registry
660
- mm_processor = mm_registry.create_processor(
660
+ model_cls = mm_registry._get_model_cls(model_config)
661
+ factories = model_cls._processor_factory
662
+ ctx = mm_registry._create_processing_ctx(
663
model_config,
662
- self.vllm_config.observability_config,
664
tokenizer=tokenizer,
665
)
665
- assert isinstance(mm_processor, EncDecMultiModalProcessor)
666
+ mm_info = factories.info(ctx)
667
667
- if mm_processor.pad_dummy_encoder_prompt:
668
- return # Skip encoder length check for Whisper
668
+ if mm_info.skip_prompt_length_check:
669
+ return
670
671
if model_config.is_multimodal_model:
672
suggestion = (