@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 51550179fc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 9, 10:00 UTC

[Refactor] Define MM data parser in processing info instead of processor itself (#33260)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 29, 2026 at 13:55 UTC 51550179fc04871f58427dc325748434cc363849
34 files changed +399 -347
tests/model_executor/test_qwen3_omni.py
+1
@@ -143,6 +143,7 @@ def test_qwen3_omni_get_updates_use_audio_in_video(
143
144 # Create processing info
145 info = Qwen3OmniMoeThinkerProcessingInfo(mock_ctx)
146 + info._get_expected_hidden_size = lambda: 100
147 info.get_hf_config = Mock(return_value=mock_qwen3_omni_config)
148 info.get_hf_processor = Mock(return_value=mock_processor)
149 info.get_tokenizer = Mock(return_value=mock_tokenizer)
vllm/model_executor/models/audioflamingo3.py
+24 -21
@@ -192,6 +192,22 @@ class AudioFlamingo3MultiModalProjector(nn.Module):
192 return hidden_states
193
194
195 +class AudioFlamingo3MultiModalDataParser(MultiModalDataParser):
196 + def _parse_audio_data(
197 + self,
198 + data: dict[str, torch.Tensor] | ModalityData[Any],
199 + ) -> ModalityDataItems[Any, Any] | None:
200 + if isinstance(data, dict):
201 + return DictEmbeddingItems(
202 + data,
203 + modality="audio",
204 + required_fields={"audio_embeds"},
205 + fields_factory=_audioflamingo3_field_config,
206 + )
207 +
208 + return super()._parse_audio_data(data)
209 +
210 +
211 class AudioFlamingo3ProcessingInfo(BaseProcessingInfo):
212 def get_hf_config(self):
213 return self.ctx.get_hf_config(AudioFlamingo3Config)
@@ -204,6 +220,14 @@ class AudioFlamingo3ProcessingInfo(BaseProcessingInfo):
220 feature_extractor = hf_processor.feature_extractor
221 return feature_extractor
222
223 + def get_data_parser(self):
224 + feature_extractor = self.get_feature_extractor()
225 +
226 + return AudioFlamingo3MultiModalDataParser(
227 + target_sr=feature_extractor.sampling_rate,
228 + expected_hidden_size=self._get_expected_hidden_size(),
229 + )
230 +
231 def get_supported_mm_limits(self) -> Mapping[str, int | None]:
232 return {"audio": None}
233
@@ -259,30 +283,9 @@ def _audioflamingo3_field_config(hf_inputs: Mapping[str, torch.Tensor]):
283 )
284
285
262 -class AudioFlamingo3MultiModalDataParser(MultiModalDataParser):
263 - def _parse_audio_data(
264 - self,
265 - data: dict[str, torch.Tensor] | ModalityData[Any],
266 - ) -> ModalityDataItems[Any, Any] | None:
267 - if isinstance(data, dict):
268 - return DictEmbeddingItems(
269 - data,
270 - modality="audio",
271 - required_fields={"audio_embeds"},
272 - fields_factory=_audioflamingo3_field_config,
273 - )
274 - return super()._parse_audio_data(data)
275 -
276 -
286 class AudioFlamingo3MultiModalProcessor(
287 BaseMultiModalProcessor[AudioFlamingo3ProcessingInfo]
288 ):
280 - def _get_data_parser(self) -> MultiModalDataParser:
281 - feature_extractor = self.info.get_feature_extractor()
282 - return AudioFlamingo3MultiModalDataParser(
283 - target_sr=feature_extractor.sampling_rate
284 - )
285 -
289 def _call_hf_processor(
290 self,
291 prompt: str,
vllm/model_executor/models/aya_vision.py
+2 -4
@@ -227,10 +227,8 @@ class AyaVisionMultiModalProcessor(BaseMultiModalProcessor[AyaVisionProcessingIn
227
228 # HF processor pops the `num_patches` kwarg, which is needed by vLLM
229 if (images := mm_data.get("images")) is not None:
230 - parsed_images = (
231 - self._get_data_parser()
232 - .parse_mm_data({"image": images})
233 - .get_items("image", ImageProcessorItems)
230 + parsed_images = self.data_parser.parse_mm_data({"image": images}).get_items(
231 + "image", ImageProcessorItems
232 )
233 image_sizes = [
234 parsed_images.get_image_size(i) for i in range(len(parsed_images))
vllm/model_executor/models/cohere2_vision.py
+2 -4
@@ -262,10 +262,8 @@ class Cohere2VisionMultiModalProcessor(
262 hf_processor = self.info.get_hf_processor(**mm_kwargs)
263
264 # Fallback calculation if HF processor didn't provide num_patches
265 - parsed_images = (
266 - self._get_data_parser()
267 - .parse_mm_data({"image": images})
268 - .get_items("image", ImageProcessorItems)
265 + parsed_images = self.data_parser.parse_mm_data({"image": images}).get_items(
266 + "image", ImageProcessorItems
267 )
268
269 num_patches = [
vllm/model_executor/models/ernie45_vl.py
+6 -5
@@ -793,6 +793,12 @@ class Ernie4_5_VLProcessingInfo(BaseProcessingInfo):
793 def get_image_processor(self, **kwargs: object):
794 return self.get_hf_processor(**kwargs).image_processor
795
796 + def get_data_parser(self):
797 + return MultiModalDataParser(
798 + video_needs_metadata=True,
799 + expected_hidden_size=self._get_expected_hidden_size(),
800 + )
801 +
802 def get_supported_mm_limits(self) -> Mapping[str, int | None]:
803 return {"image": None, "video": None}
804
@@ -947,11 +953,6 @@ class Ernie4_5_VLProcessingInfo(BaseProcessingInfo):
953
954
955 class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessingInfo]):
950 - def _get_data_parser(self) -> MultiModalDataParser:
951 - return MultiModalDataParser(
952 - video_needs_metadata=True,
953 - )
954 -
956 def _pixel_values_norm(
957 self,
958 pixel_values: torch.Tensor,
vllm/model_executor/models/funaudiochat.py
+23 -23
@@ -552,6 +552,29 @@ class FunAudioChatDiscreteEncoder(nn.Module):
552 class FunAudioChatProcessingInfo(BaseProcessingInfo):
553 token_fps: int = 25
554
555 + @cached_property
556 + def feature_extractor(self) -> WhisperFeatureExtractor:
557 + return WhisperFeatureExtractor.from_pretrained(self.model_id)
558 +
559 + @cached_property
560 + def speech_tokenizer(self) -> PreTrainedTokenizerFast:
561 + return PreTrainedTokenizerFast.from_pretrained(
562 + self.model_id, subfolder="speech_tokenizer"
563 + )
564 +
565 + def get_feature_extractor(self) -> WhisperFeatureExtractor:
566 + return self.feature_extractor
567 +
568 + def get_speech_tokenizer(self) -> PreTrainedTokenizerFast:
569 + return self.speech_tokenizer
570 +
571 + def get_data_parser(self):
572 + return MultiModalDataParser(
573 + target_sr=int(self.feature_extractor.sampling_rate),
574 + target_channels=self.get_target_channels(),
575 + expected_hidden_size=self._get_expected_hidden_size(),
576 + )
577 +
578 def get_supported_mm_limits(self) -> Mapping[str, int | None]:
579 return {"audio": None}
580
@@ -570,22 +593,6 @@ class FunAudioChatProcessingInfo(BaseProcessingInfo):
593 max_audio_tokens = int(getattr(audio_cfg, "max_source_positions", 1500))
594 return {"audio": max_audio_tokens}
595
573 - @cached_property
574 - def feature_extractor(self) -> WhisperFeatureExtractor:
575 - return WhisperFeatureExtractor.from_pretrained(self.model_id)
576 -
577 - @cached_property
578 - def speech_tokenizer(self) -> PreTrainedTokenizerFast:
579 - return PreTrainedTokenizerFast.from_pretrained(
580 - self.model_id, subfolder="speech_tokenizer"
581 - )
582 -
583 - def get_feature_extractor(self) -> WhisperFeatureExtractor:
584 - return self.feature_extractor
585 -
586 - def get_speech_tokenizer(self) -> PreTrainedTokenizerFast:
587 - return self.speech_tokenizer
588 -
596 def get_audio_group_size(self) -> int:
597 cfg = self.get_hf_config()
598 audio_cfg = getattr(cfg, "audio_config", None)
@@ -635,13 +642,6 @@ class FunAudioChatDummyInputsBuilder(
642 class FunAudioChatMultiModalProcessor(
643 BaseMultiModalProcessor[FunAudioChatProcessingInfo]
644 ):
638 - def _get_data_parser(self) -> MultiModalDataParser:
639 - feature_extractor = self.info.get_feature_extractor()
640 - return MultiModalDataParser(
641 - target_sr=int(feature_extractor.sampling_rate),
642 - target_channels=self.info.get_target_channels(),
643 - )
644 -
645 def _call_hf_processor(
646 self,
647 prompt: str,
vllm/model_executor/models/gemma3_mm.py
+2 -4
@@ -290,10 +290,8 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
290
291 # HF processor pops the `num_crops` kwarg, which is needed by vLLM
292 if (images := mm_data.get("images")) is not None:
293 - parsed_images = (
294 - self._get_data_parser()
295 - .parse_mm_data({"image": images})
296 - .get_items("image", ImageProcessorItems)
293 + parsed_images = self.data_parser.parse_mm_data({"image": images}).get_items(
294 + "image", ImageProcessorItems
295 )
296 image_sizes = [
297 parsed_images.get_image_size(i) for i in range(len(parsed_images))
vllm/model_executor/models/gemma3n_mm.py
+11 -4
@@ -107,6 +107,17 @@ class Gemma3nProcessingInfo(BaseProcessingInfo):
107 def get_hf_processor(self, **kwargs: object):
108 return self.ctx.get_hf_processor(Gemma3nProcessor, **kwargs)
109
110 + def get_feature_extractor(self, **kwargs: object) -> Gemma3nAudioFeatureExtractor:
111 + return self.get_hf_processor(**kwargs).feature_extractor
112 +
113 + def get_data_parser(self):
114 + feature_extractor = self.get_feature_extractor()
115 +
116 + return MultiModalDataParser(
117 + target_sr=feature_extractor.sampling_rate,
118 + expected_hidden_size=self._get_expected_hidden_size(),
119 + )
120 +
121 def get_supported_mm_limits(self) -> Mapping[str, int | None]:
122 return {"image": None, "audio": None}
123
@@ -200,10 +211,6 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]):
211
212
213 class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo]):
203 - def _get_data_parser(self) -> MultiModalDataParser:
204 - feature_extractor = self.info.get_hf_processor().feature_extractor
205 - return MultiModalDataParser(target_sr=feature_extractor.sampling_rate)
206 -
214 def _call_hf_processor(
215 self,
216 prompt: str,
vllm/model_executor/models/glm4_1v.py
+6 -3
@@ -822,6 +822,12 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
822 def get_video_processor(self, **kwargs: object) -> Glm4vVideoProcessor:
823 return self.get_hf_processor(**kwargs).video_processor
824
825 + def get_data_parser(self):
826 + return MultiModalDataParser(
827 + video_needs_metadata=True,
828 + expected_hidden_size=self._get_expected_hidden_size(),
829 + )
830 +
831 def _get_vision_info(
832 self,
833 *,
@@ -1222,9 +1228,6 @@ class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]):
1228
1229
1230 class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
1225 - def _get_data_parser(self) -> MultiModalDataParser:
1226 - return MultiModalDataParser(video_needs_metadata=True)
1227 -
1231 def _call_hf_processor(
1232 self,
1233 prompt: str,
vllm/model_executor/models/glmasr.py
+66 -62
@@ -620,64 +620,6 @@ class GlmAsrMultiModalProjector(nn.Module):
620 return hidden_states
621
622
623 -class GlmAsrProcessingInfo(BaseProcessingInfo):
624 - """
625 - Processing information provider for GLM-ASR model.
626 -
627 - Provides access to model configuration, processor, and feature extractor
628 - needed for audio preprocessing and multimodal integration.
629 - """
630 -
631 - def get_hf_config(self) -> GlmAsrConfig:
632 - return self.ctx.get_hf_config(GlmAsrConfig)
633 -
634 - def get_hf_processor(self, **kwargs: object) -> GlmAsrProcessor:
635 - return self.ctx.get_hf_processor(GlmAsrProcessor, **kwargs)
636 -
637 - def get_feature_extractor(self, **kwargs: object) -> WhisperFeatureExtractor:
638 - return self.get_hf_processor(**kwargs).feature_extractor
639 -
640 - def get_supported_mm_limits(self) -> Mapping[str, int | None]:
641 - return {"audio": None}
642 -
643 -
644 -class GlmAsrDummyInputsBuilder(BaseDummyInputsBuilder[GlmAsrProcessingInfo]):
645 - """
646 - Builder for dummy inputs used in profiling and testing.
647 -
648 - Generates dummy text prompts and audio data that match the expected
649 - format for GLM-ASR model inputs. Used for memory profiling and
650 - performance benchmarking.
651 - """
652 -
653 - def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
654 - num_audios = mm_counts.get("audio", 0)
655 - hf_processor = self.info.get_hf_processor()
656 - return hf_processor.audio_token * num_audios
657 -
658 - def get_dummy_mm_data(
659 - self,
660 - seq_len: int,
661 - mm_counts: Mapping[str, int],
662 - mm_options: Mapping[str, BaseDummyOptions] | None = None,
663 - ) -> MultiModalDataDict:
664 - feature_extractor = self.info.get_feature_extractor()
665 - sampling_rate = feature_extractor.sampling_rate
666 - num_audios = mm_counts.get("audio", 0)
667 - audio_overrides = mm_options.get("audio") if mm_options else None
668 -
669 - max_audio_len = getattr(
670 - self.info.get_hf_processor(), "max_audio_len", DEFAULT_MAX_AUDIO_LEN_S
671 - )
672 - audio_len = int(max_audio_len * sampling_rate)
673 -
674 - return {
675 - "audio": self._get_dummy_audios(
676 - length=audio_len, num_audios=num_audios, overrides=audio_overrides
677 - )
678 - }
679 -
680 -
623 def _glmasr_field_config(
624 hf_inputs: Mapping[str, torch.Tensor],
625 ) -> dict[str, MultiModalFieldConfig]:
@@ -737,16 +679,78 @@ class GlmAsrMultiModalDataParser(MultiModalDataParser):
679 return super()._parse_audio_data(data)
680
681
682 +class GlmAsrProcessingInfo(BaseProcessingInfo):
683 + """
684 + Processing information provider for GLM-ASR model.
685 +
686 + Provides access to model configuration, processor, and feature extractor
687 + needed for audio preprocessing and multimodal integration.
688 + """
689 +
690 + def get_hf_config(self) -> GlmAsrConfig:
691 + return self.ctx.get_hf_config(GlmAsrConfig)
692 +
693 + def get_hf_processor(self, **kwargs: object) -> GlmAsrProcessor:
694 + return self.ctx.get_hf_processor(GlmAsrProcessor, **kwargs)
695 +
696 + def get_feature_extractor(self, **kwargs: object) -> WhisperFeatureExtractor:
697 + return self.get_hf_processor(**kwargs).feature_extractor
698 +
699 + def get_data_parser(self):
700 + feature_extractor = self.get_feature_extractor()
701 +
702 + return GlmAsrMultiModalDataParser(
703 + target_sr=feature_extractor.sampling_rate,
704 + expected_hidden_size=self._get_expected_hidden_size(),
705 + )
706 +
707 + def get_supported_mm_limits(self) -> Mapping[str, int | None]:
708 + return {"audio": None}
709 +
710 +
711 +class GlmAsrDummyInputsBuilder(BaseDummyInputsBuilder[GlmAsrProcessingInfo]):
712 + """
713 + Builder for dummy inputs used in profiling and testing.
714 +
715 + Generates dummy text prompts and audio data that match the expected
716 + format for GLM-ASR model inputs. Used for memory profiling and
717 + performance benchmarking.
718 + """
719 +
720 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
721 + num_audios = mm_counts.get("audio", 0)
722 + hf_processor = self.info.get_hf_processor()
723 + return hf_processor.audio_token * num_audios
724 +
725 + def get_dummy_mm_data(
726 + self,
727 + seq_len: int,
728 + mm_counts: Mapping[str, int],
729 + mm_options: Mapping[str, BaseDummyOptions] | None = None,
730 + ) -> MultiModalDataDict:
731 + feature_extractor = self.info.get_feature_extractor()
732 + sampling_rate = feature_extractor.sampling_rate
733 + num_audios = mm_counts.get("audio", 0)
734 + audio_overrides = mm_options.get("audio") if mm_options else None
735 +
736 + max_audio_len = getattr(
737 + self.info.get_hf_processor(), "max_audio_len", DEFAULT_MAX_AUDIO_LEN_S
738 + )
739 + audio_len = int(max_audio_len * sampling_rate)
740 +
741 + return {
742 + "audio": self._get_dummy_audios(
743 + length=audio_len, num_audios=num_audios, overrides=audio_overrides
744 + )
745 + }
746 +
747 +
748 class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"]):
749 """
750 GLM-ASR processor that inherits directly from BaseMultiModalProcessor
751 for better performance and cleaner implementation.
752 """
753
746 - def _get_data_parser(self) -> MultiModalDataParser:
747 - feature_extractor = self.info.get_feature_extractor()
748 - return GlmAsrMultiModalDataParser(target_sr=feature_extractor.sampling_rate)
749 -
754 def _calculate_chunk_counts(
755 self,
756 audio_list: list[Any],
vllm/model_executor/models/granite_speech.py
+8 -5
@@ -109,6 +109,14 @@ class GraniteSpeechAudioInputs(TensorSchema):
109
110
111 class GraniteSpeechMultiModalProcessingInfo(BaseProcessingInfo):
112 + def get_data_parser(self):
113 + feature_extractor = self.get_hf_processor().audio_processor
114 +
115 + return MultiModalDataParser(
116 + target_sr=feature_extractor.melspec_kwargs["sample_rate"],
117 + expected_hidden_size=self._get_expected_hidden_size(),
118 + )
119 +
120 def get_supported_mm_limits(self) -> Mapping[str, int | None]:
121 return {"audio": 1}
122
@@ -127,11 +135,6 @@ class GraniteSpeechMultiModalProcessingInfo(BaseProcessingInfo):
135 class GraniteSpeechMultiModalProcessor(
136 BaseMultiModalProcessor[GraniteSpeechMultiModalProcessingInfo]
137 ):
130 - def _get_data_parser(self) -> MultiModalDataParser:
131 - feature_extractor = self.info.get_hf_processor().audio_processor
132 - sampling_rate = feature_extractor.melspec_kwargs["sample_rate"]
133 - return MultiModalDataParser(target_sr=sampling_rate)
134 -
138 def _get_mm_fields_config(
139 self,
140 hf_inputs: BatchFeature,
vllm/model_executor/models/hunyuan_vision.py
+5 -3
@@ -599,6 +599,11 @@ class HunYuanVLProcessingInfo(BaseProcessingInfo):
599 ) -> HunYuanVLProcessor:
600 return self.get_hf_processor(**kwargs).image_processor
601
602 + def get_data_parser(self):
603 + return HunYuanVLMultiModalDataParser(
604 + expected_hidden_size=self._get_expected_hidden_size(),
605 + )
606 +
607 def get_supported_mm_limits(self) -> Mapping[str, int | None]:
608 return {"image": None}
609
@@ -710,9 +715,6 @@ class HunYuanVLDummyInputsBuilder(BaseDummyInputsBuilder[HunYuanVLProcessingInfo
715
716
717 class HunYuanVLMultiModalProcessor(BaseMultiModalProcessor[HunYuanVLProcessingInfo]):
713 - def _get_data_parser(self) -> MultiModalDataParser:
714 - return HunYuanVLMultiModalDataParser()
715 -
718 def _call_hf_processor(
719 self,
720 prompt: str,
vllm/model_executor/models/idefics3.py
+2 -4
@@ -349,10 +349,8 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo
349 tok_kwargs,
350 )
351
352 - parsed_images = (
353 - self._get_data_parser()
354 - .parse_mm_data({"image": images})
355 - .get_items("image", ImageProcessorItems)
352 + parsed_images = self.data_parser.parse_mm_data({"image": images}).get_items(
353 + "image", ImageProcessorItems
354 )
355 image_sizes = [
356 parsed_images.get_image_size(i) for i in range(len(parsed_images))
vllm/model_executor/models/keye.py
+7 -4
@@ -984,6 +984,11 @@ class KeyeProcessingInfo(BaseProcessingInfo):
984 def get_image_processor(self, **kwargs: object):
985 return self.get_hf_processor(**kwargs).image_processor
986
987 + def get_data_parser(self):
988 + return KeyeMultiModalDataParser(
989 + expected_hidden_size=self._get_expected_hidden_size(),
990 + )
991 +
992 def get_supported_mm_limits(
993 self,
994 ) -> Mapping[str, int | None]:
@@ -1183,13 +1188,11 @@ class KeyeBaseDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
1188 return mm_data
1189
1190
1186 -class KeyeDummyInputsBuilder(KeyeBaseDummyInputsBuilder[KeyeProcessingInfo]): ...
1191 +class KeyeDummyInputsBuilder(KeyeBaseDummyInputsBuilder[KeyeProcessingInfo]):
1192 + pass
1193
1194
1195 class KeyeMultiModalProcessor(BaseMultiModalProcessor[KeyeProcessingInfo]):
1190 - def _get_data_parser(self) -> MultiModalDataParser:
1191 - return KeyeMultiModalDataParser()
1192 -
1196 def _get_prompt_updates(
1197 self,
1198 mm_items: MultiModalDataItems,
vllm/model_executor/models/keye_vl1_5.py
+15 -13
@@ -274,16 +274,6 @@ class KeyeVL1_5Projector(nn.Module):
274 return hidden_states.view(*dims, -1)
275
276
277 -class KeyeVL1_5ProcessingInfo(KeyeProcessingInfo):
278 - def get_max_frame_per_video(self) -> int:
279 - return 2048
280 -
281 - def get_supported_mm_limits(
282 - self,
283 - ) -> Mapping[str, int | None]:
284 - return {"image": None, "video": 1}
285 -
286 -
277 def _keye_field_config(
278 hf_inputs: Mapping[str, torch.Tensor],
279 ):
@@ -365,10 +355,22 @@ class KeyeVL1_5MultiModalDataParser(MultiModalDataParser):
355 return super()._parse_video_data(data)
356
357
368 -class KeyeVL1_5MultiModalProcessor(BaseMultiModalProcessor[KeyeVL1_5ProcessingInfo]):
369 - def _get_data_parser(self) -> MultiModalDataParser:
370 - return KeyeVL1_5MultiModalDataParser()
358 +class KeyeVL1_5ProcessingInfo(KeyeProcessingInfo):
359 + def get_data_parser(self):
360 + return KeyeVL1_5MultiModalDataParser(
361 + expected_hidden_size=self._get_expected_hidden_size(),
362 + )
363 +
364 + def get_max_frame_per_video(self) -> int:
365 + return 2048
366
367 + def get_supported_mm_limits(
368 + self,
369 + ) -> Mapping[str, int | None]:
370 + return {"image": None, "video": 1}
371 +
372 +
373 +class KeyeVL1_5MultiModalProcessor(BaseMultiModalProcessor[KeyeVL1_5ProcessingInfo]):
374 def _get_prompt_updates(
375 self,
376 mm_items: MultiModalDataItems,
vllm/model_executor/models/lfm2_vl.py
+2 -4
@@ -354,10 +354,8 @@ class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):
354 tok_kwargs,
355 )
356
357 - parsed_images = (
358 - self._get_data_parser()
359 - .parse_mm_data({"image": images})
360 - .get_items("image", ImageProcessorItems)
357 + parsed_images = self.data_parser.parse_mm_data({"image": images}).get_items(
358 + "image", ImageProcessorItems
359 )
360 image_sizes = [
361 parsed_images.get_image_size(i) for i in range(len(parsed_images))
vllm/model_executor/models/midashenglm.py
+8 -4
@@ -531,6 +531,14 @@ class MiDashengLMProcessingInfo(BaseProcessingInfo):
531 feature_extractor = hf_processor.feature_extractor
532 return feature_extractor
533
534 + def get_data_parser(self):
535 + feature_extractor = self.get_feature_extractor()
536 +
537 + return MultiModalDataParser(
538 + target_sr=feature_extractor.sampling_rate,
539 + expected_hidden_size=self._get_expected_hidden_size(),
540 + )
541 +
542 def get_supported_mm_limits(self) -> Mapping[str, int | None]:
543 return {"audio": None}
544
@@ -575,10 +583,6 @@ class MiDashengLMDummyInputsBuilder(BaseDummyInputsBuilder[MiDashengLMProcessing
583 class MiDashengLMMultiModalProcessor(
584 BaseMultiModalProcessor[MiDashengLMProcessingInfo]
585 ):
578 - def _get_data_parser(self) -> MultiModalDataParser:
579 - feature_extractor = self.info.get_feature_extractor()
580 - return MultiModalDataParser(target_sr=feature_extractor.sampling_rate)
581 -
586 def _call_hf_processor(
587 self,
588 prompt: str,
vllm/model_executor/models/minicpmo.py
+8 -10
@@ -53,7 +53,6 @@ from vllm.multimodal.parse import (
53 ModalityData,
54 ModalityDataItems,
55 MultiModalDataItems,
56 - MultiModalDataParser,
56 )
57 from vllm.multimodal.processing import (
58 PromptReplacement,
@@ -174,6 +173,12 @@ class MiniCPMOMultiModalDataParser(MiniCPMVMultiModalDataParser):
173 class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):
174 audio_pattern = "(<audio>./</audio>)"
175
176 + def get_data_parser(self):
177 + return MiniCPMOMultiModalDataParser(
178 + target_sr=self.get_default_audio_sampling_rate(),
179 + expected_hidden_size=self._get_expected_hidden_size(),
180 + )
181 +
182 def get_supported_mm_limits(self) -> Mapping[str, int | None]:
183 return {**super().get_supported_mm_limits(), "audio": None}
184
@@ -274,11 +279,6 @@ class MiniCPMODummyInputsBuilder(MiniCPMVDummyInputsBuilder[MiniCPMOProcessingIn
279
280
281 class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessingInfo]):
277 - def _get_data_parser(self) -> MultiModalDataParser:
278 - return MiniCPMOMultiModalDataParser(
279 - target_sr=self.info.get_default_audio_sampling_rate()
280 - )
281 -
282 def get_audio_prompt_texts(
283 self,
284 audio_lens: int,
@@ -300,10 +300,8 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
300 if (audios := mm_data.get("audios")) is None:
301 return {}
302
303 - parsed_audios = (
304 - self._get_data_parser()
305 - .parse_mm_data({"audio": audios})
306 - .get_items("audio", (MiniCPMOAudioEmbeddingItems, AudioProcessorItems))
303 + parsed_audios = self.data_parser.parse_mm_data({"audio": audios}).get_items(
304 + "audio", (MiniCPMOAudioEmbeddingItems, AudioProcessorItems)
305 )
306
307 if isinstance(parsed_audios, MiniCPMOAudioEmbeddingItems):
vllm/model_executor/models/minicpmv.py
+9 -11
@@ -557,6 +557,11 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
557 def get_image_processor(self, **kwargs: object):
558 return self.get_hf_processor(**kwargs).image_processor
559
560 + def get_data_parser(self):
561 + return MiniCPMVMultiModalDataParser(
562 + expected_hidden_size=self._get_expected_hidden_size(),
563 + )
564 +
565 def get_model_version(self):
566 return get_version_by_config(self.get_hf_config())
567
@@ -736,9 +741,6 @@ class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
741
742
743 class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
739 - def _get_data_parser(self) -> MultiModalDataParser:
740 - return MiniCPMVMultiModalDataParser()
741 -
744 def get_image_prompt_texts(self, image_size: ImageSize, image_idx: int = 0) -> str:
745 return self.info.get_slice_image_placeholder(
746 image_size,
@@ -765,10 +767,8 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
767 if (images := mm_data.get("images")) is None:
768 return {}
769
768 - parsed_images = (
769 - self._get_data_parser()
770 - .parse_mm_data({"image": images})
771 - .get_items("image", (MiniCPMVImageEmbeddingItems, ImageProcessorItems))
770 + parsed_images = self.data_parser.parse_mm_data({"image": images}).get_items(
771 + "image", (MiniCPMVImageEmbeddingItems, ImageProcessorItems)
772 )
773
774 if isinstance(parsed_images, MiniCPMVImageEmbeddingItems):
@@ -793,10 +793,8 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
793 if (videos := mm_data.get("videos")) is None:
794 return {}
795
796 - parsed_videos = (
797 - self._get_data_parser()
798 - .parse_mm_data({"video": videos})
799 - .get_items("video", (MiniCPMVVideoEmbeddingItems, VideoProcessorItems))
796 + parsed_videos = self.data_parser.parse_mm_data({"video": videos}).get_items(
797 + "video", (MiniCPMVVideoEmbeddingItems, VideoProcessorItems)
798 )
799
800 if isinstance(parsed_videos, MiniCPMVVideoEmbeddingItems):
vllm/model_executor/models/mllama4.py
+2 -4
@@ -620,10 +620,8 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo])
620 )
621
622 images = mm_data["images"]
623 - parsed_images = (
624 - self._get_data_parser()
625 - .parse_mm_data({"image": images})
626 - .get_items("image", ImageProcessorItems)
623 + parsed_images = self.data_parser.parse_mm_data({"image": images}).get_items(
624 + "image", ImageProcessorItems
625 )
626
627 tile_size = vision_config.image_size
vllm/model_executor/models/molmo2.py
+6 -3
@@ -1860,6 +1860,12 @@ def get_frame_times_and_chosen_fps(
1860
1861
1862 class Molmo2ProcessingInfo(BaseProcessingInfo):
1863 + def get_data_parser(self):
1864 + return MultiModalDataParser(
1865 + video_needs_metadata=True,
1866 + expected_hidden_size=self._get_expected_hidden_size(),
1867 + )
1868 +
1869 def get_hf_processor(self, **kwargs: object) -> Molmo2ProcessorWrapper:
1870 processor = self.ctx.get_hf_processor(**kwargs)
1871 hf_config = self.ctx.get_hf_config()
@@ -2183,9 +2189,6 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
2189
2190 return prompt_tokens
2191
2186 - def _get_data_parser(self) -> MultiModalDataParser:
2187 - return MultiModalDataParser(video_needs_metadata=True)
2188 -
2192 def _call_hf_processor(
2193 self,
2194 prompt: str,
vllm/model_executor/models/nano_nemotron_vl.py
+6 -3
@@ -1143,6 +1143,12 @@ class NanoNemotronVLProcessingInfo(BaseNanoNemotronVLProcessingInfo):
1143 def supports_video(self):
1144 return self.get_hf_processor().supports_video
1145
1146 + def get_data_parser(self):
1147 + return MultiModalDataParser(
1148 + video_needs_metadata=True,
1149 + expected_hidden_size=self._get_expected_hidden_size(),
1150 + )
1151 +
1152 def get_supported_mm_limits(self):
1153 video_limit = {"video": None} if self.supports_video else {}
1154 return {**super().get_supported_mm_limits(), **video_limit}
@@ -1274,9 +1280,6 @@ class NanoNemotronVLMultiModalProcessor(
1280 ):
1281 """MultiModalProcessor extended for video support"""
1282
1277 - def _get_data_parser(self) -> MultiModalDataParser:
1278 - return MultiModalDataParser(video_needs_metadata=True)
1279 -
1283 def _get_mm_fields_config(
1284 self,
1285 hf_inputs: BatchFeature,
vllm/model_executor/models/opencua.py
+7 -6
@@ -25,7 +25,7 @@ from vllm.multimodal.inputs import (
25 MultiModalFieldConfig,
26 MultiModalKwargsItems,
27 )
28 -from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
28 +from vllm.multimodal.parse import MultiModalDataItems
29 from vllm.multimodal.processing import (
30 BaseMultiModalProcessor,
31 PromptReplacement,
@@ -53,6 +53,12 @@ from .utils import (
53
54
55 class OpenCUAProcessingInfo(Qwen2VLProcessingInfo):
56 + def get_data_parser(self):
57 + return Qwen2VLMultiModalDataParser(
58 + self.get_hf_config().vision_config.spatial_merge_size,
59 + expected_hidden_size=self._get_expected_hidden_size(),
60 + )
61 +
62 def get_hf_config(self):
63 return self.ctx.get_hf_config()
64
@@ -125,11 +131,6 @@ class OpenCUAProcessor(Qwen2VLProcessor):
131
132
133 class OpenCUAMultiModalProcessor(BaseMultiModalProcessor[OpenCUAProcessingInfo]):
128 - def _get_data_parser(self) -> MultiModalDataParser:
129 - return Qwen2VLMultiModalDataParser(
130 - self.info.get_hf_config().vision_config.spatial_merge_size
131 - )
132 -
134 def _get_mm_fields_config(
135 self,
136 hf_inputs: BatchFeature,
vllm/model_executor/models/phi4mm.py
+9 -6
@@ -568,6 +568,15 @@ class Phi4MMProcessingInfo(BaseProcessingInfo):
568 def get_feature_extractor(self, **kwargs: object) -> SequenceFeatureExtractor:
569 return self.get_hf_processor(**kwargs).audio_processor
570
571 + def get_data_parser(self):
572 + feature_extractor = self.get_feature_extractor()
573 +
574 + return MultiModalDataParser(
575 + target_sr=feature_extractor.sampling_rate,
576 + audio_resample_method="scipy",
577 + expected_hidden_size=self._get_expected_hidden_size(),
578 + )
579 +
580 def get_supported_mm_limits(self) -> Mapping[str, int | None]:
581 return {"audio": None, "image": None}
582
@@ -844,12 +853,6 @@ class Phi4MMDummyInputsBuilder(BaseDummyInputsBuilder[Phi4MMProcessingInfo]):
853
854
855 class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
847 - def _get_data_parser(self) -> MultiModalDataParser:
848 - feature_extractor = self.info.get_feature_extractor()
849 - return MultiModalDataParser(
850 - target_sr=feature_extractor.sampling_rate, audio_resample_method="scipy"
851 - )
852 -
856 def _call_hf_processor(
857 self,
858 prompt: str,
vllm/model_executor/models/qwen2_5_omni_thinker.py
+10 -9
@@ -77,7 +77,6 @@ from vllm.multimodal.parse import (
77 DictEmbeddingItems,
78 ModalityDataItems,
79 MultiModalDataItems,
80 - MultiModalDataParser,
80 )
81 from vllm.multimodal.processing import BaseDummyInputsBuilder
82 from vllm.multimodal.processing.processor import (
@@ -227,6 +226,16 @@ class Qwen2_5OmniThinkerProcessingInfo(
226 assert isinstance(feature_extractor, WhisperFeatureExtractor)
227 return feature_extractor
228
229 + def get_data_parser(self):
230 + feature_extractor = self.get_feature_extractor()
231 +
232 + return Qwen2_5OmniThinkerMultiModalDataParser(
233 + spatial_merge_size=self.get_hf_config().vision_config.spatial_merge_size,
234 + target_sr=feature_extractor.sampling_rate,
235 + target_channels=self.get_target_channels(),
236 + expected_hidden_size=self._get_expected_hidden_size(),
237 + )
238 +
239 def get_target_channels(self) -> int:
240 """Return target audio channels for Qwen2.5 Omni models (mono)."""
241 return 1
@@ -310,14 +319,6 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(
319 class Qwen2_5OmniThinkerMultiModalProcessor(
320 BaseMultiModalProcessor[Qwen2_5OmniThinkerProcessingInfo]
321 ):
313 - def _get_data_parser(self) -> MultiModalDataParser:
314 - feature_extractor = self.info.get_feature_extractor()
315 - return Qwen2_5OmniThinkerMultiModalDataParser(
316 - spatial_merge_size=self.info.get_hf_config().vision_config.spatial_merge_size,
317 - target_sr=feature_extractor.sampling_rate,
318 - target_channels=self.info.get_target_channels(),
319 - )
320 -
322 def _call_hf_processor(
323 self,
324 prompt: str,
vllm/model_executor/models/qwen2_audio.py
+33 -31
@@ -127,6 +127,30 @@ def _get_feat_extract_output_lengths(input_lengths: torch.Tensor):
127 return feat_lengths, output_lengths
128
129
130 +def _qwen2audio_field_config(hf_inputs: Mapping[str, torch.Tensor]):
131 + return dict(
132 + audio_embeds=MultiModalFieldConfig.batched("audio"),
133 + input_features=MultiModalFieldConfig.batched("audio"),
134 + feature_attention_mask=MultiModalFieldConfig.batched("audio"),
135 + )
136 +
137 +
138 +class Qwen2AudioMultiModalDataParser(MultiModalDataParser):
139 + def _parse_audio_data(
140 + self,
141 + data: dict[str, torch.Tensor] | ModalityData[AudioItem],
142 + ) -> ModalityDataItems[Any, Any] | None:
143 + if isinstance(data, dict):
144 + return DictEmbeddingItems(
145 + data,
146 + modality="audio",
147 + required_fields={"audio_embeds"},
148 + fields_factory=_qwen2audio_field_config,
149 + )
150 +
151 + return super()._parse_audio_data(data)
152 +
153 +
154 class Qwen2AudioProcessingInfo(BaseProcessingInfo):
155 def get_hf_config(self):
156 return self.ctx.get_hf_config(Qwen2AudioConfig)
@@ -140,6 +164,15 @@ class Qwen2AudioProcessingInfo(BaseProcessingInfo):
164 assert isinstance(feature_extractor, WhisperFeatureExtractor)
165 return feature_extractor
166
167 + def get_data_parser(self):
168 + feature_extractor = self.get_feature_extractor()
169 +
170 + return Qwen2AudioMultiModalDataParser(
171 + target_sr=feature_extractor.sampling_rate,
172 + target_channels=self.get_target_channels(),
173 + expected_hidden_size=self._get_expected_hidden_size(),
174 + )
175 +
176 def get_target_channels(self) -> int:
177 """Return target audio channels for Qwen2 Audio models (mono)."""
178 return 1
@@ -178,38 +211,7 @@ class Qwen2AudioDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2AudioProcessingIn
211 }
212
213
181 -def _qwen2audio_field_config(hf_inputs: Mapping[str, torch.Tensor]):
182 - return dict(
183 - audio_embeds=MultiModalFieldConfig.batched("audio"),
184 - input_features=MultiModalFieldConfig.batched("audio"),
185 - feature_attention_mask=MultiModalFieldConfig.batched("audio"),
186 - )
187 -
188 -
189 -class Qwen2AudioMultiModalDataParser(MultiModalDataParser):
190 - def _parse_audio_data(
191 - self,
192 - data: dict[str, torch.Tensor] | ModalityData[AudioItem],
193 - ) -> ModalityDataItems[Any, Any] | None:
194 - if isinstance(data, dict):
195 - return DictEmbeddingItems(
196 - data,
197 - modality="audio",
198 - required_fields={"audio_embeds"},
199 - fields_factory=_qwen2audio_field_config,
200 - )
201 -
202 - return super()._parse_audio_data(data)
203 -
204 -
214 class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessingInfo]):
206 - def _get_data_parser(self) -> MultiModalDataParser:
207 - feature_extractor = self.info.get_feature_extractor()
208 - return Qwen2AudioMultiModalDataParser(
209 - target_sr=feature_extractor.sampling_rate,
210 - target_channels=self.info.get_target_channels(),
211 - )
212 -
215 def _call_hf_processor(
216 self,
217 prompt: str,
vllm/model_executor/models/qwen2_vl.py
+6 -5
@@ -806,6 +806,12 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
806 def get_image_processor(self, **kwargs: object) -> Qwen2VLImageProcessor:
807 return self.get_hf_processor(**kwargs).image_processor
808
809 + def get_data_parser(self):
810 + return Qwen2VLMultiModalDataParser(
811 + self.get_hf_config().vision_config.spatial_merge_size,
812 + expected_hidden_size=self._get_expected_hidden_size(),
813 + )
814 +
815 def get_supported_mm_limits(self) -> Mapping[str, int | None]:
816 return {"image": None, "video": None}
817
@@ -1039,11 +1045,6 @@ class Qwen2VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2VLProcessingInfo]):
1045
1046
1047 class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo]):
1042 - def _get_data_parser(self) -> MultiModalDataParser:
1043 - return Qwen2VLMultiModalDataParser(
1044 - self.info.get_hf_config().vision_config.spatial_merge_size
1045 - )
1046 -
1048 def _get_prompt_updates(
1049 self,
1050 mm_items: MultiModalDataItems,
vllm/model_executor/models/qwen3_vl.py
+8 -7
@@ -81,7 +81,7 @@ from vllm.multimodal.inputs import (
81 PlaceholderRange,
82 VideoItem,
83 )
84 -from vllm.multimodal.parse import ImageSize, MultiModalDataItems, MultiModalDataParser
84 +from vllm.multimodal.parse import ImageSize, MultiModalDataItems
85 from vllm.multimodal.processing import (
86 BaseDummyInputsBuilder,
87 BaseMultiModalProcessor,
@@ -624,6 +624,13 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
624 def get_video_processor(self, **kwargs: object) -> Qwen3VLVideoProcessor:
625 return self.get_hf_processor(**kwargs).video_processor
626
627 + def get_data_parser(self):
628 + return Qwen2VLMultiModalDataParser(
629 + self.get_hf_config().vision_config.spatial_merge_size,
630 + video_needs_metadata=True,
631 + expected_hidden_size=self._get_expected_hidden_size(),
632 + )
633 +
634 def _get_vision_info(
635 self,
636 *,
@@ -901,12 +908,6 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
908
909
910 class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]):
904 - def _get_data_parser(self) -> MultiModalDataParser:
905 - return Qwen2VLMultiModalDataParser(
906 - self.info.get_hf_config().vision_config.spatial_merge_size,
907 - video_needs_metadata=True,
908 - )
909 -
911 def _call_hf_processor(
912 self,
913 prompt: str,
vllm/model_executor/models/terratorch.py
+41 -45
@@ -19,6 +19,7 @@
19
20 from collections import OrderedDict
21 from collections.abc import Iterable, Mapping, Sequence
22 +from functools import cached_property
23 from typing import Any
24
25 import torch
@@ -38,7 +39,6 @@ from vllm.model_executor.layers.pooler import IdentityPooler
39 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
40 from vllm.model_executor.models.utils import AutoWeightsLoader
41 from vllm.multimodal import MULTIMODAL_REGISTRY
41 -from vllm.multimodal.cache import MultiModalProcessorOnlyCache
42 from vllm.multimodal.inputs import (
43 ImageItem,
44 ModalityData,
@@ -89,7 +89,45 @@ def _terratorch_field_factory(input_definition: InputDefinition):
89 return _terratorch_field_config
90
91
92 +class TerratorchMultiModalDataParser(MultiModalDataParser):
93 + def __init__(self, input_definition: InputDefinition, *args, **kwargs):
94 + super().__init__(*args, **kwargs)
95 +
96 + self.input_definition = input_definition
97 +
98 + def _parse_image_data(
99 + self,
100 + data: dict[str, torch.Tensor] | ModalityData[ImageItem],
101 + ) -> ModalityDataItems[Any, Any] | None:
102 + if isinstance(data, dict):
103 + return DictEmbeddingItems(
104 + data,
105 + modality="image",
106 + required_fields=_terratorch_field_names(self.input_definition),
107 + fields_factory=_terratorch_field_factory(self.input_definition),
108 + )
109 +
110 + return super()._parse_image_data(data)
111 +
112 + def parse_mm_data(self, mm_data: MultiModalDataDict) -> MultiModalDataItems:
113 + if "image" not in mm_data:
114 + mm_data = {"image": mm_data}
115 +
116 + return super().parse_mm_data(mm_data)
117 +
118 +
119 class TerratorchProcessingInfo(BaseProcessingInfo):
120 + @cached_property
121 + def input_definition(self) -> InputDefinition:
122 + pretrained_cfg = self.get_hf_config().to_dict()["pretrained_cfg"]
123 + return InputDefinition(**pretrained_cfg["input"])
124 +
125 + def get_data_parser(self):
126 + return TerratorchMultiModalDataParser(
127 + self.input_definition,
128 + expected_hidden_size=self._get_expected_hidden_size(),
129 + )
130 +
131 def get_supported_mm_limits(self) -> Mapping[str, int | None]:
132 return {"image": None}
133
@@ -123,55 +161,13 @@ class TerratorchInputBuilder(BaseDummyInputsBuilder[TerratorchProcessingInfo]):
161 return self.dummy_data_generator.get_dummy_mm_data()
162
163
126 -class TerratorchMultiModalDataParser(MultiModalDataParser):
127 - def __init__(self, input_definition: InputDefinition, *args, **kwargs):
128 - super().__init__(*args, **kwargs)
129 -
130 - self.input_definition = input_definition
131 -
132 - def _parse_image_data(
133 - self,
134 - data: dict[str, torch.Tensor] | ModalityData[ImageItem],
135 - ) -> ModalityDataItems[Any, Any] | None:
136 - if isinstance(data, dict):
137 - return DictEmbeddingItems(
138 - data,
139 - modality="image",
140 - required_fields=_terratorch_field_names(self.input_definition),
141 - fields_factory=_terratorch_field_factory(self.input_definition),
142 - )
143 -
144 - return super()._parse_image_data(data)
145 -
146 - def parse_mm_data(self, mm_data: MultiModalDataDict) -> MultiModalDataItems:
147 - if "image" not in mm_data:
148 - mm_data = {"image": mm_data}
149 -
150 - return super().parse_mm_data(mm_data)
151 -
152 -
153 -class TerratorchMultiModalProcessor(BaseMultiModalProcessor):
154 - def __init__(
155 - self,
156 - info: TerratorchProcessingInfo,
157 - dummy_inputs: "BaseDummyInputsBuilder[TerratorchProcessingInfo]",
158 - *,
159 - cache: MultiModalProcessorOnlyCache | None = None,
160 - ) -> None:
161 - pretrained_cfg = info.get_hf_config().to_dict()["pretrained_cfg"]
162 - self._input_definition = InputDefinition(**pretrained_cfg["input"])
163 -
164 - super().__init__(info=info, dummy_inputs=dummy_inputs, cache=cache)
165 -
166 - def _get_data_parser(self) -> MultiModalDataParser:
167 - return TerratorchMultiModalDataParser(self._input_definition)
168 -
164 +class TerratorchMultiModalProcessor(BaseMultiModalProcessor[TerratorchProcessingInfo]):
165 def _get_mm_fields_config(
166 self,
167 hf_inputs: BatchFeature,
168 hf_processor_mm_kwargs: Mapping[str, object],
169 ) -> Mapping[str, MultiModalFieldConfig]:
174 - return _terratorch_field_factory(self._input_definition)(hf_inputs)
170 + return _terratorch_field_factory(self.info.input_definition)(hf_inputs)
171
172 def _get_prompt_updates(
173 self,
vllm/model_executor/models/ultravox.py
+9 -7
@@ -133,6 +133,15 @@ class UltravoxProcessingInfo(BaseProcessingInfo):
133 assert isinstance(feature_extractor, WhisperFeatureExtractor)
134 return feature_extractor
135
136 + def get_data_parser(self):
137 + feature_extractor = self.get_feature_extractor()
138 +
139 + return MultiModalDataParser(
140 + target_sr=feature_extractor.sampling_rate,
141 + target_channels=self.get_target_channels(),
142 + expected_hidden_size=self._get_expected_hidden_size(),
143 + )
144 +
145 def get_target_channels(self) -> int:
146 """Return target audio channels for Ultravox models (mono)."""
147 return 1
@@ -171,13 +180,6 @@ class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo])
180
181
182 class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo]):
174 - def _get_data_parser(self) -> MultiModalDataParser:
175 - feature_extractor = self.info.get_feature_extractor()
176 - return MultiModalDataParser(
177 - target_sr=feature_extractor.sampling_rate,
178 - target_channels=self.info.get_target_channels(),
179 - )
180 -
183 def _call_hf_processor(
184 self,
185 prompt: str,
vllm/model_executor/models/voxtral.py
+6 -4
@@ -203,6 +203,12 @@ class VoxtralProcessingInfo(BaseProcessingInfo):
203 def get_hf_processor(self) -> VoxtralProcessorAdapter:
204 return VoxtralProcessorAdapter(self.get_tokenizer())
205
206 + def get_data_parser(self):
207 + return MultiModalDataParser(
208 + target_sr=self.get_hf_processor().sampling_rate,
209 + expected_hidden_size=self._get_expected_hidden_size(),
210 + )
211 +
212 def get_supported_mm_limits(self) -> Mapping[str, int | None]:
213 return {"audio": 5} # Performance tends to degrade after 5
214
@@ -335,10 +341,6 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
341 # NOTE: The tokens are already inserted by the chat template
342 return prompt_ids, mm_info, True
343
338 - def _get_data_parser(self) -> MultiModalDataParser:
339 - sampling_rate = self.info.get_hf_processor().sampling_rate
340 - return MultiModalDataParser(target_sr=sampling_rate)
341 -
344
345 @MULTIMODAL_REGISTRY.register_processor(
346 VoxtralMultiModalProcessor,
vllm/model_executor/models/whisper.py
+9 -7
@@ -644,6 +644,15 @@ class WhisperProcessingInfo(BaseProcessingInfo):
644 def get_hf_config(self) -> WhisperConfig:
645 return self.ctx.get_hf_config(WhisperConfig)
646
647 + def get_data_parser(self):
648 + feature_extractor = self.get_feature_extractor()
649 +
650 + return MultiModalDataParser(
651 + target_sr=feature_extractor.sampling_rate,
652 + target_channels=self.get_target_channels(),
653 + expected_hidden_size=self._get_expected_hidden_size(),
654 + )
655 +
656 @property
657 def skip_prompt_length_check(self) -> bool:
658 return True # Because the encoder prompt is padded
@@ -693,13 +702,6 @@ class WhisperDummyInputsBuilder(BaseDummyInputsBuilder[WhisperProcessingInfo]):
702
703
704 class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo]):
696 - def _get_data_parser(self) -> MultiModalDataParser:
697 - feature_extractor = self.info.get_feature_extractor()
698 - return MultiModalDataParser(
699 - target_sr=feature_extractor.sampling_rate,
700 - target_channels=self.info.get_target_channels(),
701 - )
702 -
705 def create_encoder_prompt(
706 self,
707 prompt: str | list[int],
vllm/multimodal/processing/context.py
+30
@@ -17,6 +17,7 @@ import torch
17 from typing_extensions import TypeVar
18
19 from vllm.logger import init_logger
20 +from vllm.multimodal.parse import MultiModalDataParser
21 from vllm.tokenizers import TokenizerLike
22 from vllm.transformers_utils.processor import cached_processor_from_config
23 from vllm.utils.func_utils import get_allowed_kwarg_only_overrides
@@ -569,6 +570,35 @@ class BaseProcessingInfo:
570 """
571 return self.ctx.get_hf_processor(**kwargs)
572
573 + def _get_expected_hidden_size(self) -> int | None:
574 + """
575 + Get expected hidden size for embedding validation if `mm_embeds` are enabled.
576 +
577 + This validates hidden dimensions to prevent a vulnerability where embeddings
578 + with correct `ndim` but wrong `shape` could cause crashes at inference time.
579 + """
580 + model_config = self.ctx.model_config
581 + mm_config = model_config.get_multimodal_config()
582 +
583 + if mm_config.enable_mm_embeds:
584 + return model_config.get_inputs_embeds_size()
585 +
586 + return None
587 +
588 + def get_data_parser(self) -> MultiModalDataParser:
589 + """
590 + Constructs a parser to preprocess multi-modal data items
591 + before passing them to
592 + [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
593 +
594 + You can support additional modalities by creating a subclass
595 + of [`MultiModalDataParser`][vllm.multimodal.parse.MultiModalDataParser]
596 + that has additional subparsers.
597 + """
598 + return MultiModalDataParser(
599 + expected_hidden_size=self._get_expected_hidden_size(),
600 + )
601 +
602 @property
603 def skip_prompt_length_check(self) -> bool:
604 return False
vllm/multimodal/processing/processor.py
+10 -22
@@ -40,7 +40,6 @@ from ..parse import (
40 DictEmbeddingItems,
41 EmbeddingItems,
42 MultiModalDataItems,
43 - MultiModalDataParser,
43 )
44 from .context import (
45 BaseProcessingInfo,
@@ -990,7 +989,16 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
989 self.dummy_inputs = dummy_inputs
990 self.cache = cache
991
993 - self.data_parser = self._get_data_parser()
992 + if hasattr(self, "_get_data_parser"):
993 + logger.warning_once(
994 + "BaseMultiModalProcessor._get_data_parser is deprecated "
995 + "and will be removed in v0.16."
996 + "You should override `info.build_data_parser` instead."
997 + )
998 +
999 + self.data_parser = self._get_data_parser() # type: ignore
1000 + else:
1001 + self.data_parser = self.info.get_data_parser()
1002
1003 # Avoid unnecessary recomputation
1004 self._supported_mm_limits = self.info.get_supported_mm_limits()
@@ -1014,26 +1022,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1022 ) -> MultiModalInputs:
1023 return self.apply(prompt, mm_data, hf_processor_mm_kwargs, mm_uuids=mm_uuids)
1024
1017 - def _get_data_parser(self) -> MultiModalDataParser:
1018 - """
1019 - Construct a parser to preprocess multi-modal data items
1020 - before passing them to
1021 - [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
1022 -
1023 - You can support additional modalities by creating a subclass
1024 - of [`MultiModalDataParser`][vllm.multimodal.parse.MultiModalDataParser]
1025 - that has additional subparsers.
1026 - """
1027 - # Get expected hidden size for embedding validation if mm_embeds enabled
1028 - # This validates hidden dimensions to prevent vulnerabilities: embeddings
1029 - # with correct ndim but wrong shape could cause crashes at inference time
1030 - mm_config = self.info.ctx.model_config.get_multimodal_config()
1031 - expected_hidden_size = None
1032 - if mm_config.enable_mm_embeds:
1033 - expected_hidden_size = self.info.ctx.model_config.get_inputs_embeds_size()
1034 -
1035 - return MultiModalDataParser(expected_hidden_size=expected_hidden_size)
1036 -
1025 def validate_num_items(
1026 self,
1027 modality: str,