0b9e018fb9 · tduncan/Nvidia-Optimization-Stack-VLLM

docs/contributing/model/multimodal.md

+22 -11

@@ -454,17 +454,17 @@ return a schema of the tensors outputted by the HF processor that are related to

454 ??? code

455

456 ```python

457 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

457 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

458 # Mistral3Processor requires text corresponding to the images

459 return self.dummy_inputs.get_dummy_text(mm_counts)

460

461 def _postprocess_hf_mm_data(

462 self,

463 - mm_data: Mapping[str, object],

464 - hf_processor_mm_kwargs: Mapping[str, object],

463 + hf_data: Mapping[str, object],

464 + hf_kwargs: Mapping[str, object],

465 processed_data: BatchFeature,

466 ) -> BatchFeature:

467 - if not mm_data:

467 + if not hf_data:

468 return processed_data

469

470 pixel_values = processed_data.get("pixel_values")

@@ -486,25 +486,36 @@ return a schema of the tensors outputted by the HF processor that are related to

486 calls the HF processor on the multi-modal data without passing any text.

487 If the HF processor instead requires text corresponding to the multi-modal items,

488 you should override

489 - [_get_hf_processor_text][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_processor_text]

489 + [_get_hf_mm_text][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_text]

490 to return the dummy text from

491 [BaseDummyInputsBuilder.get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text]

492 like in the example above. If the HF processor expects the multi-modal data

493 - under different keys than those provided by the multi-modal items

494 - (e.g. `audio` instead of `audios`), or requires additional keyword arguments

493 + under different keys than those provided by the multi-modal items,

494 + or requires additional keyword arguments

495 (e.g. `sampling_rate`), you should override

496 - [_preprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._preprocess_hf_mm_data].

496 + [_get_hf_mm_inputs][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs].

497 If you need to modify the output of the HF processor,

498 you should override

499 [_postprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_hf_mm_data].

500 + Passthrough fields have already been merged into `processed_data` when this

501 + hook is called.

502 + If you need to change how the normalized inputs are passed to the HF

503 + processor (for example, to call a method other than `__call__`, or to use

504 + different kwargs to construct and invoke it), override

505 + [_call_hf_processor][vllm.multimodal.processing.BaseMultiModalProcessor._call_hf_processor].

506 If the HF processor applies additional transformations to the prompt tokens

507 that are not reflected in its multi-modal outputs (e.g. prepending a BOS token),

508 you should override

509 [_postprocess_prompt][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_prompt]

510 to replicate them.

505 - For even more control over how the HF processor is called, you can override

511 + Only processors that require multiple HF calls or manually construct the

512 + complete output should override

513 [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main]

507 - directly.

514 + directly. Such implementations must obtain their normalized inputs through

515 + [_get_hf_mm_inputs][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs]

516 + and return through

517 + [_finalize_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._finalize_hf_mm_data]

518 + so passthrough fields and model-specific postprocessing are applied consistently.

519

520 Since `pixel_values` is now a list with one tensor per image, we can override

521 [_get_mm_fields_config][vllm.multimodal.processing.BaseMultiModalProcessor._get_mm_fields_config] as follows:

@@ -724,7 +735,7 @@ Examples:

735

736 ### Custom HF processor

737

727 -Some models don't define an HF processor class on HF Hub. In that case, you can define a custom HF processor that has the same call signature as HF processors and return it from [BaseProcessingInfo.get_hf_processor][vllm.multimodal.processing.BaseProcessingInfo.get_hf_processor]. It is then applied to the multi-modal data via [InputProcessingContext.call_hf_processor][vllm.multimodal.processing.InputProcessingContext.call_hf_processor] inside [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main].

738 +Some models don't define an HF processor class on HF Hub. In that case, you can define a custom HF processor that has the same call signature as HF processors and return it from [BaseProcessingInfo.get_hf_processor][vllm.multimodal.processing.BaseProcessingInfo.get_hf_processor]. The default [_call_hf_processor][vllm.multimodal.processing.BaseMultiModalProcessor._call_hf_processor] applies it to the normalized multi-modal data via [InputProcessingContext.call_hf_processor][vllm.multimodal.processing.InputProcessingContext.call_hf_processor]. Override `_call_hf_processor` if the custom processor instead exposes a different callable interface.

739

740 Examples:

741

docs/design/mm_processing.md

+9 -2

@@ -8,9 +8,16 @@ In vLLM's rendering pipeline (see [BaseRenderer][vllm.renderers.base.BaseRendere

8

9 Since Transformers 5.10, `ProcessorMixin` now allows multi-modal inputs to be passed by themselves. However, certain subclasses (such as `ChameleonProcessor`) and older out-of-tree implementations may still define their own `__call__` method that assumes the presence of text with corresponding placeholder tokens. This causes a problem as we don't have the original text anymore to pass to these processors.

10

11 -To work around this, each model defines how to generate dummy text based on the number of multi-modal inputs, via [get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text], which its override of [_get_hf_processor_text][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_processor_text] returns so that [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main] passes it to the HF processor together with the multi-modal inputs to obtain the processed multi-modal data.

11 +To work around this, each model defines how to generate dummy text based on the number of multi-modal inputs, via [get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text], which its override of [_get_hf_mm_text][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_text] returns. [_get_hf_mm_inputs][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs] adds this text and `truncation=False` to the normalized HF inputs. The truncation setting is attached only to calls that use dummy text, so unrelated processor calls do not receive a text-specific option.

12

13 -Similarly, since the multi-modal data extracted by vLLM may not match what a specific HF processor expects, [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main] allows each model to adapt the inputs via [_preprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._preprocess_hf_mm_data] (e.g. renaming keys like `audios` to `audio` or injecting extra keyword arguments such as `sampling_rate`) and the outputs via [_postprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_hf_mm_data], without having to reimplement the entire method.

13 +HF processing is split into four overridable stages:

14 +

15 +1. [_get_hf_mm_inputs][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs] extracts and normalizes processor data, kwargs, and passthrough data. It centrally renames vLLM's `audios` field to the standard HF `audio` keyword; processors that require another spelling can adapt it in an override.

16 +2. [_call_hf_processor][vllm.multimodal.processing.BaseMultiModalProcessor._call_hf_processor] invokes the HF processor. Models can override it when they use a different callable or need separate construction and invocation kwargs.

17 +3. [_finalize_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._finalize_hf_mm_data] merges passthrough fields into the processed output. It lazily creates an empty output when there is no processor data.

18 +4. [_postprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_hf_mm_data] performs model-specific output transformations after passthrough fields have been merged.

19 +

20 +This division lets most models customize one stage without reimplementing [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main]. Implementations that genuinely require multiple processor calls or manually construct their complete output may still override that method, but must use `_get_hf_mm_inputs` and `_finalize_hf_mm_data` to retain the common input and output behavior.

21

22 ## Prompt Update Detection

23

vllm/model_executor/models/audioflamingo3.py

+1 -13

@@ -374,21 +374,9 @@ class AudioFlamingo3MultiModalDataParser(MultiModalDataParser):

374 class AudioFlamingo3MultiModalProcessor(

375 BaseMultiModalProcessor[AudioFlamingo3ProcessingInfo]

376 ):

377 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

377 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

378 return self.dummy_inputs.get_dummy_text(mm_counts)

379

380 - def _preprocess_hf_mm_data(

381 - self,

382 - mm_data: Mapping[str, object],

383 - hf_processor_mm_kwargs: Mapping[str, object],

384 - ) -> tuple[Mapping[str, object], Mapping[str, object]]:

385 - mm_data = dict(mm_data)

386 - audios = mm_data.pop("audios", None)

387 - if audios is not None:

388 - mm_data["audio"] = audios

389 -

390 - return mm_data, hf_processor_mm_kwargs

391 -

380 def _postprocess_hf_mm_data(

381 self,

382 mm_data: Mapping[str, object],

vllm/model_executor/models/bailing_moe_v3_vl.py

+1 -1

@@ -119,7 +119,7 @@ class BailingMoeV3VLDummyInputsBuilder(

119 class BailingMoeV3VLMultiModalProcessor(

120 BaseMultiModalProcessor[BailingMoeV3VLProcessingInfo]

121 ):

122 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

122 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

123 return self.dummy_inputs.get_dummy_text(mm_counts)

124

125 def _apply_prompt_updates(

vllm/model_executor/models/cohere2_vision.py

+1 -1

@@ -222,7 +222,7 @@ class Cohere2VisionDummyInputsBuilder(

222 class Cohere2VisionMultiModalProcessor(

223 BaseMultiModalProcessor[Cohere2VisionProcessingInfo]

224 ):

225 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

225 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

226 return self.dummy_inputs.get_dummy_text(mm_counts)

227

228 def _postprocess_hf_mm_data(

vllm/model_executor/models/cohere_asr.py

+19 -29

@@ -8,7 +8,7 @@ from typing import Any, ClassVar

8 import torch

9 import torch.nn.functional as F

10 from torch import nn

11 -from transformers import BatchFeature, PretrainedConfig

11 +from transformers import PretrainedConfig

12

13 from vllm.compilation.decorators import support_torch_compile

14 from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig

@@ -50,7 +50,10 @@ from vllm.multimodal.processing import (

50 PromptUpdate,

51 TimingContext,

52 )

53 -from vllm.multimodal.processing.processor import MultiModalProcessingInfo

53 +from vllm.multimodal.processing.processor import (

54 + HFMultiModalInputs,

55 + MultiModalProcessingResult,

56 +)

57 from vllm.renderers import TokenizeParams

58 from vllm.tokenizers import cached_tokenizer_from_config

59 from vllm.transformers_utils.processors.cohere_asr import (

@@ -1962,42 +1965,29 @@ class CohereASRMultiModalProcessor(EncDecMultiModalProcessor[CohereASRProcessing

1965 ) -> list[int]:

1966 return [0]

1967

1965 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

1968 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

1969 return self.dummy_inputs.get_dummy_text(mm_counts)

1970

1968 - def _preprocess_hf_mm_data(

1971 + def _get_hf_mm_inputs(

1972 self,

1970 - mm_data: Mapping[str, object],

1971 - hf_processor_mm_kwargs: Mapping[str, object],

1972 - ) -> tuple[Mapping[str, object], Mapping[str, object]]:

1973 - feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)

1974 -

1975 - mm_data = dict(mm_data)

1976 - mm_data["audio"] = mm_data.pop("audios")

1977 -

1978 - hf_processor_mm_kwargs = dict(

1979 - **hf_processor_mm_kwargs,

1980 - sampling_rate=feature_extractor.sampling_rate,

1973 + mm_items: MultiModalDataItems,

1974 + hf_kwargs: Mapping[str, object],

1975 + ) -> HFMultiModalInputs:

1976 + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)

1977 +

1978 + feature_extractor = self.info.get_feature_extractor(**hf_kwargs)

1979 + return hf_inputs._replace(

1980 + hf_kwargs=dict(

1981 + hf_inputs.hf_kwargs,

1982 + sampling_rate=feature_extractor.sampling_rate,

1983 + )

1984 )

1985

1983 - return mm_data, hf_processor_mm_kwargs

1984 -

1985 - def _postprocess_hf_mm_data(

1986 - self,

1987 - mm_data: Mapping[str, object],

1988 - hf_processor_mm_kwargs: Mapping[str, object],

1989 - processed_data: BatchFeature,

1990 - ) -> BatchFeature:

1991 - if "labels" in processed_data:

1992 - processed_data["input_ids"] = processed_data.pop("labels")

1993 -

1994 - return processed_data

1995 -

1986 def _cached_apply_hf_processor(

1987 self,

1988 inputs: ProcessorInputs,

1989 timing_ctx: TimingContext,

2000 - ) -> MultiModalProcessingInfo:

1990 + ) -> MultiModalProcessingResult:

1991 # Dithering injects noise into the extracted features, so the

1992 # feature extractor is not a pure function of its input. Since the

1993 # processing cache assumes that processor outputs are invariant

vllm/model_executor/models/colmodernvbert.py

+7 -7

@@ -157,15 +157,14 @@ class ColModernVBertMultiModalProcessor(

157 def _apply_hf_processor_main(

158 self,

159 mm_items: MultiModalDataItems,

160 - hf_processor_mm_kwargs: Mapping[str, object],

160 + hf_kwargs: Mapping[str, object],

161 ) -> BatchFeature:

162 - valid_mm_items = mm_items.select(

163 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

162 + mm_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(

163 + mm_items, hf_kwargs

164 )

165 - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

165

166 if not mm_data:

168 - return BatchFeature(dict(passthrough_data))

167 + return self._finalize_hf_mm_data(mm_data, hf_kwargs, passthrough_data)

168

169 prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())

170

@@ -193,8 +192,9 @@ class ColModernVBertMultiModalProcessor(

192 result.update(image_outputs)

193

194 processed_data = result

196 - processed_data.update(passthrough_data)

197 - return processed_data

195 + return self._finalize_hf_mm_data(

196 + mm_data, hf_kwargs, passthrough_data, processed_data

197 + )

198

199 def _get_mm_fields_config(

200 self,

vllm/model_executor/models/colpali.py

+7 -18

@@ -20,7 +20,7 @@ from collections.abc import Iterable, Mapping

20

21 import torch

22 import torch.nn as nn

23 -from transformers import BatchFeature, PaliGemmaProcessor

23 +from transformers import PaliGemmaProcessor

24

25 from vllm.config import VllmConfig

26 from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_embed

@@ -28,6 +28,7 @@ from vllm.model_executor.layers.pooler.tokwise.heads import TokenPoolerHead

28 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

29 from vllm.multimodal import MULTIMODAL_REGISTRY

30 from vllm.multimodal.parse import MultiModalDataItems

31 +from vllm.multimodal.processing.processor import HFMultiModalInputs

32

33 from .interfaces import SupportsLateInteraction

34 from .interfaces_base import default_pooling_type

@@ -59,31 +60,19 @@ class ColPaliProcessingInfo(PaliGemmaProcessingInfo):

60 class ColPaliMultiModalProcessor(PaliGemmaMultiModalProcessor):

61 """Multimodal processor for ColPali."""

62

62 - def _apply_hf_processor_main(

63 + def _get_hf_mm_inputs(

64 self,

65 mm_items: MultiModalDataItems,

65 - hf_processor_mm_kwargs: Mapping[str, object],

66 - ) -> BatchFeature:

67 - valid_mm_items = mm_items.select(

68 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

69 - )

70 - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

71 -

72 - if not mm_data:

73 - return BatchFeature(dict(passthrough_data))

66 + hf_kwargs: Mapping[str, object],

67 + ) -> HFMultiModalInputs:

68 + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)

69

70 # The ColPali tokenizer_config.json ships with a small default

71 # max_length (50) that truncates the 1024 image tokens inserted

72 # by PaliGemmaProcessor, causing a token-count mismatch.

73 # vLLM enforces its own max_model_len, so we disable HF

74 # truncation to keep all image + text tokens intact.

80 - processed_data = self.info.ctx.call_hf_processor(

81 - self.info.get_hf_processor(**hf_processor_mm_kwargs),

82 - mm_data,

83 - dict(**hf_processor_mm_kwargs, truncation=False),

84 - )

85 - processed_data.update(passthrough_data)

86 - return processed_data

75 + return hf_inputs._replace(hf_kwargs=dict(hf_inputs.hf_kwargs, truncation=False))

76

77

78 @default_pooling_type(seq_pooling_type="CLS", tok_pooling_type="ALL")

vllm/model_executor/models/deepseek_ocr.py

+11 -26

@@ -46,10 +46,10 @@ from vllm.multimodal.processing import (

46 PromptReplacement,

47 PromptUpdate,

48 )

49 +from vllm.multimodal.processing.processor import HFMultiModalInputs

50 from vllm.sampling_params import SamplingParams

51 from vllm.sequence import IntermediateTensors

52 from vllm.tokenizers import cached_tokenizer_from_config

52 -from vllm.tokenizers.hf import HfTokenizer

53 from vllm.transformers_utils.configs.deepseek_vl2 import DeepseekVLV2Config

54 from vllm.transformers_utils.processors.deepseek_ocr import (

55 BASE_SIZE,

@@ -287,34 +287,19 @@ class DeepseekOCRDummyInputsBuilder(BaseDummyInputsBuilder[DeepseekOCRProcessing

287 class DeepseekOCRMultiModalProcessor(

288 BaseMultiModalProcessor[DeepseekOCRProcessingInfo]

289 ):

290 - def _apply_hf_processor_main(

290 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

291 + return self.dummy_inputs.get_dummy_text(mm_counts)

292 +

293 + def _get_hf_mm_inputs(

294 self,

295 mm_items: MultiModalDataItems,

293 - hf_processor_mm_kwargs: Mapping[str, object],

294 - ) -> BatchFeature:

295 - valid_mm_items = mm_items.select(

296 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

297 - )

298 - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

299 -

300 - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())

301 -

302 - if mm_data:

303 - processed_data = self.info.ctx.call_hf_processor(

304 - self.info.get_hf_processor(**hf_processor_mm_kwargs),

305 - dict(prompt=prompt_text, **mm_data),

306 - hf_processor_mm_kwargs,

307 - )

308 -

309 - else:

310 - tokenizer = self.info.get_tokenizer()

311 - assert isinstance(tokenizer, HfTokenizer)

312 - processed_data = tokenizer(

313 - prompt_text, add_special_tokens=True, return_tensors="pt"

314 - )

296 + hf_kwargs: Mapping[str, object],

297 + ) -> HFMultiModalInputs:

298 + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)

299 + if "text" in hf_inputs.hf_data:

300 + hf_inputs.hf_data["prompt"] = hf_inputs.hf_data.pop("text")

301

316 - processed_data.update(passthrough_data)

317 - return processed_data

302 + return hf_inputs

303

304 def _get_mm_fields_config(

305 self,

vllm/model_executor/models/deepseek_ocr2.py

+11 -27

@@ -45,9 +45,9 @@ from vllm.multimodal.processing import (

45 PromptReplacement,

46 PromptUpdate,

47 )

48 +from vllm.multimodal.processing.processor import HFMultiModalInputs

49 from vllm.sequence import IntermediateTensors

50 from vllm.tokenizers import cached_tokenizer_from_config

50 -from vllm.tokenizers.hf import HfTokenizer

51 from vllm.transformers_utils.configs.deepseek_vl2 import DeepseekVLV2Config

52 from vllm.transformers_utils.processors.deepseek_ocr import (

53 BASE_SIZE,

@@ -158,35 +158,19 @@ class DeepseekOCR2DummyInputsBuilder(

158 class DeepseekOCR2MultiModalProcessor(

159 BaseMultiModalProcessor[DeepseekOCR2ProcessingInfo]

160 ):

161 - def _apply_hf_processor_main(

161 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

162 + return self.dummy_inputs.get_dummy_text(mm_counts)

163 +

164 + def _get_hf_mm_inputs(

165 self,

166 mm_items: MultiModalDataItems,

164 - hf_processor_mm_kwargs: Mapping[str, object],

165 - ) -> BatchFeature:

166 - valid_mm_items = mm_items.select(

167 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

168 - )

169 - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

170 -

171 - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())

172 -

173 - if mm_data:

174 - processed_data = self.info.ctx.call_hf_processor(

175 - self.info.get_hf_processor(**hf_processor_mm_kwargs),

176 - dict(prompt=prompt_text, **mm_data),

177 - hf_processor_mm_kwargs,

178 - )

179 -

180 - else:

181 - tokenizer = self.info.get_tokenizer()

182 - assert isinstance(tokenizer, HfTokenizer)

183 - processed_data = tokenizer(

184 - prompt_text, add_special_tokens=True, return_tensors="pt"

185 - )

186 -

187 - processed_data.update(passthrough_data)

167 + hf_kwargs: Mapping[str, object],

168 + ) -> HFMultiModalInputs:

169 + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)

170 + if "text" in hf_inputs.hf_data:

171 + hf_inputs.hf_data["prompt"] = hf_inputs.hf_data.pop("text")

172

189 - return processed_data

173 + return hf_inputs

174

175 def _get_mm_fields_config(

176 self,

vllm/model_executor/models/deepseek_vl2.py

+3 -3

@@ -35,7 +35,7 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder

35 from vllm.multimodal.processing.processor import (

36 BaseMultiModalProcessor,

37 BaseProcessingInfo,

38 - MultiModalProcessingInfo,

38 + MultiModalProcessingResult,

39 ProcessorInputs,

40 PromptReplacement,

41 PromptUpdate,

@@ -233,7 +233,7 @@ class DeepseekVL2DummyInputsBuilder(BaseDummyInputsBuilder[DeepseekVL2Processing

233 class DeepseekVL2MultiModalProcessor(

234 BaseMultiModalProcessor[DeepseekVL2ProcessingInfo]

235 ):

236 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

236 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

237 return self.dummy_inputs.get_dummy_text(mm_counts)

238

239 def _postprocess_hf_mm_data(

@@ -307,7 +307,7 @@ class DeepseekVL2MultiModalProcessor(

307 self,

308 inputs: ProcessorInputs,

309 timing_ctx: TimingContext,

310 - ) -> MultiModalProcessingInfo:

310 + ) -> MultiModalProcessingResult:

311 # The processor logic is different for len(images) <= 2 vs > 2

312 # Since the processing cache assumes that the processor output is

313 # invariant of how many images are passed per prompt, we only

vllm/model_executor/models/ernie45_vl.py

+50 -57

@@ -71,6 +71,7 @@ from vllm.multimodal.processing import (

71 PromptUpdate,

72 cached_encode,

73 )

74 +from vllm.multimodal.processing.processor import HFMultiModalInputs

75 from vllm.sequence import IntermediateTensors

76 from vllm.utils.gpu_sync_debug import gpu_sync_allowed

77 from vllm.utils.tensor_schema import TensorSchema, TensorShape

@@ -1112,34 +1113,32 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin

1113 pixel_values = pixel_values.to(hf_config.dtype)

1114 return pixel_values

1115

1115 - def _apply_hf_processor_main(

1116 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

1117 + return self.dummy_inputs.get_dummy_text(mm_counts)

1118 +

1119 + def _get_hf_mm_inputs(

1120 self,

1121 mm_items: MultiModalDataItems,

1118 - hf_processor_mm_kwargs: Mapping[str, object],

1119 - ) -> BatchFeature:

1120 - valid_mm_items = mm_items.select(

1121 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

1122 - )

1123 - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

1124 -

1125 - if "images" not in mm_data and "videos" not in mm_data:

1126 - return BatchFeature(dict(passthrough_data))

1122 + hf_kwargs: Mapping[str, object],

1123 + ) -> HFMultiModalInputs:

1124 + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)

1125 + hf_data = hf_inputs.hf_data

1126

1128 - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())

1127 + if "images" not in hf_data and "videos" not in hf_data:

1128 + return hf_inputs

1129

1130 - mm_data = dict(mm_data)

1131 - if "images" not in mm_data:

1132 - mm_data["images"] = []

1133 - if "videos" not in mm_data:

1134 - mm_data["videos"] = []

1130 + prompt_text = hf_data["text"]

1131 + hf_data["text"] = [prompt_text]

1132 + hf_data.setdefault("images", [])

1133 + hf_data.setdefault("videos", [])

1134

1135 # Check if HF processor supports video metadata

1137 - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)

1136 + hf_processor = self.info.get_hf_processor(**hf_inputs.hf_kwargs)

1137 supports_video_metadata = getattr(

1138 hf_processor, "supports_video_metadata", False

1139 )

1140

1142 - videos = mm_data["videos"]

1141 + videos = hf_data["videos"]

1142 assert isinstance(videos, Sequence)

1143 if videos and not supports_video_metadata:

1144 # Old HF processor, unwrap tuple to pure frames

@@ -1147,49 +1146,43 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin

1146 "HF processor doesn't support video metadata. "

1147 "Timestamps will NOT be rendered. Please upgrade the model."

1148 )

1150 - mm_data["videos"] = [v[0] if isinstance(v, tuple) else v for v in videos]

1149 + hf_data["videos"] = [v[0] if isinstance(v, tuple) else v for v in videos]

1150

1152 - processor_output = self.info.ctx.call_hf_processor(

1153 - hf_processor,

1154 - dict(

1155 - text=[prompt_text], images=mm_data["images"], videos=mm_data["videos"]

1156 - ),

1157 - hf_processor_mm_kwargs,

1158 - )

1151 + return hf_inputs

1152 +

1153 + def _postprocess_hf_mm_data(

1154 + self,

1155 + hf_data: Mapping[str, object],

1156 + hf_kwargs: Mapping[str, object],

1157 + processed_data: BatchFeature,

1158 + ) -> BatchFeature:

1159 + if "images" not in hf_data and "videos" not in hf_data:

1160 + return processed_data

1161

1162 # Divide the processor_output into two modalities: image and video.

1161 - if processor_output is not None:

1162 - pixel_values = processor_output["images"]

1163 - if pixel_values is not None:

1164 - processor_output["images"] = self._pixel_values_norm(

1165 - pixel_values, hf_processor_mm_kwargs

1166 - )

1167 - for key in list(processor_output.keys()):

1168 - if processor_output[key] is None:

1169 - del processor_output[key]

1170 - continue

1171 - if key == "grid_thw":

1172 - grid_thw = processor_output["grid_thw"]

1173 - pixel_values_all = processor_output["images"]

1174 - # Identify elements where the first

1175 - # dimension is greater than 1 and

1176 - # treat them as the video modality

1177 - mask = grid_thw[:, 0] > 1

1178 - processor_output["video_grid_thw"] = grid_thw[mask]

1179 - processor_output["image_grid_thw"] = grid_thw[~mask]

1180 - image_patch_num = (

1181 - processor_output["image_grid_thw"].prod(dim=1).sum()

1182 - )

1183 - processor_output["pixel_values"] = pixel_values_all[

1184 - :image_patch_num

1185 - ]

1186 - processor_output["pixel_values_videos"] = pixel_values_all[

1187 - image_patch_num:

1188 - ]

1189 - del processor_output["images"]

1163 + pixel_values = processed_data["images"]

1164 + if pixel_values is not None:

1165 + processed_data["images"] = self._pixel_values_norm(pixel_values, hf_kwargs)

1166 + for key in list(processed_data.keys()):

1167 + if processed_data[key] is None:

1168 + del processed_data[key]

1169 + continue

1170 + if key == "grid_thw":

1171 + grid_thw = processed_data["grid_thw"]

1172 + pixel_values_all = processed_data["images"]

1173 + # Identify elements where the first

1174 + # dimension is greater than 1 and

1175 + # treat them as the video modality

1176 + mask = grid_thw[:, 0] > 1

1177 + processed_data["video_grid_thw"] = grid_thw[mask]

1178 + processed_data["image_grid_thw"] = grid_thw[~mask]

1179 + image_patch_num = processed_data["image_grid_thw"].prod(dim=1).sum()

1180 + processed_data["pixel_values"] = pixel_values_all[:image_patch_num]

1181 + processed_data["pixel_values_videos"] = pixel_values_all[

1182 + image_patch_num:

1183 + ]

1184 + del processed_data["images"]

1185

1191 - processed_data = processor_output

1192 - processed_data.update(passthrough_data)

1186 return processed_data

1187

1188 def _get_prompt_updates(

vllm/model_executor/models/fireredasr2.py

+14 -26

@@ -38,6 +38,7 @@ from vllm.multimodal.processing import (

38 PromptUpdate,

39 PromptUpdateDetails,

40 )

41 +from vllm.multimodal.processing.processor import HFMultiModalInputs

42 from vllm.transformers_utils.processor import cached_processor_from_config

43 from vllm.transformers_utils.processors.fireredasr2 import (

44 FireRedASR2FeatureExtractor,

@@ -228,37 +229,24 @@ class FireRedASR2DummyInputsBuilder(BaseDummyInputsBuilder[FireRedASR2Processing

229 class FireRedASR2MultiModalProcessor(

230 BaseMultiModalProcessor[FireRedASR2ProcessingInfo]

231 ):

231 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

232 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

233 return self.dummy_inputs.get_dummy_text(mm_counts)

234

234 - def _preprocess_hf_mm_data(

235 + def _get_hf_mm_inputs(

236 self,

236 - mm_data: Mapping[str, object],

237 - hf_processor_mm_kwargs: Mapping[str, object],

238 - ) -> tuple[Mapping[str, object], Mapping[str, object]]:

239 - feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)

240 -

241 - mm_data = dict(mm_data)

242 - mm_data["audio"] = mm_data.pop("audios")

243 -

244 - hf_processor_mm_kwargs = dict(

245 - **hf_processor_mm_kwargs,

246 - sampling_rate=feature_extractor.sampling_rate,

237 + mm_items: MultiModalDataItems,

238 + hf_kwargs: Mapping[str, object],

239 + ) -> HFMultiModalInputs:

240 + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)

241 +

242 + feature_extractor = self.info.get_feature_extractor(**hf_kwargs)

243 + return hf_inputs._replace(

244 + hf_kwargs=dict(

245 + hf_inputs.hf_kwargs,

246 + sampling_rate=feature_extractor.sampling_rate,

247 + )

248 )

249

249 - return mm_data, hf_processor_mm_kwargs

250 -

251 - def _postprocess_hf_mm_data(

252 - self,

253 - mm_data: Mapping[str, object],

254 - hf_processor_mm_kwargs: Mapping[str, object],

255 - processed_data: BatchFeature,

256 - ) -> BatchFeature:

257 - if "labels" in processed_data:

258 - processed_data["input_ids"] = processed_data.pop("labels")

259 -

260 - return processed_data

261 -

250 def _get_mm_fields_config(

251 self,

252 hf_inputs: BatchFeature,

vllm/model_executor/models/funasr.py

+18 -28

@@ -49,7 +49,10 @@ from vllm.multimodal.processing import (

49 PromptUpdate,

50 TimingContext,

51 )

52 -from vllm.multimodal.processing.processor import MultiModalProcessingInfo

52 +from vllm.multimodal.processing.processor import (

53 + HFMultiModalInputs,

54 + MultiModalProcessingResult,

55 +)

56 from vllm.transformers_utils.processor import cached_processor_from_config

57 from vllm.transformers_utils.processors.funasr import FunASRFeatureExtractor

58 from vllm.utils.tensor_schema import TensorSchema, TensorShape

@@ -752,42 +755,29 @@ class FunASRDummyInputsBuilder(BaseDummyInputsBuilder[FunASRProcessingInfo]):

755

756

757 class FunASRMultiModalProcessor(BaseMultiModalProcessor[FunASRProcessingInfo]):

755 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

758 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

759 return self.dummy_inputs.get_dummy_text(mm_counts)

760

758 - def _preprocess_hf_mm_data(

761 + def _get_hf_mm_inputs(

762 self,

760 - mm_data: Mapping[str, object],

761 - hf_processor_mm_kwargs: Mapping[str, object],

762 - ) -> tuple[Mapping[str, object], Mapping[str, object]]:

763 - feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)

764 -

765 - mm_data = dict(mm_data)

766 - mm_data["audio"] = mm_data.pop("audios")

767 -

768 - hf_processor_mm_kwargs = dict(

769 - **hf_processor_mm_kwargs,

770 - sampling_rate=feature_extractor.sampling_rate,

763 + mm_items: MultiModalDataItems,

764 + hf_kwargs: Mapping[str, object],

765 + ) -> HFMultiModalInputs:

766 + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)

767 +

768 + feature_extractor = self.info.get_feature_extractor(**hf_kwargs)

769 + return hf_inputs._replace(

770 + hf_kwargs=dict(

771 + hf_inputs.hf_kwargs,

772 + sampling_rate=feature_extractor.sampling_rate,

773 + )

774 )

775

773 - return mm_data, hf_processor_mm_kwargs

774 -

775 - def _postprocess_hf_mm_data(

776 - self,

777 - mm_data: Mapping[str, object],

778 - hf_processor_mm_kwargs: Mapping[str, object],

779 - processed_data: BatchFeature,

780 - ) -> BatchFeature:

781 - if "labels" in processed_data:

782 - processed_data["input_ids"] = processed_data.pop("labels")

783 -

784 - return processed_data

785 -

776 def _cached_apply_hf_processor(

777 self,

778 inputs: ProcessorInputs,

779 timing_ctx: TimingContext,

790 - ) -> MultiModalProcessingInfo:

780 + ) -> MultiModalProcessingResult:

781 # Dithering injects noise into the extracted features, so the

782 # feature extractor is not a pure function of its input. Since the

783 # processing cache assumes that processor outputs are invariant

vllm/model_executor/models/funaudiochat.py

+12 -11

@@ -616,29 +616,29 @@ class FunAudioChatMultiModalProcessor(

616 def _apply_hf_processor_main(

617 self,

618 mm_items: MultiModalDataItems,

619 - hf_processor_mm_kwargs: Mapping[str, object],

619 + hf_kwargs: Mapping[str, object],

620 ) -> BatchFeature:

621 - valid_mm_items = mm_items.select(

622 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

621 + mm_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(

622 + mm_items, hf_kwargs

623 )

624 - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

624

625 if not mm_data:

627 - return BatchFeature(dict(passthrough_data))

626 + return self._finalize_hf_mm_data(mm_data, hf_kwargs, passthrough_data)

627

628 prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())

629

630 tokenizer = self.info.get_tokenizer()

631 input_ids = torch.tensor([tokenizer.encode(prompt_text)])

632

634 - audios = mm_data.get("audios", [])

633 + audios = mm_data.get("audio", [])

634 if not audios:

635 processed_data = BatchFeature({"input_ids": input_ids})

637 - processed_data.update(passthrough_data)

638 - return processed_data

636 + return self._finalize_hf_mm_data(

637 + mm_data, hf_kwargs, passthrough_data, processed_data

638 + )

639 assert isinstance(audios, Sequence)

640

641 - feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)

641 + feature_extractor = self.info.get_feature_extractor(**hf_kwargs)

642 sr = int(feature_extractor.sampling_rate)

643 min_samples = int(getattr(feature_extractor, "n_fft", 400) or 400)

644

@@ -690,8 +690,9 @@ class FunAudioChatMultiModalProcessor(

690 }

691

692 processed_data = BatchFeature({"input_ids": input_ids, **mm_inputs})

693 - processed_data.update(passthrough_data)

694 - return processed_data

693 + return self._finalize_hf_mm_data(

694 + mm_data, hf_kwargs, passthrough_data, processed_data

695 + )

696

697 def _get_mm_fields_config(

698 self,

vllm/model_executor/models/gemma3_mm.py

+1 -1

@@ -263,7 +263,7 @@ class Gemma3DummyInputsBuilder(BaseDummyInputsBuilder[Gemma3ProcessingInfo]):

263

264

265 class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):

266 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

266 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

267 return self.dummy_inputs.get_dummy_text(mm_counts)

268

269 def _postprocess_hf_mm_data(

vllm/model_executor/models/gemma3n_mm.py

+1 -12

@@ -265,20 +265,9 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]):

265

266

267 class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo]):

268 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

268 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

269 return self.dummy_inputs.get_dummy_text(mm_counts)

270

271 - def _preprocess_hf_mm_data(

272 - self,

273 - mm_data: Mapping[str, object],

274 - hf_processor_mm_kwargs: Mapping[str, object],

275 - ) -> tuple[Mapping[str, object], Mapping[str, object]]:

276 - mm_data = dict(mm_data)

277 - if "audios" in mm_data:

278 - mm_data["audio"] = mm_data.pop("audios")

279 -

280 - return mm_data, hf_processor_mm_kwargs

281 -

271 def _postprocess_hf_mm_data(

272 self,

273 mm_data: Mapping[str, object],

vllm/model_executor/models/gemma4_mm.py

+20 -23

@@ -595,22 +595,25 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):

595

596

597 class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):

598 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

599 + return self.dummy_inputs.get_dummy_text(mm_counts)

600 +

601 def _apply_hf_processor_main(

602 self,

603 mm_items: MultiModalDataItems,

601 - hf_processor_mm_kwargs: Mapping[str, object],

604 + hf_kwargs: Mapping[str, object],

605 ) -> BatchFeature:

603 - valid_mm_items = mm_items.select(

604 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

606 + hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(

607 + mm_items, hf_kwargs

608 )

606 - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

609

608 - if not mm_data:

609 - return BatchFeature(dict(passthrough_data))

610 + if not hf_data:

611 + return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)

612

611 - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())

613 + prompt_text = hf_data.pop("text")

614 + assert isinstance(prompt_text, str)

615

613 - merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)

616 + merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_kwargs)

617 val, is_top_level_max_soft_tokens = _get_max_soft_tokens(merged_kwargs)

618

619 if val is not None and val not in _SUPPORTED_SOFT_TOKENS:

@@ -619,14 +622,12 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):

622 f"Valid values are {_SUPPORTED_SOFT_TOKENS}."

623 )

624

622 - mm_data = dict(mm_data)

623 -

625 # ---- VIDEO HANDLING ----

626 # Gemma4 decomposes video into timestamped image frames.

627 # Each frame is processed with max_soft_tokens=70 through the

628 # same vision tower, matching transformers processing_gemma4.py.

629 video_outputs: dict[str, Any] = {}

629 - if videos := mm_data.pop("videos", []):

630 + if videos := hf_data.pop("videos", []):

631 assert isinstance(videos, list)

632 processor = self.info.get_hf_processor()

633

@@ -656,7 +657,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):

657 timestamps = [idx / fps for idx in frame_indices]

658

659 # Process frames as images with max_soft_tokens=70

659 - video_mm_kwargs = dict(hf_processor_mm_kwargs)

660 + video_mm_kwargs = dict(hf_kwargs)

661 # Override configured image options too: these inputs are video

662 # frames, whose cache keys exclude images_kwargs.

663 video_mm_kwargs["images_kwargs"] = {}

@@ -726,18 +727,14 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):

727 "video_timestamps": video_timestamps_per_video,

728 }

729

729 - # The processor accepts 'audio' not 'audios'.

730 - if "audios" in mm_data:

731 - mm_data["audio"] = mm_data.pop("audios")

732 -

730 # Warn if any audio waveform exceeds the model's max duration.

734 - if "audio" in mm_data:

731 + if "audio" in hf_data:

732 processor = self.info.get_hf_processor()

733 sr = processor.feature_extractor.sampling_rate

734 max_tokens = processor.audio_seq_length

735 ms_per_tok = processor.audio_ms_per_token

736 max_duration_s = max_tokens * ms_per_tok / 1000.0

740 - audios = mm_data["audio"]

737 + audios = hf_data["audio"]

738 if not isinstance(audios, (list, tuple)):

739 audios = [audios]

740 for i, waveform in enumerate(audios):

@@ -760,13 +757,13 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):

757 # NOTE: This requires a corresponding type annotation on the

758 # HF side (Gemma4ProcessorKwargs.images_kwargs) so that

759 # _merge_kwargs routes max_soft_tokens into images_kwargs.

763 - patched_mm_kwargs = dict(hf_processor_mm_kwargs)

760 + patched_mm_kwargs = dict(hf_kwargs)

761 if val is not None and is_top_level_max_soft_tokens:

762 patched_mm_kwargs["max_soft_tokens"] = val

763

764 processed_data = self.info.ctx.call_hf_processor(

765 self.info.get_hf_processor(**patched_mm_kwargs),

769 - dict(text=prompt_text, **mm_data),

766 + dict(text=prompt_text, **hf_data),

767 patched_mm_kwargs,

768 )

769

@@ -797,9 +794,9 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):

794 processed_data["input_features_mask"] = unpadded_masks

795

796 processed_data.update(video_outputs)

800 - processed_data.update(passthrough_data)

801 -

802 - return processed_data

797 + return self._finalize_hf_mm_data(

798 + hf_data, hf_kwargs, passthrough_data, processed_data

799 + )

800

801 def _get_mm_fields_config(

802 self,

vllm/model_executor/models/glm4_1v.py

+30 -23

@@ -1582,52 +1582,59 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):

1582

1583 return prepared_data, prepared_kwargs

1584

1585 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

1586 + return self.dummy_inputs.get_dummy_text(mm_counts)

1587 +

1588 def _apply_hf_processor_main(

1589 self,

1590 mm_items: MultiModalDataItems,

1588 - hf_processor_mm_kwargs: Mapping[str, object],

1591 + hf_kwargs: Mapping[str, object],

1592 ) -> BatchFeature:

1590 - valid_mm_items = mm_items.select(

1591 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

1593 + hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(

1594 + mm_items, hf_kwargs

1595 )

1593 - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

1594 -

1595 - if not mm_data:

1596 - return BatchFeature(dict(passthrough_data))

1596

1598 - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())

1597 + if not hf_data:

1598 + return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)

1599

1600 - mm_data = dict(mm_data)

1600 + prompt_text = hf_data.pop("text")

1601 + assert isinstance(prompt_text, str)

1602

1602 - processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)

1603 + processor = self.info.get_hf_processor(**hf_kwargs)

1604

1605 use_direct_path = (

1606 not isinstance(processor, Glm4vProcessor) and TRANSFORMERS_WITH_GA

1607 )

1608 if use_direct_path:

1609 prepared_data, prepared_kwargs = self._get_direct_path_inputs(

1609 - mm_data, hf_processor_mm_kwargs

1610 + hf_data, hf_kwargs

1611 )

1612 processed_data = self.info.ctx.call_hf_processor(

1613 self.info.get_hf_processor(**prepared_kwargs),

1614 dict(text=prompt_text, **prepared_data),

1615 prepared_kwargs,

1616 )

1616 - processed_data.update(passthrough_data)

1617 - return processed_data

1617 + return self._finalize_hf_mm_data(

1618 + hf_data, hf_kwargs, passthrough_data, processed_data

1619 + )

1620

1619 - videos = mm_data.get("videos")

1620 - if isinstance(videos, list) and videos:

1621 + if (

1622 + "videos" in hf_data

1623 + and isinstance(videos := hf_data["videos"], list)

1624 + and len(videos) > 0

1625 + ):

1626 + # Videos are processed item-wise below; remove them from hf_data so

1627 + # the combined call at the end does not re-sample the raw frames.

1628 + del hf_data["videos"]

1629 video_grid_thw_lst = []

1630 pixel_values_videos_lst = []

1631 frame_embed_token_id = self.info._get_video_frame_embed_token_id(processor)

1632 swap_video_frame_tokens = frame_embed_token_id == processor.image_token_id

1625 - mm_data.pop("videos")

1633 for item in videos:

1634 video_array, metadata = item

1635

1636 # don't update mm_kwargs inplace

1630 - video_mm_kwargs = dict(**hf_processor_mm_kwargs)

1637 + video_mm_kwargs = dict(**hf_kwargs)

1638 video_mm_kwargs["do_sample_frames"] = metadata.get(

1639 "do_sample_frames", True

1640 )

@@ -1667,9 +1674,9 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):

1674 swap_video_frame_tokens = False

1675

1676 processed_data = self.info.ctx.call_hf_processor(

1670 - self.info.get_hf_processor(**hf_processor_mm_kwargs),

1671 - dict(text=prompt_text, **mm_data),

1672 - hf_processor_mm_kwargs,

1677 + self.info.get_hf_processor(**hf_kwargs),

1678 + dict(text=prompt_text, **hf_data),

1679 + hf_kwargs,

1680 )

1681 if swap_video_frame_tokens:

1682 input_ids = processed_data["input_ids"]

@@ -1677,9 +1684,9 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):

1684 processed_data["input_ids"] = input_ids

1685

1686 processed_data.update(video_outputs)

1680 - processed_data.update(passthrough_data)

1681 -

1682 - return processed_data

1687 + return self._finalize_hf_mm_data(

1688 + hf_data, hf_kwargs, passthrough_data, processed_data

1689 + )

1690

1691 def _get_mm_fields_config(

1692 self,

vllm/model_executor/models/glmasr.py

+14 -15

@@ -44,6 +44,7 @@ from vllm.multimodal.processing import (

44 PromptUpdate,

45 PromptUpdateDetails,

46 )

47 +from vllm.multimodal.processing.processor import HFMultiModalInputs

48 from vllm.sequence import IntermediateTensors

49 from vllm.tokenizers import cached_tokenizer_from_config

50 from vllm.transformers_utils.processor import cached_processor_from_config

@@ -739,26 +740,24 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])

740 chunk_counts.append(min(n_chunks, max_windows))

741 return chunk_counts

742

742 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

743 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

744 return self.dummy_inputs.get_dummy_text(mm_counts)

745

745 - def _preprocess_hf_mm_data(

746 + def _get_hf_mm_inputs(

747 self,

747 - mm_data: Mapping[str, object],

748 - hf_processor_mm_kwargs: Mapping[str, object],

749 - ) -> tuple[Mapping[str, object], Mapping[str, object]]:

750 - mm_data = dict(mm_data)

751 - if "audios" in mm_data:

752 - mm_data["audio"] = mm_data.pop("audios")

753 -

754 - feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)

755 - hf_processor_mm_kwargs = dict(

756 - **hf_processor_mm_kwargs,

757 - sampling_rate=feature_extractor.sampling_rate,

748 + mm_items: MultiModalDataItems,

749 + hf_kwargs: Mapping[str, object],

750 + ) -> HFMultiModalInputs:

751 + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)

752 +

753 + feature_extractor = self.info.get_feature_extractor(**hf_kwargs)

754 + return hf_inputs._replace(

755 + hf_kwargs=dict(

756 + hf_inputs.hf_kwargs,

757 + sampling_rate=feature_extractor.sampling_rate,

758 + )

759 )

760

760 - return mm_data, hf_processor_mm_kwargs

761 -

761 def _postprocess_hf_mm_data(

762 self,

763 mm_data: Mapping[str, object],

vllm/model_executor/models/granite_speech.py

+5 -19

@@ -110,7 +110,7 @@ class GraniteSpeechAudioInputs(TensorSchema):

110 """List of audio embedding sizes for each item in batch."""

111

112

113 -class GraniteSpeechMultiModalProcessingInfo(BaseProcessingInfo):

113 +class GraniteSpeechProcessingInfo(BaseProcessingInfo):

114 def get_data_parser(self):

115 feature_extractor = self.get_hf_processor().audio_processor

116

@@ -135,7 +135,7 @@ class GraniteSpeechMultiModalProcessingInfo(BaseProcessingInfo):

135

136 ### Input Processing & Multimodal utils

137 class GraniteSpeechMultiModalProcessor(

138 - BaseMultiModalProcessor[GraniteSpeechMultiModalProcessingInfo]

138 + BaseMultiModalProcessor[GraniteSpeechProcessingInfo]

139 ):

140 def _get_mm_fields_config(

141 self,

@@ -181,23 +181,9 @@ class GraniteSpeechMultiModalProcessor(

181 )

182 ]

183

184 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

184 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

185 return self.dummy_inputs.get_dummy_text(mm_counts)

186

187 - def _preprocess_hf_mm_data(

188 - self,

189 - mm_data: Mapping[str, object],

190 - hf_processor_mm_kwargs: Mapping[str, object],

191 - ) -> tuple[Mapping[str, object], Mapping[str, object]]:

192 - mm_data = dict(mm_data)

193 - audios = mm_data.pop("audios", [])

194 -

195 - if audios:

196 - # GraniteSpeechFeatureExtractor accepts "audio"

197 - mm_data["audio"] = audios

198 -

199 - return mm_data, hf_processor_mm_kwargs

200 -

187 def _postprocess_hf_mm_data(

188 self,

189 mm_data: Mapping[str, object],

@@ -216,7 +202,7 @@ class GraniteSpeechMultiModalProcessor(

202

203

204 class GraniteSpeechDummyInputsBuilder(

219 - BaseDummyInputsBuilder[GraniteSpeechMultiModalProcessingInfo]

205 + BaseDummyInputsBuilder[GraniteSpeechProcessingInfo]

206 ):

207 def get_dummy_mm_data(

208 self,

@@ -565,7 +551,7 @@ class GraniteSpeechCTCEncoder(nn.Module):

551

552 @MULTIMODAL_REGISTRY.register_processor(

553 GraniteSpeechMultiModalProcessor,

568 - info=GraniteSpeechMultiModalProcessingInfo,

554 + info=GraniteSpeechProcessingInfo,

555 dummy_inputs=GraniteSpeechDummyInputsBuilder,

556 )

557 class GraniteSpeechForConditionalGeneration(

vllm/model_executor/models/granite_speech_plus.py

+2 -2

@@ -12,8 +12,8 @@ from .granite_speech import (

12 GraniteSpeechCTCEncoder,

13 GraniteSpeechDummyInputsBuilder,

14 GraniteSpeechForConditionalGeneration,

15 - GraniteSpeechMultiModalProcessingInfo,

15 GraniteSpeechMultiModalProcessor,

16 + GraniteSpeechProcessingInfo,

17 )

18

19 ISO639_1_SUPPORTED_LANGS = {

@@ -60,7 +60,7 @@ class GraniteSpeechPlusCTCEncoder(GraniteSpeechCTCEncoder):

60

61 @MULTIMODAL_REGISTRY.register_processor(

62 GraniteSpeechMultiModalProcessor,

63 - info=GraniteSpeechMultiModalProcessingInfo,

63 + info=GraniteSpeechProcessingInfo,

64 dummy_inputs=GraniteSpeechDummyInputsBuilder,

65 )

66 class GraniteSpeechPlusForConditionalGeneration(GraniteSpeechForConditionalGeneration):

vllm/model_executor/models/h2ovl.py

+2 -2

@@ -21,7 +21,7 @@ from vllm.multimodal.parse import (

21 MultiModalDataItems,

22 )

23 from vllm.multimodal.processing.processor import (

24 - MultiModalProcessingInfo,

24 + MultiModalProcessingResult,

25 ProcessorInputs,

26 PromptReplacement,

27 TimingContext,

@@ -139,7 +139,7 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn

139 self,

140 inputs: ProcessorInputs,

141 timing_ctx: TimingContext,

142 - ) -> MultiModalProcessingInfo:

142 + ) -> MultiModalProcessingResult:

143 # The processor logic is different for len(images) <= 1 vs > 1

144 # Since the processing cache assumes that the processor output is

145 # invariant of how many images are passed per prompt, we only

vllm/model_executor/models/hyperclovax_vision_v2.py

+2 -38

@@ -261,44 +261,8 @@ class HCXVisionV2MultiModalProcessor(

261 ):

262 """Multimodal processor for HyperCLOVAX V2 (32B Think model)."""

263

264 - def _apply_hf_processor_main(

265 - self,

266 - mm_items: MultiModalDataItems,

267 - hf_processor_mm_kwargs: Mapping[str, object],

268 - ) -> BatchFeature:

269 - valid_mm_items = mm_items.select(

270 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

271 - )

272 - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

273 -

274 - if not mm_data:

275 - return BatchFeature(dict(passthrough_data))

276 -

277 - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())

278 -

279 - images = mm_data.get("images")

280 - videos = mm_data.get("videos")

281 -

282 - # Get the HF processor

283 - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)

284 -

285 - # Build data dict for HF processor (images/videos only)

286 - # NOTE: We pass the prompt as-is without token normalization.

287 - # Token expansion is handled by vLLM via _get_prompt_updates.

288 - data: dict[str, object] = dict(

289 - text=prompt_text,

290 - images=images,

291 - videos=videos,

292 - )

293 -

294 - processed_data = self.info.ctx.call_hf_processor(

295 - hf_processor=hf_processor,

296 - data=data,

297 - kwargs=hf_processor_mm_kwargs,

298 - )

299 - processed_data.update(passthrough_data)

300 -

301 - return processed_data

264 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

265 + return self.dummy_inputs.get_dummy_text(mm_counts)

266

267 def _get_prompt_updates(

268 self,

vllm/model_executor/models/idefics3.py

+21 -21

@@ -52,6 +52,7 @@ from vllm.multimodal.processing import (

52 PromptUpdateDetails,

53 cached_encode,

54 )

55 +from vllm.multimodal.processing.processor import HFMultiModalInputs

56 from vllm.sequence import IntermediateTensors

57 from vllm.utils.gpu_sync_debug import gpu_sync_allowed

58 from vllm.utils.tensor_schema import TensorSchema, TensorShape

@@ -240,31 +241,32 @@ class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo])

241

242

243 class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo]):

243 - def _apply_hf_processor_main(

244 + def _get_hf_mm_inputs(

245 self,

246 mm_items: MultiModalDataItems,

246 - hf_processor_mm_kwargs: Mapping[str, object],

247 - ) -> BatchFeature:

248 - valid_mm_items = mm_items.select(

249 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

250 - )

251 - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

252 -

253 - if not mm_data:

254 - return BatchFeature(dict(passthrough_data))

247 + hf_kwargs: Mapping[str, object],

248 + ) -> HFMultiModalInputs:

249 + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)

250

251 image_processor = self.info.get_hf_processor().image_processor

252 if getattr(image_processor, "backend", "pil") == "pil":

258 - hf_processor_mm_kwargs = {

259 - "input_data_format": "channels_last",

260 - **hf_processor_mm_kwargs,

261 - }

253 + hf_inputs = hf_inputs._replace(

254 + hf_kwargs={

255 + "input_data_format": "channels_last",

256 + **hf_inputs.hf_kwargs,

257 + }

258 + )

259

263 - processed_data = self.info.ctx.call_hf_processor(

264 - self.info.get_hf_processor(**hf_processor_mm_kwargs),

265 - mm_data,

266 - hf_processor_mm_kwargs,

267 - )

260 + return hf_inputs

261 +

262 + def _postprocess_hf_mm_data(

263 + self,

264 + mm_data: Mapping[str, object],

265 + hf_processor_mm_kwargs: Mapping[str, object],

266 + processed_data: BatchFeature,

267 + ) -> BatchFeature:

268 + if not mm_data:

269 + return processed_data

270

271 images = mm_data.get("images", [])

272 mm_items = self.info.parse_mm_data({"image": images}, validate=False)

@@ -289,8 +291,6 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo

291 processed_data["pixel_values"].squeeze_(0)

292 processed_data["pixel_attention_mask"].squeeze_(0)

293

292 - processed_data.update(passthrough_data)

293 -

294 return processed_data

295

296 def _get_mm_fields_config(

vllm/model_executor/models/interns1.py

+20 -18

@@ -335,28 +335,30 @@ class InternS1DummyInputsBuilder(BaseDummyInputsBuilder[InternS1ProcessingInfo])

335 class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo]):

336 """Basic image-only MultiModalProcessor for InternS1-style models."""

337

338 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

339 + return self.dummy_inputs.get_dummy_text(mm_counts)

340 +

341 def _apply_hf_processor_main(

342 self,

343 mm_items: MultiModalDataItems,

341 - hf_processor_mm_kwargs: Mapping[str, object],

344 + hf_kwargs: Mapping[str, object],

345 ) -> BatchFeature:

343 - valid_mm_items = mm_items.select(

344 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

346 + hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(

347 + mm_items, hf_kwargs

348 )

346 - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

349

348 - if not mm_data:

349 - return BatchFeature(dict(passthrough_data))

350 + if not hf_data:

351 + return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)

352

351 - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())

353 + prompt_text = hf_data.pop("text")

354 + assert isinstance(prompt_text, str)

355

353 - mm_data = dict(mm_data)

354 - videos = mm_data.pop("videos", [])

355 - images = mm_data.pop("images", [])

356 + videos = hf_data.pop("videos", [])

357 + images = hf_data.pop("images", [])

358 assert isinstance(videos, list)

359 assert isinstance(images, list)

360

359 - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)

361 + hf_processor = self.info.get_hf_processor(**hf_kwargs)

362 tokenizer = hf_processor.tokenizer

363 vocab = tokenizer.get_vocab()

364

@@ -374,9 +376,9 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo

376 image_pixel_values = []

377 for image in images:

378 processed_data = self.info.ctx.call_hf_processor(

377 - self.info.get_hf_processor(**hf_processor_mm_kwargs),

379 + self.info.get_hf_processor(**hf_kwargs),

380 dict(text=hf_processor.image_token, **{"images": image}),

379 - hf_processor_mm_kwargs,

381 + hf_kwargs,

382 )

383 image_pixel_values.append(processed_data.pop("pixel_values"))

384

@@ -398,9 +400,9 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo

400 video_pixel_values = []

401 for video in videos:

402 processed_data = self.info.ctx.call_hf_processor(

401 - self.info.get_hf_processor(**hf_processor_mm_kwargs),

403 + self.info.get_hf_processor(**hf_kwargs),

404 dict(text=hf_processor.video_token, **{"videos": video}),

403 - hf_processor_mm_kwargs,

405 + hf_kwargs,

406 )

407 video_pixel_values.append(processed_data.pop("pixel_values"))

408

@@ -430,9 +432,9 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo

432 processed_data.update(text_outputs)

433 processed_data.update(image_outputs)

434 processed_data.update(video_outputs)

433 - processed_data.update(passthrough_data)

434 -

435 - return processed_data

435 + return self._finalize_hf_mm_data(

436 + hf_data, hf_kwargs, passthrough_data, processed_data

437 + )

438

439 def _get_mm_fields_config(

440 self,

vllm/model_executor/models/jina_vl.py

+8 -7

@@ -13,7 +13,7 @@ from vllm.model_executor.layers.pooler import DispatchPooler

13 from vllm.multimodal import MULTIMODAL_REGISTRY

14 from vllm.multimodal.inputs import MultiModalKwargsItems

15 from vllm.multimodal.processing.processor import (

16 - MultiModalProcessingInfo,

16 + MultiModalProcessingResult,

17 ProcessorInputs,

18 TimingContext,

19 )

@@ -63,19 +63,19 @@ class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor):

63 self,

64 inputs: ProcessorInputs,

65 timing_ctx: TimingContext,

66 - ) -> MultiModalProcessingInfo:

67 - mm_info = super()._cached_apply_hf_processor(inputs, timing_ctx)

66 + ) -> MultiModalProcessingResult:

67 + mm_res = super()._cached_apply_hf_processor(inputs, timing_ctx)

68

69 # Score inputs are query-first, while the prompt template is document-first.

70 mm_kwargs = MultiModalKwargsItems(

71 {

72 modality: list(reversed(items))

73 - for modality, items in mm_info.kwargs.items()

73 + for modality, items in mm_res.kwargs.items()

74 }

75 )

76 mm_hashes = {

77 modality: list(reversed(hashes))

78 - for modality, hashes in mm_info.hashes.items()

78 + for modality, hashes in mm_res.hashes.items()

79 }

80 mm_prompt_updates = {

81 modality: [

@@ -85,10 +85,11 @@ class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor):

85 ]

86 for item_idx, updates in enumerate(reversed(item_updates))

87 ]

88 - for modality, item_updates in mm_info.prompt_updates.items()

88 + for modality, item_updates in mm_res.prompt_updates.items()

89 }

90

91 - return MultiModalProcessingInfo(

91 + return MultiModalProcessingResult(

92 + prompt_ids=mm_res.prompt_ids,

93 kwargs=mm_kwargs,

94 hashes=mm_hashes,

95 prompt_updates=mm_prompt_updates,

vllm/model_executor/models/kanana_v.py

+14 -10

@@ -459,24 +459,27 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])

459 def media_token_id(self) -> int:

460 return self.info.get_hf_config().text_config.eos_token_id + 1

461

462 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

463 + return self.dummy_inputs.get_dummy_text(mm_counts)

464 +

465 def _apply_hf_processor_main(

466 self,

467 mm_items: MultiModalDataItems,

465 - hf_processor_mm_kwargs: Mapping[str, object],

468 + hf_kwargs: Mapping[str, object],

469 ) -> BatchFeature:

470 """Run the underlying HF processor on text and image data."""

468 - valid_mm_items = mm_items.select(

469 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

471 + hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(

472 + mm_items, hf_kwargs

473 )

471 - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

474

473 - if not mm_data or not mm_data.get("images", []):

474 - return BatchFeature(dict(passthrough_data))

475 + if not hf_data or not hf_data.get("images", []):

476 + return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)

477

476 - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())

478 + prompt_text = hf_data.pop("text")

479 + assert isinstance(prompt_text, str)

480

481 # Images

479 - image_inputs = mm_data.get("images", [])

482 + image_inputs = hf_data.get("images", [])

483 pixel_sizes = []

484 if not isinstance(image_inputs[0], Image.Image):

485 image_inputs = [Image.fromarray(image) for image in image_inputs]

@@ -530,8 +533,9 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])

533 pixel_sizes=torch.tensor(pixel_sizes),

534 )

535 processed_data = BatchFeature(combined_outputs, tensor_type="pt")

533 - processed_data.update(passthrough_data)

534 - return processed_data

536 + return self._finalize_hf_mm_data(

537 + hf_data, hf_kwargs, passthrough_data, processed_data

538 + )

539

540 def _get_prompt_updates(

541 self,

vllm/model_executor/models/keye.py

+1 -1

@@ -1160,7 +1160,7 @@ class KeyeDummyInputsBuilder(KeyeBaseDummyInputsBuilder[KeyeProcessingInfo]):

1160

1161

1162 class KeyeMultiModalProcessor(BaseMultiModalProcessor[KeyeProcessingInfo]):

1163 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

1163 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

1164 return self.dummy_inputs.get_dummy_text(mm_counts)

1165

1166 def _get_prompt_updates(

vllm/model_executor/models/keye_vl1_5.py

+1 -1

@@ -373,7 +373,7 @@ class KeyeVL1_5ProcessingInfo(KeyeProcessingInfo):

373

374

375 class KeyeVL1_5MultiModalProcessor(BaseMultiModalProcessor[KeyeVL1_5ProcessingInfo]):

376 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

376 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

377 return self.dummy_inputs.get_dummy_text(mm_counts)

378

379 def _get_prompt_updates(

vllm/model_executor/models/kimi_audio.py

+8 -24

@@ -47,6 +47,7 @@ from vllm.multimodal.processing import (

47 )

48 from vllm.multimodal.processing.processor import (

49 BaseMultiModalProcessor,

50 + HFMultiModalInputs,

51 ProcessorInputs,

52 )

53 from vllm.sequence import IntermediateTensors

@@ -227,27 +228,17 @@ class KimiAudioMultiModalDataParser(MultiModalDataParser):

228 class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingInfo]):

229 """vLLM multi-modal processor wrapper for Kimi-Audio."""

230

230 - def _apply_hf_processor_main(

231 + def _get_hf_mm_inputs(

232 self,

233 mm_items: MultiModalDataItems,

233 - hf_processor_mm_kwargs: Mapping[str, object],

234 - ) -> BatchFeature:

235 - """Call the HuggingFace processor."""

236 - valid_mm_items = mm_items.select(

237 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

238 - )

239 - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

240 -

241 - if not mm_data:

242 - return BatchFeature(dict(passthrough_data))

243 -

244 - # Convert mm_data format: {'audios': [...]} -> {'audio': ...}

245 - mm_data = dict(mm_data)

246 - audios = mm_data.pop("audios", [])

234 + hf_kwargs: Mapping[str, object],

235 + ) -> HFMultiModalInputs:

236 + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)

237

238 # Convert audio format: [(array, sr), ...] -> [array, ...]

239 # KimiAudioProcessor expects raw numpy arrays

250 - if audios:

240 + mm_data = hf_inputs.hf_data

241 + if audios := mm_data.pop("audio", []):

242 audio_arrays = []

243 for aud in audios:

244 if isinstance(aud, (tuple, list)) and len(aud) == 2:

@@ -259,14 +250,7 @@ class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingIn

250 audio_arrays.append(aud)

251 mm_data["audio"] = audio_arrays

252

262 - # Use the context's call_hf_processor for proper handling

263 - processed_data = self.info.ctx.call_hf_processor(

264 - self.info.get_hf_processor(**hf_processor_mm_kwargs),

265 - mm_data,

266 - hf_processor_mm_kwargs,

267 - )

268 - processed_data.update(passthrough_data)

269 - return processed_data

253 + return hf_inputs

254

255 def _get_mm_fields_config(

256 self,

vllm/model_executor/models/kimi_vl.py

+1 -1

@@ -236,7 +236,7 @@ class KimiVLDummyInputsBuilder(BaseDummyInputsBuilder[KimiVLProcessingInfo]):

236

237

238 class KimiVLMultiModalProcessor(BaseMultiModalProcessor[KimiVLProcessingInfo]):

239 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

239 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

240 return self.dummy_inputs.get_dummy_text(mm_counts)

241

242 def _get_mm_fields_config(

vllm/model_executor/models/lfm2_vl.py

+1 -1

@@ -406,7 +406,7 @@ class Lfm2VLDummyInputsBuilder(BaseDummyInputsBuilder[Lfm2VLProcessingInfo]):

406

407

408 class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):

409 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

409 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

410 return self.dummy_inputs.get_dummy_text(mm_counts)

411

412 def _postprocess_hf_mm_data(

vllm/model_executor/models/lightonocr.py

+2 -26

@@ -1,7 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 from collections.abc import Iterable, Mapping, Sequence

4 -from typing import TypeVar

4

5 import torch

6 import torch.nn as nn

@@ -35,8 +34,6 @@ from vllm.multimodal.processing import (

34 PromptUpdateDetails,

35 )

36

38 -_I = TypeVar("_I", bound=Mistral3ProcessingInfo)

39 -

37

38 class LightOnOCRProcessingInfo(Mistral3ProcessingInfo):

39 def get_vision_encoder_info(

@@ -49,8 +46,8 @@ class LightOnOCRProcessingInfo(Mistral3ProcessingInfo):

46 return PixtralHFEncoderInfo(self.get_hf_config())

47

48

52 -class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):

53 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

49 +class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[LightOnOCRProcessingInfo]):

50 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

51 return self.dummy_inputs.get_dummy_text(mm_counts)

52

53 def _postprocess_hf_mm_data(

@@ -62,27 +59,6 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingIn

59 if not mm_data:

60 return processed_data

61

65 - input_ids = processed_data.get("input_ids")

66 - if input_ids is not None:

67 - processor = self.info.get_hf_processor()

68 - tokenizer = self.info.get_tokenizer()

69 - vocab = tokenizer.get_vocab()

70 -

71 - break_id = vocab.get(processor.image_break_token)

72 - end_id = vocab.get(processor.image_end_token)

73 -

74 - # create mask to remove break/end tokens

75 - keep_mask = ~torch.isin(

76 - input_ids,

77 - torch.tensor([break_id, end_id]),

78 - )

79 -

80 - processed_data["input_ids"] = input_ids[keep_mask].unsqueeze(0)

81 - if "attention_mask" in processed_data:

82 - processed_data["attention_mask"] = processed_data["attention_mask"][

83 - keep_mask

84 - ].unsqueeze(0)

85 -

62 # un-pad pixel_values per-image so caches remain independent.

63 pixel_values = processed_data.get("pixel_values")

64 if pixel_values is not None:

vllm/model_executor/models/llava.py

+1 -1

@@ -320,7 +320,7 @@ class PixtralHFProcessingInfo(BaseLlavaProcessingInfo):

320

321

322 class PixtralHFMultiModalProcessor(BaseMultiModalProcessor[PixtralHFProcessingInfo]):

323 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

323 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

324 return self.dummy_inputs.get_dummy_text(mm_counts)

325

326 def _postprocess_hf_mm_data(

vllm/model_executor/models/llava_next.py

+1 -1

@@ -209,7 +209,7 @@ class BaseLlavaNextMultiModalProcessor(BaseLlavaMultiModalProcessor[_I]):

209

210

211 class LlavaNextMultiModalProcessor(BaseLlavaNextMultiModalProcessor[_I]):

212 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

212 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

213 return self.dummy_inputs.get_dummy_text(mm_counts)

214

215 def _get_mm_fields_config(

vllm/model_executor/models/llava_onevision.py

+25 -21

@@ -324,33 +324,36 @@ class LlavaOnevisionMultiModalProcessor(

324 pixel_values_videos=MultiModalFieldConfig.batched("video"),

325 )

326

327 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

328 + return self.dummy_inputs.get_dummy_text(mm_counts)

329 +

330 def _apply_hf_processor_main(

331 self,

332 mm_items: MultiModalDataItems,

330 - hf_processor_mm_kwargs: Mapping[str, object],

333 + hf_kwargs: Mapping[str, object],

334 ) -> BatchFeature:

332 - valid_mm_items = mm_items.select(

333 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

335 + hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(

336 + mm_items, hf_kwargs

337 )

335 - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

338

337 - if not mm_data:

338 - return BatchFeature(dict(passthrough_data))

339 + if not hf_data:

340 + return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)

341

340 - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())

342 + prompt_text = hf_data.pop("text")

343 + assert isinstance(prompt_text, str)

344

342 - mm_data = dict(mm_data)

343 - videos = mm_data.pop("videos", [])

345 + videos = hf_data.pop("videos", [])

346 assert isinstance(videos, list)

347

348 if not videos:

349 processed_data = self.info.ctx.call_hf_processor(

348 - self.info.get_hf_processor(**hf_processor_mm_kwargs),

349 - dict(text=prompt_text, **mm_data),

350 - hf_processor_mm_kwargs,

350 + self.info.get_hf_processor(**hf_kwargs),

351 + dict(text=prompt_text, **hf_data),

352 + hf_kwargs,

353 + )

354 + return self._finalize_hf_mm_data(

355 + hf_data, hf_kwargs, passthrough_data, processed_data

356 )

352 - processed_data.update(passthrough_data)

353 - return processed_data

357

358 # LLaVA-OneVision processor doesn't support multiple videos

359 # with different sizes when converting back to tensors

@@ -360,13 +363,13 @@ class LlavaOnevisionMultiModalProcessor(

363 image_token = processor.image_token

364 video_token = processor.video_token

365

363 - images = mm_data.pop("images", [])

366 + images = hf_data.pop("images", [])

367 assert isinstance(images, list)

368 if images:

369 processor_outputs = self.info.ctx.call_hf_processor(

367 - self.info.get_hf_processor(**hf_processor_mm_kwargs),

370 + self.info.get_hf_processor(**hf_kwargs),

371 dict(text=image_token * len(images), **{"images": images}),

369 - hf_processor_mm_kwargs,

372 + hf_kwargs,

373 )

374 image_outputs = {

375 k: v

@@ -379,9 +382,9 @@ class LlavaOnevisionMultiModalProcessor(

382 pixel_values_videos = []

383 for video in videos:

384 item_outputs = self.info.ctx.call_hf_processor(

382 - self.info.get_hf_processor(**hf_processor_mm_kwargs),

385 + self.info.get_hf_processor(**hf_kwargs),

386 dict(text=video_token, **{"videos": video}),

384 - hf_processor_mm_kwargs,

387 + hf_kwargs,

388 )

389

390 pixel_values_videos.append(item_outputs["pixel_values_videos"][0])

@@ -394,8 +397,9 @@ class LlavaOnevisionMultiModalProcessor(

397 **video_outputs,

398 )

399 processed_data = BatchFeature(combined_outputs)

397 - processed_data.update(passthrough_data)

398 - return processed_data

400 + return self._finalize_hf_mm_data(

401 + hf_data, hf_kwargs, passthrough_data, processed_data

402 + )

403

404 def _get_prompt_updates(

405 self,

vllm/model_executor/models/llava_onevision2.py

+37 -33

@@ -1556,23 +1556,26 @@ class LlavaOnevision2MultiModalDataParser(MultiModalDataParser):

1556 class LlavaOnevision2MultiModalProcessor(

1557 BaseMultiModalProcessor[LlavaOnevision2ProcessingInfo]

1558 ):

1559 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

1560 + return self.dummy_inputs.get_dummy_text(mm_counts)

1561 +

1562 def _apply_hf_processor_main(

1563 self,

1564 mm_items: MultiModalDataItems,

1562 - hf_processor_mm_kwargs: Mapping[str, object],

1565 + hf_kwargs: Mapping[str, object],

1566 ) -> BatchFeature:

1564 - valid_mm_items = mm_items.select(

1565 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

1567 + hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(

1568 + mm_items, hf_kwargs

1569 )

1567 - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

1570

1569 - if not mm_data:

1570 - return BatchFeature(dict(passthrough_data))

1571 + if not hf_data:

1572 + return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)

1573

1572 - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())

1574 + prompt_text = hf_data.pop("text")

1575 + assert isinstance(prompt_text, str)

1576

1574 - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)

1575 - merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)

1577 + hf_processor = self.info.get_hf_processor(**hf_kwargs)

1578 + merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_kwargs)

1579 merged_kwargs.setdefault("return_tensors", "pt")

1580 call_kwargs = {

1581 k: v

@@ -1589,17 +1592,17 @@ class LlavaOnevision2MultiModalProcessor(

1592 "codec_config",

1593 }

1594 }

1592 - mm_data = dict(mm_data)

1595 +

1596 # Explicit None + length checks: ``mm_data[...]`` may be a list, numpy

1597 # array, or tensor, and ``and <array>`` would raise on the ambiguous

1598 # truth value of a multi-element array.

1596 - _videos = mm_data.get("videos")

1599 + _videos = hf_data.get("videos")

1600 if _videos is not None and not isinstance(_videos, Sized):

1601 raise TypeError(f"object of type '{type(_videos).__name__}' has no len()")

1602 videos_present = _videos is not None and len(_videos) > 0

1603

1604 codec_video_paths = (

1602 - _extract_codec_video_paths(mm_data["videos"]) if videos_present else None

1605 + _extract_codec_video_paths(hf_data["videos"]) if videos_present else None

1606 )

1607 is_codec_marker = codec_video_paths is not None

1608 # Fallback: caller passed video_backend=codec via mm_processor_kwargs

@@ -1613,7 +1616,7 @@ class LlavaOnevision2MultiModalProcessor(

1616 and call_kwargs.get("video_backend") == "codec"

1617 )

1618 if is_codec_kwarg:

1616 - raw = mm_data["videos"]

1619 + raw = hf_data["videos"]

1620 if isinstance(raw, str):

1621 codec_video_paths = [raw]

1622 elif isinstance(raw, (list, tuple)) and all(

@@ -1638,7 +1641,7 @@ class LlavaOnevision2MultiModalProcessor(

1641 # Codec backend: HF processor consumes the path string directly and

1642 # performs decode + canvas-packing internally. The dummy ndarray

1643 # we attached during prepare_codec_video_input is discarded here.

1641 - mm_data["videos"] = (

1644 + hf_data["videos"] = (

1645 codec_video_paths

1646 if len(codec_video_paths) > 1

1647 else codec_video_paths[0]

@@ -1648,11 +1651,9 @@ class LlavaOnevision2MultiModalProcessor(

1651 # ``video_backend="codec"`` via mm_kwargs so the wrapped processor

1652 # dispatches to its codec branch.

1653 output = self.info.ctx.call_hf_processor(

1651 - self.info.get_hf_processor(

1652 - **{**hf_processor_mm_kwargs, "video_backend": "codec"}

1653 - ),

1654 - dict(text=prompt_text, **mm_data),

1655 - {**hf_processor_mm_kwargs, "video_backend": "codec"},

1654 + self.info.get_hf_processor(**{**hf_kwargs, "video_backend": "codec"}),

1655 + dict(text=prompt_text, **hf_data),

1656 + {**hf_kwargs, "video_backend": "codec"},

1657 )

1658 data = dict(output)

1659 processed_data = BatchFeature(

@@ -1660,8 +1661,9 @@ class LlavaOnevision2MultiModalProcessor(

1661 data, codec_video_paths, hf_processor

1662 )

1663 )

1663 - processed_data.update(passthrough_data)

1664 - return processed_data

1664 + return self._finalize_hf_mm_data(

1665 + hf_data, hf_kwargs, passthrough_data, processed_data

1666 + )

1667

1668 # ---- Frame backend (registered LlavaOnevision2VideoBackend) ------

1669 # Every non-codec video reaches here as a ``(frames_ndarray, metadata)``

@@ -1676,7 +1678,7 @@ class LlavaOnevision2MultiModalProcessor(

1678 # provided by the backend's ``compute_frames_index_to_sample``; SSRF /

1679 # local-file gating is enforced by the connector before decoding.

1680 if videos_present:

1679 - timestamp_decimals = hf_processor_mm_kwargs.get(

1681 + timestamp_decimals = hf_kwargs.get(

1682 "timestamp_decimals", _DEFAULT_TIMESTAMP_DECIMALS

1683 )

1684 if not isinstance(

@@ -1690,7 +1692,7 @@ class LlavaOnevision2MultiModalProcessor(

1692

1693 per_video_frames: list[list[Image.Image]] = []

1694 per_video_timestamps: list[list[float]] = []

1693 - videos = mm_data["videos"]

1695 + videos = hf_data["videos"]

1696 if not isinstance(videos, Iterable):

1697 raise TypeError(f"Unsupported videos input: {type(videos)}")

1698 for item in videos:

@@ -1714,7 +1716,7 @@ class LlavaOnevision2MultiModalProcessor(

1716 # order) -- otherwise mixed image+video requests bind frames to the

1717 # wrong placeholder. ``row_is_video`` labels each grid row so outputs

1718 # can be split back into per-modality keys below.

1717 - merged_mm_data = dict(mm_data)

1719 + merged_mm_data = dict(hf_data)

1720 existing_images = merged_mm_data.pop("images", None)

1721 caller_images: list[Image.Image] = []

1722 if existing_images:

@@ -1760,9 +1762,9 @@ class LlavaOnevision2MultiModalProcessor(

1762 # **kwargs to the image processor, so passing the full merged kwarg

1763 # set here is a no-op beyond return_tensors/padding.

1764 output = self.info.ctx.call_hf_processor(

1763 - self.info.get_hf_processor(**hf_processor_mm_kwargs),

1765 + self.info.get_hf_processor(**hf_kwargs),

1766 dict(text=new_prompt, **merged_mm_data),

1765 - hf_processor_mm_kwargs,

1767 + hf_kwargs,

1768 )

1769 data = dict(output)

1770

@@ -1820,20 +1822,22 @@ class LlavaOnevision2MultiModalProcessor(

1822 )

1823

1824 processed_data = BatchFeature(data)

1823 - processed_data.update(passthrough_data)

1824 - return processed_data

1825 + return self._finalize_hf_mm_data(

1826 + hf_data, hf_kwargs, passthrough_data, processed_data

1827 + )

1828

1829 # ---- Image-only / text-only call --------------------------------

1830 # No videos present: delegate to ``ctx.call_hf_processor``, which

1831 # runs the wrapped processor over the (possibly empty) image set and

1832 # applies float-tensor dtype postprocessing automatically.

1833 processed_data = self.info.ctx.call_hf_processor(

1831 - self.info.get_hf_processor(**hf_processor_mm_kwargs),

1832 - dict(text=prompt_text, **mm_data),

1833 - hf_processor_mm_kwargs,

1834 + self.info.get_hf_processor(**hf_kwargs),

1835 + dict(text=prompt_text, **hf_data),

1836 + hf_kwargs,

1837 + )

1838 + return self._finalize_hf_mm_data(

1839 + hf_data, hf_kwargs, passthrough_data, processed_data

1840 )

1835 - processed_data.update(passthrough_data)

1836 - return processed_data

1841

1842 def _rename_codec_outputs_to_video(

1843 self,

vllm/model_executor/models/midashenglm.py

+24 -25

@@ -61,6 +61,7 @@ from vllm.multimodal.processing import (

61 PromptUpdate,

62 PromptUpdateDetails,

63 )

64 +from vllm.multimodal.processing.processor import HFMultiModalInputs

65 from vllm.sequence import IntermediateTensors

66 from vllm.transformers_utils.configs.midashenglm import DashengConfig

67 from vllm.utils.tensor_schema import TensorSchema, TensorShape

@@ -582,35 +583,33 @@ class MiDashengLMDummyInputsBuilder(BaseDummyInputsBuilder[MiDashengLMProcessing

583 class MiDashengLMMultiModalProcessor(

584 BaseMultiModalProcessor[MiDashengLMProcessingInfo]

585 ):

585 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

586 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

587 return self.dummy_inputs.get_dummy_text(mm_counts)

588

588 - def _preprocess_hf_mm_data(

589 + def _get_hf_mm_inputs(

590 self,

590 - mm_data: Mapping[str, object],

591 - hf_processor_mm_kwargs: Mapping[str, object],

592 - ) -> tuple[Mapping[str, object], Mapping[str, object]]:

593 - mm_data = dict(mm_data)

594 - audios = mm_data.pop("audios", [])

595 - assert isinstance(audios, list)

596 -

597 - min_audio_len = self.info.get_min_audio_len()

598 - processed_audios = [

599 - np.pad(

600 - audio,

601 - (0, min_audio_len - audio.shape[-1]),

602 - mode="constant",

603 - constant_values=0,

604 - )

605 - if isinstance(audio, np.ndarray) and audio.shape[-1] < min_audio_len

606 - else audio

607 - for audio in audios

608 - ]

609 -

610 - if processed_audios:

611 - mm_data["audio"] = processed_audios

591 + mm_items: MultiModalDataItems,

592 + hf_kwargs: Mapping[str, object],

593 + ) -> HFMultiModalInputs:

594 + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)

595 +

596 + if audios := hf_inputs.hf_data.get("audio"):

597 + assert isinstance(audios, list)

598 +

599 + min_audio_len = self.info.get_min_audio_len()

600 + hf_inputs.hf_data["audio"] = [

601 + np.pad(

602 + audio,

603 + (0, min_audio_len - audio.shape[-1]),

604 + mode="constant",

605 + constant_values=0,

606 + )

607 + if isinstance(audio, np.ndarray) and audio.shape[-1] < min_audio_len

608 + else audio

609 + for audio in audios

610 + ]

611

613 - return mm_data, hf_processor_mm_kwargs

612 + return hf_inputs

613

614 def _get_mm_fields_config(

615 self,

vllm/model_executor/models/mimo_v2_omni.py

+10 -28

@@ -46,6 +46,7 @@ from vllm.multimodal.processing import (

46 PromptUpdate,

47 PromptUpdateDetails,

48 )

49 +from vllm.multimodal.processing.processor import HFMultiModalInputs

50 from vllm.transformers_utils.configs.mimo_v2_omni import Mimo_VLVisionConfig

51 from vllm.transformers_utils.processors.mimo_v2_omni import (

52 MiMoOmniProcessor,

@@ -930,27 +931,14 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing

931 fields["va_audio_features"] = MultiModalFieldConfig.batched("va_audio")

932 return fields

933

933 - def _apply_hf_processor_main(

934 + def _get_hf_mm_inputs(

935 self,

936 mm_items: MultiModalDataItems,

936 - hf_processor_mm_kwargs: Mapping[str, object],

937 - ) -> BatchFeature:

938 - """Convert numpy video arrays to (TCHW, timestamps) tuples for MiMo.

939 - Also remap 'audios' → 'audio' since MiMoOmniProcessor.__call__ uses

940 - the singular form.

941 - """

942 - valid_mm_items = mm_items.select(

943 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

944 - )

945 - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

946 -

947 - if not mm_data:

948 - return BatchFeature(dict(passthrough_data))

949 -

950 - # Remap audios → audio (MiMoOmniProcessor uses singular param name)

951 - if "audios" in mm_data:

952 - mm_data = {**mm_data, "audio": mm_data["audios"]}

953 - mm_data = {k: v for k, v in mm_data.items() if k != "audios"}

937 + hf_kwargs: Mapping[str, object],

938 + ) -> HFMultiModalInputs:

939 + """Convert numpy video arrays to (TCHW, timestamps) tuples for MiMo."""

940 + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)

941 + mm_data = hf_inputs.hf_data

942

943 # Handle video_audio items: convert video part to (TCHW, timestamps) tuple

944 if "video_audio" in mm_data:

@@ -992,7 +980,7 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing

980 audio=va_item.audio,

981 )

982 )

995 - mm_data = {**mm_data, "video_audio": va_converted}

983 + mm_data["video_audio"] = va_converted

984

985 if "videos" in mm_data:

986 converted: list[tuple[torch.Tensor, torch.Tensor]] = []

@@ -1026,15 +1014,9 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing

1014 timestamps = torch.arange(T, dtype=torch.float32) / self._INPUT_FPS

1015 converted.append((frames, timestamps))

1016

1029 - mm_data = {**mm_data, "videos": converted}

1017 + mm_data["videos"] = converted

1018

1031 - processed_data = self.info.ctx.call_hf_processor(

1032 - self.info.get_hf_processor(**hf_processor_mm_kwargs),

1033 - mm_data,

1034 - hf_processor_mm_kwargs,

1035 - )

1036 - processed_data.update(passthrough_data)

1037 - return processed_data

1019 + return hf_inputs

1020

1021 def _get_prompt_updates(

1022 self,

vllm/model_executor/models/minicpmo.py

+1 -1

@@ -445,7 +445,7 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing

445 mm_data: Mapping[str, object],

446 mm_kwargs: Mapping[str, object],

447 ) -> Mapping[str, NestedTensors]:

448 - if (audios := mm_data.get("audios")) is None:

448 + if (audios := mm_data.get("audio")) is None:

449 return {}

450

451 mm_items = self.info.parse_mm_data({"audio": audios}, validate=False)

vllm/model_executor/models/minicpmv.py

+7 -7

@@ -1059,19 +1059,18 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

1059 def _apply_hf_processor_main(

1060 self,

1061 mm_items: MultiModalDataItems,

1062 - hf_processor_mm_kwargs: Mapping[str, object],

1062 + hf_kwargs: Mapping[str, object],

1063 ) -> BatchFeature:

1064 - valid_mm_items = mm_items.select(

1065 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

1064 + mm_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(

1065 + mm_items, hf_kwargs

1066 )

1067 - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

1067

1068 prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())

1069

1070 tokenizer = self.info.get_tokenizer()

1071

1072 input_ids = torch.tensor([tokenizer.encode(prompt_text)])

1074 - mm_inputs = self.process_mm_inputs(mm_data, hf_processor_mm_kwargs)

1073 + mm_inputs = self.process_mm_inputs(mm_data, hf_kwargs)

1074

1075 processed_data = BatchFeature(

1076 {

@@ -1079,8 +1078,9 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

1078 **mm_inputs,

1079 }

1080 )

1082 - processed_data.update(passthrough_data)

1083 - return processed_data

1081 + return self._finalize_hf_mm_data(

1082 + mm_data, hf_kwargs, passthrough_data, processed_data

1083 + )

1084

1085 def _get_prompt_updates(

1086 self,

vllm/model_executor/models/mistral3.py

+2 -2

@@ -248,7 +248,7 @@ class Mistral3ProcessingInfo(BaseProcessingInfo):

248 return ImageSize(width=width, height=height)

249

250

251 -class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder[Mistral3ProcessingInfo]):

251 +class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder):

252 def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:

253 num_images = mm_counts.get("image", 0)

254

@@ -280,7 +280,7 @@ class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder[Mistral3ProcessingInfo])

280

281

282 class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):

283 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

283 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

284 return self.dummy_inputs.get_dummy_text(mm_counts)

285

286 def _postprocess_hf_mm_data(

vllm/model_executor/models/mllama4.py

+1 -1

@@ -589,7 +589,7 @@ class Mllama4ProcessingInfo(BaseProcessingInfo):

589

590

591 class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo]):

592 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

592 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

593 return self.dummy_inputs.get_dummy_text(mm_counts)

594

595 def _postprocess_hf_mm_data(

vllm/model_executor/models/molmo.py

+21 -20

@@ -1122,6 +1122,9 @@ class MolmoDummyInputsBuilder(BaseDummyInputsBuilder[MolmoProcessingInfo]):

1122

1123

1124 class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):

1125 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

1126 + return self.dummy_inputs.get_dummy_text(mm_counts)

1127 +

1128 def _postprocess_prompt(self, prompt: list[int]) -> list[int]:

1129 processor = self.info.get_hf_processor()

1130

@@ -1140,28 +1143,28 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):

1143 dict(tokens=tokens),

1144 )["input_ids"].tolist()

1145

1143 - def _apply_hf_processor_main(

1146 + def _call_hf_processor(

1147 self,

1145 - mm_items: MultiModalDataItems,

1146 - hf_processor_mm_kwargs: Mapping[str, object],

1148 + hf_data: Mapping[str, object],

1149 + hf_kwargs: Mapping[str, object],

1150 ) -> BatchFeature:

1148 - mm_counts = mm_items.get_all_counts()

1149 -

1150 - valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})

1151 - processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

1152 -

1153 - if not processor_data:

1154 - return BatchFeature(dict(passthrough_data))

1155 -

1156 - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)

1157 - prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)

1158 -

1159 - processed_data = self.info.ctx.call_hf_processor(

1151 + hf_processor = self.info.get_hf_processor(**hf_kwargs)

1152 + return self.info.ctx.call_hf_processor(

1153 hf_processor.process,

1161 - dict(text=prompt_text, **processor_data),

1162 - hf_processor_mm_kwargs,

1154 + hf_data,

1155 + hf_kwargs,

1156 )

1157

1158 + def _postprocess_hf_mm_data(

1159 + self,

1160 + hf_data: Mapping[str, object],

1161 + hf_kwargs: Mapping[str, object],

1162 + processed_data: BatchFeature,

1163 + ) -> BatchFeature:

1164 + if not hf_data:

1165 + return processed_data

1166 +

1167 + hf_processor = self.info.get_hf_processor(**hf_kwargs)

1168 tokenizer = hf_processor.tokenizer

1169 image_patch_id = tokenizer.vocab[IMAGE_PATCH_TOKEN]

1170

@@ -1169,7 +1172,7 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):

1172

1173 processed_data.pop("input_ids")

1174

1172 - if (images := processor_data.get("images")) is not None:

1175 + if (images := hf_data.get("images")) is not None:

1176 mm_items = self.info.parse_mm_data({"image": images}, validate=False)

1177 parsed_images = mm_items.get_items("image", ImageProcessorItems)

1178 image_sizes = [

@@ -1193,8 +1196,6 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):

1196 processed_data["num_crops"] = num_crops

1197 processed_data["img_patch_id"] = image_patch_id

1198

1196 - processed_data.update(passthrough_data)

1197 -

1199 return processed_data

1200

1201 def _get_mm_fields_config(

vllm/model_executor/models/molmo2.py

+23 -18

@@ -1932,6 +1932,9 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]):

1932

1933

1934 class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):

1935 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

1936 + return self.dummy_inputs.get_dummy_text(mm_counts)

1937 +

1938 def _postprocess_prompt(self, prompt: list[int]) -> list[int]:

1939 processor = self.info.get_hf_processor()

1940 tokenizer = processor.tokenizer

@@ -1946,19 +1949,20 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):

1949 def _apply_hf_processor_main(

1950 self,

1951 mm_items: MultiModalDataItems,

1949 - hf_processor_mm_kwargs: Mapping[str, object],

1952 + hf_kwargs: Mapping[str, object],

1953 ) -> BatchFeature:

1951 - mm_counts = mm_items.get_all_counts()

1952 -

1953 - valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})

1954 - processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

1954 + hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(

1955 + mm_items, hf_kwargs

1956 + )

1957

1956 - prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)

1958 + if not hf_data:

1959 + return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)

1960

1958 - mm_data = dict(processor_data)

1961 + prompt_text = hf_data.pop("text")

1962 + assert isinstance(prompt_text, str)

1963

1964 hf_config = self.info.get_hf_config()

1961 - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)

1965 + hf_processor = self.info.get_hf_processor(**hf_kwargs)

1966

1967 def patched_call(text=None, images=None, videos=None, **kwargs) -> BatchFeature:

1968 res = hf_processor(text=text, images=images, videos=videos, **kwargs)

@@ -1973,7 +1977,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):

1977 tokenizer = hf_processor.tokenizer

1978 image_processor = hf_processor.image_processor

1979

1976 - if videos := mm_data.pop("videos", []):

1980 + if videos := hf_data.pop("videos", []):

1981 assert isinstance(videos, Sequence)

1982 bos_token_id = tokenizer.bos_token_id or tokenizer.eos_token_id

1983

@@ -1993,16 +1997,16 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):

1997 # NOTE: metadata.frames_indices indicates

1998 # the sampled frames indices of pre-sampled videos, which is

1999 # used to calculate the timestamps. Make sure that

1996 - # do_sample_frames in hf_processor_mm_kwargs is false for

2000 + # do_sample_frames in hf_kwargs is false for

2001 # presampled videos.

2002

1999 - # NOTE: a copy of hf_processor_mm_kwargs is created to update

2003 + # NOTE: a copy of hf_kwargs is created to update

2004 # do_sample_frames, otherwise mm_hash for the object will be

2005 # incorrect.

2002 - video_mm_kwargs = dict(**hf_processor_mm_kwargs)

2006 + video_mm_kwargs = dict(**hf_kwargs)

2007 if "do_sample_frames" not in video_mm_kwargs:

2008 # molmo_utils already has "do_sample_frames" in

2005 - # hf_processor_mm_kwargs, don't overwrite it.

2009 + # hf_kwargs, don't overwrite it.

2010 video_mm_kwargs["do_sample_frames"] = metadata.get(

2011 "do_sample_frames", False

2012 )

@@ -2067,11 +2071,11 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):

2071

2072 processed_data = self.info.ctx.call_hf_processor(

2073 patched_call,

2070 - dict(text=prompt_text, **mm_data),

2071 - hf_processor_mm_kwargs,

2074 + dict(text=prompt_text, **hf_data),

2075 + hf_kwargs,

2076 )

2077

2074 - if (images := mm_data.get("images")) is not None:

2078 + if (images := hf_data.get("images")) is not None:

2079 mm_items = self.info.parse_mm_data({"image": images}, validate=False)

2080 parsed_images = mm_items.get_items("image", ImageProcessorItems)

2081 image_sizes = [

@@ -2113,10 +2117,11 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):

2117 )

2118

2119 processed_data.update(all_video_outputs)

2116 - processed_data.update(passthrough_data)

2120 processed_data.pop("input_ids")

2121

2119 - return processed_data

2122 + return self._finalize_hf_mm_data(

2123 + hf_data, hf_kwargs, passthrough_data, processed_data

2124 + )

2125

2126 def _get_mm_fields_config(

2127 self,

vllm/model_executor/models/moss_audio.py

+6 -21

@@ -1356,32 +1356,17 @@ class MossAudioDummyInputsBuilder(BaseDummyInputsBuilder[MossAudioProcessingInfo

1356

1357

1358 class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingInfo]):

1359 - def _apply_hf_processor_main(

1359 + def _call_hf_processor(

1360 self,

1361 - mm_items: MultiModalDataItems,

1362 - hf_processor_mm_kwargs: Mapping[str, object],

1361 + hf_data: Mapping[str, object],

1362 + hf_kwargs: Mapping[str, object],

1363 ) -> BatchFeature:

1364 - valid_mm_items = mm_items.select(

1365 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

1366 - )

1367 - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

1368 -

1369 - if not mm_data:

1370 - return BatchFeature(dict(passthrough_data))

1371 -

1372 - mm_data = dict(mm_data)

1373 - audios = mm_data.pop("audios", [])

1374 - if audios:

1375 - mm_data["audio"] = audios

1376 - hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs)

1377 - processor_kwargs = _filter_moss_audio_processor_config(hf_processor_mm_kwargs)

1378 - processed_data = self.info.ctx.call_hf_processor(

1364 + processor_kwargs = _filter_moss_audio_processor_config(dict(hf_kwargs))

1365 + return self.info.ctx.call_hf_processor(

1366 self.info.get_hf_processor(**processor_kwargs),

1380 - mm_data,

1367 + hf_data,

1368 {},

1369 )

1383 - processed_data.update(passthrough_data)

1384 - return processed_data

1370

1371 def _get_mm_fields_config(

1372 self,

vllm/model_executor/models/moss_transcribe_diarize.py

+10 -8

@@ -66,7 +66,7 @@ from vllm.multimodal.processing import (

66 PromptUpdateDetails,

67 cached_encode,

68 )

69 -from vllm.multimodal.processing.processor import ProcessorInputs

69 +from vllm.multimodal.processing.processor import HFMultiModalInputs, ProcessorInputs

70 from vllm.sequence import IntermediateTensors

71 from vllm.transformers_utils.processor import cached_processor_from_config

72 from vllm.utils.tensor_schema import TensorSchema, TensorShape

@@ -465,17 +465,19 @@ class MossTranscribeDiarizeDummyInputsBuilder(

465 class MossTranscribeDiarizeMultiModalProcessor(

466 BaseMultiModalProcessor[MossTranscribeDiarizeProcessingInfo]

467 ):

468 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

468 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

469 return self.dummy_inputs.get_dummy_text(mm_counts)

470

471 - def _preprocess_hf_mm_data(

471 + def _get_hf_mm_inputs(

472 self,

473 - mm_data: Mapping[str, object],

474 - hf_processor_mm_kwargs: Mapping[str, object],

475 - ) -> tuple[Mapping[str, object], Mapping[str, object]]:

476 - audios = _get_audios_from_mm_data(mm_data)

473 + mm_items: MultiModalDataItems,

474 + hf_kwargs: Mapping[str, object],

475 + ) -> HFMultiModalInputs:

476 + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)

477 + if hf_inputs.hf_data:

478 + hf_inputs.hf_data["audio"] = _get_audios_from_mm_data(hf_inputs.hf_data)

479

478 - return dict(audio=audios), hf_processor_mm_kwargs

480 + return hf_inputs

481

482 def _postprocess_hf_mm_data(

483 self,

vllm/model_executor/models/muse_glimmer.py

+10 -7

@@ -275,16 +275,18 @@ class MuseGlimmerMultiModalProcessor(

275 def _apply_hf_processor_main(

276 self,

277 mm_items: MultiModalDataItems,

278 - hf_processor_mm_kwargs: Mapping[str, object],

278 + hf_kwargs: Mapping[str, object],

279 ) -> BatchFeature:

280 - valid_mm_items = mm_items.select(

281 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

280 + mm_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(

281 + mm_items, hf_kwargs

282 )

283 - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

283 +

284 + if not mm_data:

285 + return self._finalize_hf_mm_data(mm_data, hf_kwargs, passthrough_data)

286

287 prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())

288

287 - processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)

289 + processor = self.info.get_hf_processor(**hf_kwargs)

290 tokenizer = processor.tokenizer

291 config = self.info.get_hf_config()

292 images = mm_data.get("images", ())

@@ -371,8 +373,9 @@ class MuseGlimmerMultiModalProcessor(

373 video_feature_sizes=torch.tensor(video_sizes),

374 )

375 processed_data = BatchFeature(data=data, tensor_type=None)

374 - processed_data.update(passthrough_data)

375 - return processed_data

376 + return self._finalize_hf_mm_data(

377 + mm_data, hf_kwargs, passthrough_data, processed_data

378 + )

379

380 def _get_mm_fields_config(

381 self,

vllm/model_executor/models/nano_nemotron_vl.py

+23 -8

@@ -70,6 +70,7 @@ from vllm.multimodal.processing import (

70 from vllm.multimodal.processing.processor import (

71 BaseMultiModalProcessor,

72 BaseProcessingInfo,

73 + HFMultiModalInputs,

74 PromptReplacement,

75 PromptUpdate,

76 cached_encode,

@@ -196,6 +197,9 @@ NanoNemotronVLVideoInputs: TypeAlias = (

197

198 class NanoNemotronVLProcessingInfo(BaseProcessingInfo):

199 def get_hf_processor(self, **kwargs: object) -> NanoNemotronVLProcessor:

200 + # Not accepted by NanoNemotronVLProcessor.__init__

201 + kwargs.pop("truncation", None)

202 +

203 return self.ctx.init_processor(

204 NanoNemotronVLProcessor,

205 config=self.get_hf_config(),

@@ -358,9 +362,23 @@ class NanoNemotronVLProcessingInfo(BaseProcessingInfo):

362 class NanoNemotronVLMultiModalProcessor(

363 BaseMultiModalProcessor[NanoNemotronVLProcessingInfo]

364 ):

361 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

365 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

366 return self.dummy_inputs.get_dummy_text(mm_counts)

367

368 + def _get_hf_mm_inputs(

369 + self,

370 + mm_items: MultiModalDataItems,

371 + hf_kwargs: Mapping[str, object],

372 + ) -> HFMultiModalInputs:

373 + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)

374 +

375 + # NanoNemotronVLProcessor expects "audios" instead of "audio"

376 + hf_data = hf_inputs.hf_data

377 + if "audio" in hf_data:

378 + hf_data["audios"] = hf_data.pop("audio")

379 +

380 + return hf_inputs

381 +

382 def _get_image_fields_config(self, hf_inputs: BatchFeature):

383 if self.info.is_dynamic_tiler:

384 pixel_values_flat = MultiModalFieldConfig.batched("image")

@@ -730,15 +748,12 @@ class NanoNemotronVLMultiModalProcessor(

748 # Bypass the cached path: the HF processor must receive the

749 # prompt (with injected <so_embedding>) and the audio data

750 # together so it can perform audio-token replacement natively.

733 - mm_info = self._apply_hf_processor(inputs, timing_ctx)

734 - prompt_ids = self._postprocess_prompt(inputs.prompt)

751 + mm_res = self._apply_hf_processor(inputs, timing_ctx)

752

753 with timing_ctx.record("apply_prompt_updates"):

754 prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(

755 mm_items=mm_items,

739 - prompt_ids=prompt_ids,

740 - mm_kwargs=mm_info.kwargs,

741 - mm_prompt_updates=mm_info.prompt_updates,

756 + mm_res=mm_res,

757 )

758

759 mm_placeholder_ranges = {

@@ -749,8 +764,8 @@ class NanoNemotronVLMultiModalProcessor(

764 return MultiModalInput(

765 type="multimodal",

766 prompt_token_ids=prompt_ids,

752 - mm_kwargs=mm_info.kwargs,

753 - mm_hashes=mm_info.hashes,

767 + mm_kwargs=mm_res.kwargs,

768 + mm_hashes=mm_res.hashes,

769 mm_placeholders=mm_placeholder_ranges,

770 )

771

vllm/model_executor/models/ovis.py

+3 -3

@@ -326,6 +326,9 @@ class OvisDummyInputsBuilder(BaseDummyInputsBuilder[OvisProcessingInfo]):

326

327

328 class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):

329 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

330 + return self.dummy_inputs.get_dummy_text(mm_counts)

331 +

332 def image_indicators_to_visual_tokens(

333 self,

334 image_indicators: list[int],

@@ -340,9 +343,6 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):

343 # -300 is image_atom token, filter them out

344 return [vte_vocab_size + x + 300 for x in image_indicators if x < -300]

345

343 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

344 - return self.dummy_inputs.get_dummy_text(mm_counts)

345 -

346 def _postprocess_hf_mm_data(

347 self,

348 mm_data: Mapping[str, object],

vllm/model_executor/models/ovis2_5.py

+1 -1

@@ -327,7 +327,7 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo])

327 if x >= INDICATOR_IDS[0]

328 ]

329

330 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

330 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

331 return self.dummy_inputs.get_dummy_text(mm_counts)

332

333 def _postprocess_hf_mm_data(

vllm/model_executor/models/paddleocr_vl.py

+20 -19

@@ -243,36 +243,37 @@ class PaddleOCRVLDummyInputsBuilder(BaseDummyInputsBuilder[PaddleOCRVLProcessing

243 class PaddleOCRVLMultiModalProcessor(

244 BaseMultiModalProcessor[PaddleOCRVLProcessingInfo]

245 ):

246 - def _apply_hf_processor_main(

246 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

247 + return self.dummy_inputs.get_dummy_text(mm_counts)

248 +

249 + def _call_hf_processor(

250 self,

248 - mm_items: MultiModalDataItems,

249 - hf_processor_mm_kwargs: Mapping[str, object],

251 + hf_data: Mapping[str, object],

252 + hf_kwargs: Mapping[str, object],

253 ) -> BatchFeature:

251 - valid_mm_items = mm_items.select(

252 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

253 - )

254 - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

255 -

256 - if not mm_data:

257 - return BatchFeature(dict(passthrough_data))

258 -

259 - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())

260 -

261 - final_mm_kwargs = dict(hf_processor_mm_kwargs or {})

254 + final_mm_kwargs = dict(hf_kwargs or {})

255 final_mm_kwargs.setdefault("images_kwargs", {})

256 # vLLM use PIL.Image, always set channel_last

257 final_mm_kwargs["input_data_format"] = ChannelDimension.LAST

265 - processed_data = self.info.ctx.call_hf_processor(

258 + return self.info.ctx.call_hf_processor(

259 self.info.get_hf_processor(**final_mm_kwargs),

267 - dict(text=prompt_text, **mm_data),

268 - hf_processor_mm_kwargs,

260 + hf_data,

261 + hf_kwargs,

262 )

263 +

264 + def _postprocess_hf_mm_data(

265 + self,

266 + hf_data: Mapping[str, object],

267 + hf_kwargs: Mapping[str, object],

268 + processed_data: BatchFeature,

269 + ) -> BatchFeature:

270 + if not hf_data:

271 + return processed_data

272 +

273 num_patches_per_image = processed_data["image_grid_thw"].prod(-1)

274 processed_data["pixel_values"] = processed_data["pixel_values"].split(

275 num_patches_per_image.tolist()

276 )

274 - processed_data.update(passthrough_data)

275 -

277 return processed_data

278

279 def _get_mm_fields_config(

vllm/model_executor/models/phi3v.py

+1 -20

@@ -390,28 +390,9 @@ class Phi3VDummyInputsBuilder(BaseDummyInputsBuilder[Phi3VProcessingInfo]):

390

391

392 class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):

393 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

393 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

394 return self.dummy_inputs.get_dummy_text(mm_counts)

395

396 - def _postprocess_hf_mm_data(

397 - self,

398 - mm_data: Mapping[str, object],

399 - hf_processor_mm_kwargs: Mapping[str, object],

400 - processed_data: BatchFeature,

401 - ) -> BatchFeature:

402 - if not mm_data:

403 - return processed_data

404 -

405 - input_ids = processed_data["input_ids"]

406 - assert isinstance(input_ids, torch.Tensor)

407 -

408 - # Phi3v processor has inserted -1, -2 etc as placeholder in prompt_ids,

409 - # which will cause OverflowError when decoding the prompt_ids.

410 - # Therefore, we need to do an early replacement here

411 - input_ids.masked_fill_(input_ids < 0, _IMAGE_TOKEN_ID)

412 -

413 - return processed_data

414 -

396 def _get_mm_fields_config(

397 self,

398 hf_inputs: BatchFeature,

vllm/model_executor/models/phi4mm.py

+35 -31

@@ -45,6 +45,7 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder

45 from vllm.multimodal.processing.processor import (

46 BaseMultiModalProcessor,

47 BaseProcessingInfo,

48 + HFMultiModalInputs,

49 PromptReplacement,

50 PromptUpdate,

51 ResolvedPromptUpdate,

@@ -867,40 +868,38 @@ class Phi4MMDummyInputsBuilder(BaseDummyInputsBuilder[Phi4MMProcessingInfo]):

868

869

870 class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):

870 - def _apply_hf_processor_main(

871 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

872 + return self.dummy_inputs.get_dummy_text(mm_counts)

873 +

874 + def _get_hf_mm_inputs(

875 self,

876 mm_items: MultiModalDataItems,

873 - hf_processor_mm_kwargs: Mapping[str, object],

874 - ) -> BatchFeature:

875 - valid_mm_items = mm_items.select(

876 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

877 - )

878 - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

877 + hf_kwargs: Mapping[str, object],

878 + ) -> HFMultiModalInputs:

879 + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)

880

880 - if not mm_data:

881 - return BatchFeature(dict(passthrough_data))

881 + # The Phi4MM processor expects "audios" instead of "audio"

882 + hf_data = hf_inputs.hf_data

883 + if "audio" in hf_data:

884 + hf_data["audios"] = hf_data.pop("audio")

885

883 - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())

886 + sr = self.info.get_feature_extractor(**hf_inputs.hf_kwargs).sampling_rate

887 + if audio_data := hf_data.get("audios", []):

888 + assert isinstance(audio_data, Sequence)

889 + hf_data["audios"] = [(data, sr) for data in audio_data]

890

885 - sr = self.info.get_feature_extractor(**hf_processor_mm_kwargs).sampling_rate

886 - raw_audio_data = mm_data.get("audios", [])

887 - assert isinstance(raw_audio_data, Sequence)

888 - audio_data = []

889 - for data in raw_audio_data:

890 - assert isinstance(data, (list, np.ndarray, torch.Tensor))

891 - audio_data.append(data)

891 + return hf_inputs

892

893 - processor_data = dict(text=prompt_text, **mm_data)

894 - if audio_data:

895 - processor_data["audios"] = [(data, sr) for data in audio_data]

896 -

897 - processed_data = self.info.ctx.call_hf_processor(

898 - self.info.get_hf_processor(**hf_processor_mm_kwargs),

899 - processor_data,

900 - hf_processor_mm_kwargs,

901 - )

893 + def _postprocess_hf_mm_data(

894 + self,

895 + hf_data: Mapping[str, object],

896 + hf_kwargs: Mapping[str, object],

897 + processed_data: BatchFeature,

898 + ) -> BatchFeature:

899 + if not hf_data:

900 + return processed_data

901

903 - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)

902 + hf_processor = self.info.get_hf_processor(**hf_kwargs)

903 num_img_tokens = [

904 self.info.get_num_image_tokens(

905 image_width=img_size[0],

@@ -911,21 +910,26 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):

910 ]

911 processed_data["num_img_tokens"] = num_img_tokens

912

914 - audio_features = processed_data["input_audio_embeds"]

913 + audios = hf_data.get("audios", [])

914 + assert isinstance(audios, list)

915 + audio_features = processed_data.get("input_audio_embeds")

916 + if audio_features is None:

917 + return processed_data

918 +

919 + sr = self.info.get_feature_extractor(**hf_kwargs).sampling_rate

920 feature_sizes = [

916 - self.info.get_audio_num_frames(len(audio), sr) for audio in audio_data

921 + self.info.get_audio_num_frames(len(audio), sr) for audio, _ in audios

922 ]

923 processed_data["input_audio_embeds"] = [

924 audio_features[idx, :size] for idx, size in enumerate(feature_sizes)

925 ]

921 - processed_data.update(passthrough_data)

926

927 return processed_data

928

929 def _get_mm_fields_config(

930 self,

931 hf_inputs: BatchFeature,

928 - hf_processor_mm_kwargs: Mapping[str, object],

932 + hf_kwargs: Mapping[str, object],

933 ) -> Mapping[str, MultiModalFieldConfig]:

934 return dict(

935 input_image_embeds=MultiModalFieldConfig.batched("image"),

vllm/model_executor/models/phi4siglip.py

+2 -33

@@ -143,39 +143,8 @@ class Phi4SiglipDummyInputsBuilder(

143 class Phi4SiglipMultiModalProcessor(

144 BaseMultiModalProcessor[Phi4SiglipProcessingInfo],

145 ):

146 - def _apply_hf_processor_main(

147 - self,

148 - mm_items: MultiModalDataItems,

149 - hf_processor_mm_kwargs: Mapping[str, object],

150 - ) -> BatchFeature:

151 - valid_mm_items = mm_items.select(

152 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

153 - )

154 - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

155 -

156 - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())

157 -

158 - processed = self.info.ctx.call_hf_processor(

159 - self.info.get_hf_processor(**hf_processor_mm_kwargs),

160 - dict(text=prompt_text, **mm_data),

161 - hf_processor_mm_kwargs,

162 - )

163 -

164 - # The HF processor's tokenizer_image_token() replaces the "<image>"

165 - # string with IMAGE_TOKEN_INDEX (-200) in input_ids. This breaks

166 - # vLLM's prompt-replacement pipeline which needs to find "<image>"

167 - # as normal sub-tokens. Re-tokenize with the plain tokenizer so

168 - # that "<image>" stays as sub-tokens and can be located by

169 - # PromptReplacement.

170 - # NOTE: tokenizer.__call__() (not .encode()) must be used so that

171 - # added/special tokens like <|user|>, <|end|> are kept as single IDs.

172 - tokenizer = self.info.get_tokenizer()

173 - new_ids = tokenizer(prompt_text).input_ids

174 - processed["input_ids"] = torch.tensor([new_ids])

175 -

176 - processed_data = processed

177 - processed_data.update(passthrough_data)

178 - return processed_data

146 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

147 + return self.dummy_inputs.get_dummy_text(mm_counts)

148

149 def _get_mm_fields_config(

150 self,

vllm/model_executor/models/pixtral.py

+16 -37

@@ -14,7 +14,7 @@ from mistral_common.protocol.instruct.chunk import ImageChunk, TextChunk

14 from mistral_common.protocol.instruct.messages import UserMessage

15 from mistral_common.protocol.instruct.request import ChatCompletionRequest

16 from packaging.version import Version

17 -from transformers import BatchFeature, PixtralVisionConfig

17 +from transformers import PixtralVisionConfig

18 from transformers.models.pixtral.image_processing_pixtral import (

19 _num_image_tokens as _get_pixtral_hf_num_image_tokens,

20 )

@@ -41,7 +41,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems

41 from vllm.multimodal.inputs import (

42 MultiModalFieldConfig,

43 MultiModalKwargsItem,

44 - MultiModalKwargsOptionalItems,

44 NestedTensors,

45 )

46 from vllm.multimodal.parse import (

@@ -53,7 +52,8 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder

52 from vllm.multimodal.processing.processor import (

53 BaseMultiModalProcessor,

54 BaseProcessingInfo,

56 - MultiModalPromptUpdates,

55 + HFMultiModalInputs,

56 + MultiModalProcessingResult,

57 PlaceholderFeaturesInfo,

58 ProcessorInputs,

59 PromptReplacement,

@@ -67,7 +67,6 @@ from vllm.transformers_utils.processors.pixtral import (

67 MistralCommonImageProcessor,

68 MistralCommonPixtralProcessor,

69 )

70 -from vllm.utils.collection_utils import is_list_of

70 from vllm.utils.tensor_schema import TensorSchema, TensorShape

71 from vllm.v1.attention.backends.registry import AttentionBackendEnum

72

@@ -268,21 +267,19 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])

267 def _maybe_apply_prompt_updates(

268 self,

269 mm_items: MultiModalDataItems,

271 - prompt_ids: list[int],

272 - mm_kwargs: MultiModalKwargsOptionalItems,

273 - mm_prompt_updates: MultiModalPromptUpdates,

270 + mm_res: MultiModalProcessingResult,

271 ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:

272 mm_item_counts = mm_items.get_all_counts()

276 - self._validate_mm_kwargs(mm_kwargs, mm_item_counts)

277 - self._validate_mm_updates(mm_prompt_updates, mm_item_counts)

273 + self._validate_mm_kwargs(mm_res.kwargs, mm_item_counts)

274 + self._validate_mm_updates(mm_res.prompt_updates, mm_item_counts)

275

276 mm_placeholders = self._find_mm_placeholders(

280 - prompt_ids,

281 - mm_prompt_updates,

277 + mm_res.prompt_ids,

278 + mm_res.prompt_updates,

279 )

280 self._validate_mm_placeholders(mm_placeholders, mm_item_counts)

281

285 - return prompt_ids, mm_placeholders

282 + return mm_res.prompt_ids, mm_placeholders

283

284 def _get_mm_fields_config(

285 self,

@@ -291,36 +288,18 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])

288 ) -> Mapping[str, MultiModalFieldConfig]:

289 return dict(images=MultiModalFieldConfig.batched("image"))

290

294 - def _apply_hf_processor_main(

291 + def _get_hf_mm_inputs(

292 self,

293 mm_items: MultiModalDataItems,

297 - hf_processor_mm_kwargs: Mapping[str, object],

298 - ) -> BatchFeature:

299 - valid_mm_items = mm_items.select(

300 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

301 - )

302 - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

303 -

304 - if not mm_data:

305 - return BatchFeature(dict(passthrough_data))

294 + hf_kwargs: Mapping[str, object],

295 + ) -> HFMultiModalInputs:

296 + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)

297

307 - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())

308 -

309 - outputs = self.info.ctx.call_hf_processor(

310 - self.info.get_hf_processor(**hf_processor_mm_kwargs),

311 - dict(text=prompt_text, **mm_data),

312 - # Avoid padding issue

313 - dict(**hf_processor_mm_kwargs, return_tensors=None),

298 + # Avoid padding issue

299 + return hf_inputs._replace(

300 + hf_kwargs=dict(hf_inputs.hf_kwargs, return_tensors=None)

301 )

302

316 - # Missing batch dimension

317 - if is_list_of(outputs["input_ids"], int):

318 - outputs["input_ids"] = [outputs["input_ids"]]

319 -

320 - processed_data = outputs

321 - processed_data.update(passthrough_data)

322 - return processed_data

323 -

303 def _get_prompt_updates(

304 self,

305 mm_items: MultiModalDataItems,

vllm/model_executor/models/qwen2_5_omni_thinker.py

+80 -86

@@ -82,6 +82,8 @@ from vllm.multimodal.processing import (

82 )

83 from vllm.multimodal.processing.processor import (

84 BaseMultiModalProcessor,

85 + HFMultiModalInputs,

86 + MultiModalProcessingResult,

87 MultiModalPromptUpdates,

88 PlaceholderFeaturesInfo,

89 PromptReplacement,

@@ -498,6 +500,64 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(

500 class Qwen2_5OmniThinkerMultiModalProcessor(

501 BaseMultiModalProcessor[Qwen2_5OmniThinkerProcessingInfo]

502 ):

503 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

504 + return self.dummy_inputs.get_dummy_text(mm_counts)

505 +

506 + def _get_hf_mm_inputs(

507 + self,

508 + mm_items: MultiModalDataItems,

509 + hf_kwargs: Mapping[str, object],

510 + ) -> HFMultiModalInputs:

511 + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)

512 +

513 + mm_counts = mm_items.get_all_counts()

514 + if hf_kwargs.get("use_audio_in_video"):

515 + num_videos = mm_counts.get("video", 0)

516 + if mm_counts.get("audio", 0) < num_videos:

517 + raise ValueError(

518 + "Video doesn't have audio track with `audio_in_video=True`"

519 + )

520 +

521 + if num_videos and "text" in hf_inputs.hf_data:

522 + standalone_mm_counts = dict(mm_counts)

523 + standalone_mm_counts["audio"] -= num_videos

524 + hf_inputs.hf_data["text"] = self._get_hf_mm_text(standalone_mm_counts)

525 +

526 + hf_data = hf_inputs.hf_data

527 + normalized_kwargs = hf_inputs.hf_kwargs

528 + audios = hf_data.pop("audio", [])

529 +

530 + # NOTE: WhisperFeatureExtractor cannot handle empty list of audios

531 + if audios:

532 + hf_data["audio"] = audios

533 + elif (

534 + normalized_kwargs.get("use_audio_in_video")

535 + and "audio" not in hf_data

536 + and hf_data.get("videos")

537 + ):

538 + raise ValueError(

539 + "Video doesn't have audio track with `audio_in_video=True`"

540 + )

541 +

542 + merged = self.info.ctx.get_merged_mm_kwargs(normalized_kwargs)

543 + if hf_data.get("videos") and (

544 + merged.keys() & {"size", "min_pixels", "max_pixels"}

545 + ):

546 + normalized_kwargs = dict(normalized_kwargs)

547 + video_size = dict(self.info.get_hf_processor().video_processor.size)

548 + size_override = merged.get("size")

549 + if size_override is not None:

550 + video_size = video_size | size_override

551 + min_pixels = merged.get("min_pixels")

552 + if min_pixels is not None:

553 + video_size["shortest_edge"] = min_pixels

554 + max_pixels = merged.get("max_pixels")

555 + if max_pixels is not None:

556 + video_size["longest_edge"] = max_pixels

557 + normalized_kwargs["size"] = video_size

558 +

559 + return hf_inputs._replace(hf_kwargs=normalized_kwargs)

560 +

561 def _get_mm_fields_config(

562 self,

563 hf_inputs: BatchFeature,

@@ -570,12 +630,14 @@ class Qwen2_5OmniThinkerMultiModalProcessor(

630 def _maybe_apply_prompt_updates(

631 self,

632 mm_items: MultiModalDataItems,

573 - prompt_ids: list[int],

574 - mm_kwargs: MultiModalKwargsItems,

575 - mm_prompt_updates: MultiModalPromptUpdates,

633 + mm_res: MultiModalProcessingResult,

634 ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:

635 """Qwen2.5-Omni reimplements this function to handle `use_audio_in_video`."""

636 mm_item_counts = mm_items.get_all_counts()

637 + prompt_ids = mm_res.prompt_ids

638 + mm_kwargs = mm_res.kwargs

639 + mm_prompt_updates = mm_res.prompt_updates

640 +

641 self._validate_mm_kwargs(mm_kwargs, mm_item_counts)

642 self._validate_mm_updates(mm_prompt_updates, mm_item_counts)

643

@@ -789,101 +851,33 @@ class Qwen2_5OmniThinkerMultiModalProcessor(

851 ),

852 ]

853

792 - def _apply_hf_processor_main(

854 + def _postprocess_hf_mm_data(

855 self,

794 - mm_items: MultiModalDataItems,

795 - hf_processor_mm_kwargs: Mapping[str, object],

856 + hf_data: Mapping[str, object],

857 + hf_kwargs: Mapping[str, object],

858 + processed_data: BatchFeature,

859 ) -> BatchFeature:

797 - """Qwen2.5-Omni reimplements this function to handle `use_audio_in_video`."""

798 - mm_counts = mm_items.get_all_counts()

799 -

800 - use_audio_in_video = hf_processor_mm_kwargs.get("use_audio_in_video", False)

801 - if use_audio_in_video and "video" in mm_counts:

802 - if mm_counts.get("audio", 0) < mm_counts["video"]:

803 - raise ValueError(

804 - "Video doesn't have audio track with `audio_in_video=True`"

805 - )

806 - mm_counts["audio"] -= mm_counts["video"]

807 -

808 - valid_mm_items = mm_items.select(

809 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

810 - )

811 - processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

812 -

813 - prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)

814 -

815 - mm_data = dict(processor_data)

816 - audios = mm_data.pop("audios", [])

817 -

818 - # NOTE: WhisperFeatureExtractor cannot handle empty list of audios

819 - if audios:

820 - # NOTE: Qwen2.5-Omni processor accept "audio"

821 - mm_data["audio"] = audios

822 - hf_processor_mm_kwargs = dict(

823 - **hf_processor_mm_kwargs,

824 - )

825 - elif (

826 - hf_processor_mm_kwargs.get("use_audio_in_video")

827 - and "audio" not in mm_data

828 - and mm_data.get("videos")

829 - ):

830 - # A subclass (e.g. Qwen3-Omni) may have already popped "audios"

831 - # and populated mm_data["audio"] itself before delegating here,

832 - # in which case this isn't a genuine "no audio" case. Likewise,

833 - # mm_data can be empty on a multimodal-processor-cache hit, where

834 - # there's nothing to (re-)process this call and the real result

835 - # comes from the cache — not a genuine "no audio" case either.

836 - raise ValueError(

837 - "Video doesn't have audio track with `audio_in_video=True`"

838 - )

839 -

840 - merged = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)

841 - if mm_data.get("videos") and (

842 - merged.keys() & {"size", "min_pixels", "max_pixels"}

843 - ):

844 - hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs)

845 - video_size = dict(self.info.get_hf_processor().video_processor.size)

846 - size_override = merged.get("size")

847 - if size_override is not None:

848 - video_size = video_size | size_override

849 - min_pixels = merged.get("min_pixels")

850 - if min_pixels is not None:

851 - video_size["shortest_edge"] = min_pixels

852 - max_pixels = merged.get("max_pixels")

853 - if max_pixels is not None:

854 - video_size["longest_edge"] = max_pixels

855 - hf_processor_mm_kwargs["size"] = video_size

856 -

857 - hf_inputs = self.info.ctx.call_hf_processor(

858 - self.info.get_hf_processor(**hf_processor_mm_kwargs),

859 - dict(text=prompt_text, **mm_data),

860 - hf_processor_mm_kwargs,

861 - )

862 -

863 - input_features = hf_inputs.pop("input_features", None)

864 - feature_attention_mask = hf_inputs.get("feature_attention_mask", None)

865 - if "input_audio_features" not in hf_inputs and input_features is not None:

860 + input_features = processed_data.pop("input_features", None)

861 + feature_attention_mask = processed_data.get("feature_attention_mask", None)

862 + if "input_audio_features" not in processed_data and input_features is not None:

863 if feature_attention_mask is not None:

864 input_features = input_features.permute(0, 2, 1)[

865 feature_attention_mask.bool()

866 ].permute(1, 0)

870 - hf_inputs["input_audio_features"] = input_features

867 + processed_data["input_audio_features"] = input_features

868 if (

872 - "audio_feature_lengths" not in hf_inputs

869 + "audio_feature_lengths" not in processed_data

870 and feature_attention_mask is not None

871 ):

875 - hf_inputs["audio_feature_lengths"] = feature_attention_mask.sum(-1)

872 + processed_data["audio_feature_lengths"] = feature_attention_mask.sum(-1)

873

877 - video_second_per_grid = hf_inputs.get("video_second_per_grid", None)

874 + video_second_per_grid = processed_data.get("video_second_per_grid", None)

875 if video_second_per_grid is not None:

879 - hf_inputs["second_per_grid_ts"] = video_second_per_grid

880 -

881 - use_audio_in_video = hf_processor_mm_kwargs.get("use_audio_in_video", False)

882 - hf_inputs["use_audio_in_video"] = torch.tensor(use_audio_in_video)

876 + processed_data["second_per_grid_ts"] = video_second_per_grid

877

884 - processed_data = hf_inputs

885 - processed_data.update(passthrough_data)

886 - processed_data.pop("input_ids")

878 + use_audio_in_video = hf_kwargs.get("use_audio_in_video", False)

879 + processed_data["use_audio_in_video"] = torch.tensor(use_audio_in_video)

880 + processed_data.pop("input_ids", None)

881

882 return processed_data

883

vllm/model_executor/models/qwen2_audio.py

+15 -18

@@ -59,6 +59,7 @@ from vllm.multimodal.processing import (

59 PromptReplacement,

60 PromptUpdate,

61 )

62 +from vllm.multimodal.processing.processor import HFMultiModalInputs

63 from vllm.sequence import IntermediateTensors

64 from vllm.utils.gpu_sync_debug import gpu_sync_allowed

65 from vllm.utils.tensor_schema import TensorSchema, TensorShape

@@ -233,29 +234,25 @@ class Qwen2AudioDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2AudioProcessingIn

234

235

236 class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessingInfo]):

236 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

237 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

238 return self.dummy_inputs.get_dummy_text(mm_counts)

239

239 - def _preprocess_hf_mm_data(

240 + def _get_hf_mm_inputs(

241 self,

241 - mm_data: Mapping[str, object],

242 - hf_processor_mm_kwargs: Mapping[str, object],

243 - ) -> tuple[Mapping[str, object], Mapping[str, object]]:

244 - feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)

245 -

246 - mm_data = dict(mm_data)

247 - audios = mm_data.pop("audios", [])

248 - if audios:

249 - mm_data["audio"] = audios

250 -

251 - hf_processor_mm_kwargs = dict(

252 - **hf_processor_mm_kwargs,

253 - sampling_rate=feature_extractor.sampling_rate,

254 - truncation=True,

242 + mm_items: MultiModalDataItems,

243 + hf_kwargs: Mapping[str, object],

244 + ) -> HFMultiModalInputs:

245 + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)

246 +

247 + feature_extractor = self.info.get_feature_extractor(**hf_kwargs)

248 + return hf_inputs._replace(

249 + hf_kwargs=dict(

250 + hf_inputs.hf_kwargs,

251 + sampling_rate=feature_extractor.sampling_rate,

252 + truncation=True,

253 + )

254 )

255

257 - return mm_data, hf_processor_mm_kwargs

258 -

256 def _get_mm_fields_config(

257 self,

258 hf_inputs: BatchFeature,

vllm/model_executor/models/qwen3_asr_realtime.py

+5 -6

@@ -37,11 +37,10 @@ from vllm.model_executor.models.qwen3_asr import (

37 )

38 from vllm.multimodal import MULTIMODAL_REGISTRY

39 from vllm.multimodal.cache import _I, BaseMultiModalProcessorCache

40 -from vllm.multimodal.inputs import MultiModalKwargsOptionalItems

40 from vllm.multimodal.parse import MultiModalDataItems

41 from vllm.multimodal.processing.processor import (

42 BaseDummyInputsBuilder,

44 - MultiModalPromptUpdates,

43 + MultiModalProcessingResult,

44 PlaceholderFeaturesInfo,

45 cached_encode,

46 )

@@ -114,10 +113,10 @@ class Qwen3ASRRealtimeMultiModalProcessor(Qwen3ASRMultiModalProcessor):

113 def _maybe_apply_prompt_updates(

114 self,

115 mm_items: MultiModalDataItems,

117 - prompt_ids: list[int],

118 - mm_kwargs: MultiModalKwargsOptionalItems,

119 - mm_prompt_updates: MultiModalPromptUpdates,

116 + mm_res: MultiModalProcessingResult,

117 ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:

118 + mm_kwargs = mm_res.kwargs

119 +

120 audios = mm_kwargs.get("audio", [])

121 assert len(audios) == 1, (

122 f"Expected only one audio input for realtime, got {len(audios)}"

@@ -147,7 +146,7 @@ class Qwen3ASRRealtimeMultiModalProcessor(Qwen3ASRMultiModalProcessor):

146 # Find the audio_pad token position and expand it to audio_len tokens

147 expanded_ids = list[int]()

148 pad_start_idx = -1

150 - for i, tid in enumerate(prompt_ids):

149 + for i, tid in enumerate(mm_res.prompt_ids):

150 if tid == audio_pad_id and pad_start_idx == -1:

151 pad_start_idx = i

152 expanded_ids.extend([audio_pad_id] * audio_len)

vllm/model_executor/models/qwen3_omni_moe_thinker.py

+32 -32

@@ -73,6 +73,8 @@ from vllm.multimodal.inputs import (

73 )

74 from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataItems

75 from vllm.multimodal.processing.processor import (

76 + HFMultiModalInputs,

77 + MultiModalProcessingResult,

78 MultiModalPromptUpdates,

79 PlaceholderFeaturesInfo,

80 PromptReplacement,

@@ -1219,13 +1221,26 @@ Qwen3OmniMoeThinkerDummyInputsBuilder = Qwen2_5OmniThinkerDummyInputsBuilder

1221 class Qwen3OmniMoeThinkerMultiModalProcessor(

1222 Qwen2_5OmniThinkerMultiModalProcessor,

1223 ):

1222 - def _get_hf_mm_data(

1224 + def _get_hf_mm_inputs(

1225 self,

1226 mm_items: MultiModalDataItems,

1225 - ) -> tuple[Mapping[str, object], Mapping[str, object]]:

1226 - processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)

1227 + hf_kwargs: Mapping[str, object],

1228 + ) -> HFMultiModalInputs:

1229 + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)

1230

1228 - audios = processor_data.get("audios")

1231 + if mm_items.get_count("audio", strict=False):

1232 + # TODO(Isotr0py): Remove this patch after upstream fix PR

1233 + # released and Transformers version update:

1234 + # https://github.com/huggingface/transformers/pull/41473

1235 + hf_inputs = hf_inputs._replace(

1236 + hf_kwargs={

1237 + **hf_inputs.hf_kwargs,

1238 + "audio_kwargs": dict(hf_inputs.hf_kwargs.get("audio_kwargs") or {}),

1239 + "text_kwargs": dict(hf_inputs.hf_kwargs.get("text_kwargs") or {}),

1240 + }

1241 + )

1242 +

1243 + audios = hf_inputs.hf_data.get("audio")

1244 if audios:

1245 feature_extractor = self.info.get_feature_extractor()

1246 hop_length = feature_extractor.hop_length

@@ -1239,47 +1254,30 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(

1254

1255 # To make sure the cache works with padding=True, we pre-padded

1256 # the audio to a multiple of hop_length.

1242 - processor_data = dict(processor_data)

1243 - processor_data["audios"] = [

1257 + hf_inputs.hf_data["audio"] = [

1258 pad_to_hop_length(audio)

1259 if isinstance(audio, np.ndarray)

1260 else (pad_to_hop_length(audio[0]), audio[1])

1261 for audio in audios

1262 ]

1263

1250 - return processor_data, passthrough_data

1264 + return hf_inputs

1265

1252 - def _apply_hf_processor_main(

1266 + def _postprocess_hf_mm_data(

1267 self,

1254 - mm_items: MultiModalDataItems,

1268 + mm_data: Mapping[str, object],

1269 hf_processor_mm_kwargs: Mapping[str, object],

1270 + processed_data: BatchFeature,

1271 ) -> BatchFeature:

1257 - if mm_items.get_all_counts().get("audio", 0):

1258 - # TODO(Isotr0py): Remove this patch after upstream fix PR

1259 - # released and Transformers version update:

1260 - # https://github.com/huggingface/transformers/pull/41473

1261 - hf_processor_mm_kwargs = dict(hf_processor_mm_kwargs)

1262 - hf_processor_mm_kwargs["audio_kwargs"] = dict(

1263 - hf_processor_mm_kwargs.get("audio_kwargs") or {}

1264 - )

1265 - hf_processor_mm_kwargs["text_kwargs"] = dict(

1266 - hf_processor_mm_kwargs.get("text_kwargs") or {}

1267 - )

1268 -

1269 - processed_data = super()._apply_hf_processor_main(

1270 - mm_items,

1271 - hf_processor_mm_kwargs,

1272 + processed_data = super()._postprocess_hf_mm_data(

1273 + mm_data, hf_processor_mm_kwargs, processed_data

1274 )

1275

1276 if (

1277 "audio_feature_lengths" in processed_data

1278 and "feature_attention_mask" in processed_data

1279 ):

1278 - valid_mm_items = mm_items.select(

1279 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

1280 - )

1281 - processor_data, _ = self._get_hf_mm_data(valid_mm_items)

1282 - audios = processor_data.get("audios", [])

1280 + audios = mm_data.get("audio", [])

1281 if audios:

1282 feature_extractor = self.info.get_feature_extractor(

1283 **hf_processor_mm_kwargs

@@ -1312,12 +1310,14 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(

1310 def _maybe_apply_prompt_updates(

1311 self,

1312 mm_items: MultiModalDataItems,

1315 - prompt_ids: list[int],

1316 - mm_kwargs: MultiModalKwargsItems,

1317 - mm_prompt_updates: MultiModalPromptUpdates,

1313 + mm_res: MultiModalProcessingResult,

1314 ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:

1315 """Qwen3-Omni reimplements this function to handle `use_audio_in_video`."""

1316 mm_item_counts = mm_items.get_all_counts()

1317 + prompt_ids = mm_res.prompt_ids

1318 + mm_kwargs = mm_res.kwargs

1319 + mm_prompt_updates = mm_res.prompt_updates

1320 +

1321 self._validate_mm_kwargs(mm_kwargs, mm_item_counts)

1322

1323 use_audio_in_video = False

vllm/model_executor/models/qwen3_vl.py

+16 -22

@@ -1327,27 +1327,25 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])

1327 proc_impl = getattr(type(hf_processor), "replace_video_token", None)

1328 return proc_impl is not None and proc_impl is not mixin_impl

1329

1330 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

1331 + return self.dummy_inputs.get_dummy_text(mm_counts)

1332 +

1333 def _apply_hf_processor_main(

1334 self,

1335 mm_items: MultiModalDataItems,

1333 - hf_processor_mm_kwargs: Mapping[str, object],

1336 + hf_kwargs: Mapping[str, object],

1337 ) -> BatchFeature:

1335 - valid_mm_items = mm_items.select(

1336 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

1338 + hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(

1339 + mm_items, hf_kwargs

1340 )

1338 - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

1339 -

1340 - if not mm_data:

1341 - return BatchFeature(dict(passthrough_data))

1342 -

1343 - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())

1341

1345 - mm_data = dict(mm_data)

1342 + if not hf_data:

1343 + return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)

1344

1345 # Separate video processing from image processing. Because the videos

1346 # are processed into several image patches

1347 video_input_ids_lst: list[list[int]] = []

1350 - if videos := mm_data.pop("videos", []):

1348 + if videos := hf_data.pop("videos", []):

1349 video_grid_thw_lst = []

1350 pixel_values_videos_lst = []

1351 timestamps_per_video = []

@@ -1370,10 +1368,8 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])

1368

1369 # NOTE: a copy of is created to update do_sample_frames,

1370 # otherwise mm_hash for the object will be incorrect.

1373 - video_mm_kwargs = dict(**hf_processor_mm_kwargs)

1374 - merged = self.info.ctx.get_merged_mm_kwargs(

1375 - hf_processor_mm_kwargs, modality="video"

1376 - )

1371 + video_mm_kwargs = dict(**hf_kwargs)

1372 + merged = self.info.ctx.get_merged_mm_kwargs(hf_kwargs, modality="video")

1373 if merged.keys() & {"size", "min_pixels", "max_pixels"}:

1374 video_size = dict(self.info.get_video_processor().size)

1375 size_override = merged.get("size")

@@ -1485,13 +1481,11 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])

1481 # fps/num_frames are video-only kwargs already consumed by the loop;

1482 # exclude them so the text/image processor call below never gets a list.

1483 non_video_mm_kwargs = {

1488 - k: v

1489 - for k, v in hf_processor_mm_kwargs.items()

1490 - if k not in ("fps", "num_frames")

1484 + k: v for k, v in hf_kwargs.items() if k not in ("fps", "num_frames")

1485 }

1486 processed_data = self.info.ctx.call_hf_processor(

1487 self.info.get_hf_processor(**non_video_mm_kwargs),

1494 - dict(text=prompt_text, **mm_data),

1488 + hf_data,

1489 non_video_mm_kwargs,

1490 )

1491

@@ -1516,9 +1510,9 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])

1510 processed_data["input_ids"] = [expanded_ids]

1511

1512 processed_data.update(video_outputs)

1519 - processed_data.update(passthrough_data)

1520 -

1521 - return processed_data

1513 + return self._finalize_hf_mm_data(

1514 + hf_data, hf_kwargs, passthrough_data, processed_data

1515 + )

1516

1517 def _get_mm_fields_config(

1518 self,

vllm/model_executor/models/terratorch.py

+3 -1

@@ -202,7 +202,9 @@ class TerratorchMultiModalProcessor(BaseMultiModalProcessor[TerratorchProcessing

202 hf_processor_mm_kwargs = inputs.hf_processor_mm_kwargs

203

204 with timing_ctx.record("apply_hf_processor"):

205 - _, passthrough_data = self._get_hf_mm_data(mm_items)

205 + passthrough_data = self._get_hf_mm_inputs(

206 + mm_items, hf_processor_mm_kwargs

207 + ).passthrough_data

208 mm_processed_data = BatchFeature(

209 {

210 k: torch.as_tensor(v).unsqueeze(0)

vllm/model_executor/models/transformers/multimodal.py

+44 -35

@@ -219,18 +219,6 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]

219 Subclasses add the strategy for locating placeholders in the prompt.

220 """

221

222 - def _get_hf_mm_data(

223 - self,

224 - mm_items: MultiModalDataItems,

225 - ) -> tuple[Mapping[str, object], Mapping[str, object]]:

226 - """Rename the parser's `audios` key to the `audio` argument HF audio

227 - processors take.

228 - """

229 - processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)

230 - if self.info._is_audio_model() and "audios" in processor_data:

231 - processor_data["audio"] = processor_data.pop("audios")

232 - return processor_data, passthrough_data

233 -

222 def _get_modality_field_names(self, modality: str) -> set[str]:

223 """Names of the fields the sub-processor for `modality` produces."""

224 # TODO: use else branch only once huggingface/transformers#44394 lands.

@@ -353,6 +341,9 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]

341

342 return mm_fields

343

344 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

345 + return self.dummy_inputs.get_dummy_text(mm_counts)

346 +

347 def _unpad_audios(

348 self,

349 hf_inputs: "BatchFeature",

@@ -597,21 +588,31 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):

588 # work for Transformers. The vision path has logic tied to

589 # `mm_tokens_per_modality` in _apply_vision()

590 hf_processor_mm_kwargs = {

600 - **hf_processor_mm_kwargs,

591 + # vLLM needs the untruncated sequence to keep placeholder

592 + # tokens aligned. Note that the text inputs are just dummy

593 + # text, not the original prompt. The original prompt is

594 + # already tokenized by the renderer.

595 + "truncation": False,

596 # HF processors only accept text, and the decoded string already

597 # contains any special tokens, so don't let them be added again

598 # (the HF processor call disables `add_special_tokens`).

599 "add_special_tokens": False,

600 + **hf_processor_mm_kwargs,

601 }

602

607 - valid_mm_items = mm_items.select(

608 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

603 + processor_data, _, passthrough_data = self._get_hf_mm_inputs(

604 + mm_items, hf_processor_mm_kwargs

605 )

610 - processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

606 +

607 + # The real prompt is fed to the processor below and the placeholder

608 + # ranges are read back out of its output, so the dummy text meant for

609 + # media-only callers is discarded here.

610 + processor_data.pop("text", None)

611 + has_mm_data = any(processor_data.values())

612

613 # Ask which modality owns each token of the expanded prompt, which

614 # is the only marking of the tokens an expansion adds around an item

614 - if any(processor_data.values()):

615 + if has_mm_data:

616 processor_data = {**processor_data, "return_mm_token_type_ids": True}

617

618 try:

@@ -621,7 +622,7 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):

622 hf_processor_mm_kwargs,

623 )

624 except ValueError:

624 - if any(processor_data.values()):

625 + if has_mm_data:

626 raise

627 # Some processors reject a prompt holding placeholders with

628 # no data to go with them, so tokenize it without them

@@ -792,31 +793,37 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):

793 except (AttributeError, KeyError, TypeError):

794 return None

795

796 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

797 + return self.dummy_inputs.get_dummy_text(mm_counts)

798 +

799 def _apply_hf_processor_main(

800 self,

801 mm_items: MultiModalDataItems,

798 - hf_processor_mm_kwargs: Mapping[str, object],

802 + hf_kwargs: Mapping[str, object],

803 ) -> "BatchFeature":

800 - mm_counts = mm_items.get_all_counts()

804 + hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(

805 + mm_items, hf_kwargs

806 + )

807

802 - valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})

803 - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

808 + if not hf_data:

809 + return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)

810

805 - prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)

811 + prompt_text = hf_data.pop("text")

812 + assert isinstance(prompt_text, str)

813

814 # Ask for the replacement each placeholder expands to, and record it as

815 # per-item fields: its token ids, and the tokens or patches behind them

809 - if has_mm_data := any(mm_data.values()):

810 - mm_data = {**mm_data, "return_text_replacement_offsets": True}

816 + if has_mm_data := any(hf_data.values()):

817 + hf_data = {**hf_data, "return_text_replacement_offsets": True}

818

819 try:

820 hf_inputs = self.info.ctx.call_hf_processor(

814 - self.info.get_hf_processor(**hf_processor_mm_kwargs),

815 - dict(text=prompt_text, **mm_data),

816 - hf_processor_mm_kwargs,

821 + self.info.get_hf_processor(**hf_kwargs),

822 + dict(text=prompt_text, **hf_data),

823 + hf_kwargs,

824 )

825 except ValueError:

819 - if any(mm_data.values()):

826 + if any(hf_data.values()):

827 raise

828 # Some processors reject a prompt holding placeholders with

829 # no data to go with them, so tokenize it without them

@@ -825,7 +832,7 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):

832 dict(input_ids=[tokenizer.encode(prompt_text)]), tensor_type="pt"

833 )

834 self._unpad_images(hf_inputs)

828 - self._unpad_audios(hf_inputs, mm_data, hf_processor_mm_kwargs)

835 + self._unpad_audios(hf_inputs, hf_data, hf_kwargs)

836

837 # Drop the inputs the model would reject

838 hf_inputs.pop("mm_token_type_ids", None)

@@ -845,9 +852,10 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):

852 "so it can be fixed, and install `transformers<5.15.0` in the "

853 "meantime to locate placeholders in the expanded prompt instead."

854 )

848 - hf_inputs.update(passthrough_data)

855 hf_inputs.pop("input_ids", None)

850 - return hf_inputs

856 + return self._finalize_hf_mm_data(

857 + hf_data, hf_kwargs, passthrough_data, hf_inputs

858 + )

859

860 tokenizer = self.info.get_tokenizer()

861 replacements = defaultdict[str, list[list[int]]](list)

@@ -865,7 +873,7 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):

873 )

874 if modality == "image":

875 hf_inputs["num_image_patches"] = self._get_num_image_patches(

868 - hf_inputs, mm_data, len(seqs)

876 + hf_inputs, hf_data, len(seqs)

877 )

878 elif modality == "audio":

879 counts = []

@@ -874,10 +882,11 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):

882 counts.append(int(ids.eq(_get_embed_token_id(ids)).sum()))

883 hf_inputs["num_audio_tokens"] = torch.tensor(counts)

884

877 - hf_inputs.update(passthrough_data)

885 hf_inputs.pop("input_ids")

886

880 - return hf_inputs

887 + return self._finalize_hf_mm_data(

888 + hf_data, hf_kwargs, passthrough_data, hf_inputs

889 + )

890

891

892 # From this version on, a processor reporting no offsets is an error rather than a

vllm/model_executor/models/ultravox.py

+22 -61

@@ -42,6 +42,7 @@ from vllm.multimodal.processing import (

42 PromptUpdate,

43 cached_encode,

44 )

45 +from vllm.multimodal.processing.processor import HFMultiModalInputs

46 from vllm.renderers import TokenizeParams

47 from vllm.sequence import IntermediateTensors

48 from vllm.transformers_utils.configs.ultravox import UltravoxConfig

@@ -190,72 +191,32 @@ class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo])

191

192

193 class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo]):

193 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

194 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

195 return self.dummy_inputs.get_dummy_text(mm_counts)

196

196 - def _preprocess_hf_mm_data(

197 - self,

198 - mm_data: Mapping[str, object],

199 - hf_processor_mm_kwargs: Mapping[str, object],

200 - ) -> tuple[Mapping[str, object], Mapping[str, object]]:

201 - audios = mm_data.get("audios", [])

202 - assert isinstance(audios, list)

203 -

204 - feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)

205 - hf_processor_mm_kwargs = dict(

206 - **hf_processor_mm_kwargs,

207 - sampling_rate=feature_extractor.sampling_rate,

208 - include_audio_num_chunks=True,

209 - )

210 -

211 - return mm_data, hf_processor_mm_kwargs

212 -

213 - def _apply_hf_processor_main(

197 + def _get_hf_mm_inputs(

198 self,

199 mm_items: MultiModalDataItems,

216 - hf_processor_mm_kwargs: Mapping[str, object],

217 - ) -> BatchFeature:

218 - valid_mm_items = mm_items.select(

219 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

220 - )

221 - processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

222 -

223 - if processor_data:

224 - processor_data, hf_processor_mm_kwargs = self._preprocess_hf_mm_data(

225 - processor_data,

226 - hf_processor_mm_kwargs,

200 + hf_kwargs: Mapping[str, object],

201 + ) -> HFMultiModalInputs:

202 + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)

203 +

204 + # The Ultravox processor accepts "audios" instead of "audio"

205 + hf_data = hf_inputs.hf_data

206 + if "audio" in hf_data:

207 + hf_data["audios"] = hf_data.pop("audio")

208 +

209 + feature_extractor = self.info.get_feature_extractor(**hf_kwargs)

210 + normalized_kwargs = dict(hf_inputs.hf_kwargs)

211 + # The remote processor already hardcodes truncation=False and forwards

212 + # its kwargs to the audio processor, where another value would collide.

213 + normalized_kwargs.pop("truncation", None)

214 + return hf_inputs._replace(

215 + hf_kwargs=dict(

216 + normalized_kwargs,

217 + sampling_rate=feature_extractor.sampling_rate,

218 + include_audio_num_chunks=True,

219 )

228 -

229 - prompt_text = self._get_hf_processor_text(mm_items.get_all_counts())

230 - if prompt_text is not None:

231 - processor_data = dict(text=prompt_text, **processor_data)

232 -

233 - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)

234 -

235 - def patched_call(**kwargs) -> BatchFeature:

236 - # The remote UltravoxProcessor does not support the

237 - # `truncation` kwarg injected by `call_hf_processor`: it

238 - # passes `truncation` to its audio processor by itself while

239 - # also forwarding `**kwargs` there, causing a duplicate

240 - # keyword argument error. Its hardcoded `truncation=False`

241 - # already matches vLLM's intended behavior.

242 - kwargs.pop("truncation", None)

243 -

244 - return hf_processor(**kwargs)

245 -

246 - processed_data = self.info.ctx.call_hf_processor(

247 - patched_call,

248 - processor_data,

249 - hf_processor_mm_kwargs,

250 - )

251 - processed_data.update(passthrough_data)

252 - else:

253 - processed_data = BatchFeature(dict(passthrough_data))

254 -

255 - return self._postprocess_hf_mm_data(

256 - processor_data,

257 - hf_processor_mm_kwargs,

258 - processed_data,

220 )

221

222 def _postprocess_hf_mm_data(

vllm/model_executor/models/unlimited_ocr.py

+2 -2

@@ -75,7 +75,7 @@ from vllm.multimodal.parse import (

75 from vllm.multimodal.processing import PromptReplacement, PromptUpdate

76 from vllm.multimodal.processing.context import TimingContext

77 from vllm.multimodal.processing.inputs import ProcessorInputs

78 -from vllm.multimodal.processing.processor import MultiModalProcessingInfo

78 +from vllm.multimodal.processing.processor import MultiModalProcessingResult

79 from vllm.transformers_utils.processors.deepseek_ocr import (

80 BASE_SIZE,

81 CROP_MODE,

@@ -225,7 +225,7 @@ class UnlimitedOCRMultiModalProcessor(DeepseekOCRMultiModalProcessor):

225 self,

226 inputs: ProcessorInputs,

227 timing_ctx: TimingContext,

228 - ) -> MultiModalProcessingInfo:

228 + ) -> MultiModalProcessingResult:

229 # The processor logic differs for single-image (crop) vs multi-image

230 # (no crop) requests. The processing cache assumes per-item output is

231 # invariant of how many images are passed per prompt, so we only cache

vllm/model_executor/models/voxtral.py

+18 -39

@@ -16,7 +16,7 @@ from mistral_common.protocol.instruct.messages import UserMessage

16 from mistral_common.protocol.instruct.request import ChatCompletionRequest

17 from mistral_common.protocol.transcription.request import TranscriptionRequest

18 from mistral_common.tokens.tokenizers.audio import Audio

19 -from transformers import BatchFeature, WhisperConfig

19 +from transformers import WhisperConfig

20

21 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig

22 from vllm.config.multimodal import BaseDummyOptions

@@ -36,7 +36,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY

36 from vllm.multimodal.inputs import (

37 MultiModalFieldConfig,

38 MultiModalKwargsItems,

39 - MultiModalKwargsOptionalItems,

39 NestedTensors,

40 )

41 from vllm.multimodal.parse import (

@@ -48,7 +47,8 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder

47 from vllm.multimodal.processing.processor import (

48 BaseMultiModalProcessor,

49 BaseProcessingInfo,

51 - MultiModalPromptUpdates,

50 + HFMultiModalInputs,

51 + MultiModalProcessingResult,

52 PlaceholderFeaturesInfo,

53 ProcessorInputs,

54 PromptReplacement,

@@ -61,7 +61,6 @@ from vllm.transformers_utils.processors.voxtral import (

61 MistralCommonFeatureExtractor,

62 MistralCommonVoxtralProcessor,

63 )

64 -from vllm.utils.collection_utils import is_list_of

64

65 from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsTranscription

66 from .utils import init_vllm_registered_model, maybe_prefix

@@ -209,21 +208,19 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])

208 def _maybe_apply_prompt_updates(

209 self,

210 mm_items: MultiModalDataItems,

212 - prompt_ids: list[int],

213 - mm_kwargs: MultiModalKwargsOptionalItems,

214 - mm_prompt_updates: MultiModalPromptUpdates,

211 + mm_res: MultiModalProcessingResult,

212 ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:

213 mm_item_counts = mm_items.get_all_counts()

217 - self._validate_mm_kwargs(mm_kwargs, mm_item_counts)

218 - self._validate_mm_updates(mm_prompt_updates, mm_item_counts)

214 + self._validate_mm_kwargs(mm_res.kwargs, mm_item_counts)

215 + self._validate_mm_updates(mm_res.prompt_updates, mm_item_counts)

216

217 mm_placeholders = self._find_mm_placeholders(

221 - prompt_ids,

222 - mm_prompt_updates,

218 + mm_res.prompt_ids,

219 + mm_res.prompt_updates,

220 )

221 self._validate_mm_placeholders(mm_placeholders, mm_item_counts)

222

226 - return prompt_ids, mm_placeholders

223 + return mm_res.prompt_ids, mm_placeholders

224

225 def _get_mm_fields_config(

226 self,

@@ -241,38 +238,20 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])

238 # skip validation here

239 pass

240

244 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

241 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

242 return self.dummy_inputs.get_dummy_text(mm_counts)

243

247 - def _preprocess_hf_mm_data(

244 + def _get_hf_mm_inputs(

245 self,

249 - mm_data: Mapping[str, object],

250 - hf_processor_mm_kwargs: Mapping[str, object],

251 - ) -> tuple[Mapping[str, object], Mapping[str, object]]:

252 - mm_data = dict(mm_data)

253 - audios = mm_data.pop("audios", [])

254 -

255 - if audios:

256 - # MistralCommonVoxtralProcessor accepts "audio"

257 - mm_data["audio"] = audios

246 + mm_items: MultiModalDataItems,

247 + hf_kwargs: Mapping[str, object],

248 + ) -> HFMultiModalInputs:

249 + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)

250

251 # Avoid padding issue

260 - hf_processor_mm_kwargs = dict(**hf_processor_mm_kwargs, return_tensors=None)

261 -

262 - return mm_data, hf_processor_mm_kwargs

263 -

264 - def _postprocess_hf_mm_data(

265 - self,

266 - mm_data: Mapping[str, object],

267 - hf_processor_mm_kwargs: Mapping[str, object],

268 - processed_data: BatchFeature,

269 - ) -> BatchFeature:

270 - # Missing batch dimension

271 - input_ids = processed_data.get("input_ids")

272 - if is_list_of(input_ids, int):

273 - processed_data["input_ids"] = [input_ids]

274 -

275 - return processed_data

252 + return hf_inputs._replace(

253 + hf_kwargs=dict(hf_inputs.hf_kwargs, return_tensors=None)

254 + )

255

256 def _get_prompt_updates(

257 self,

vllm/model_executor/models/voxtral_realtime.py

+5 -6

@@ -28,11 +28,10 @@ from vllm.model_executor.models.voxtral import (

28 )

29 from vllm.multimodal import MULTIMODAL_REGISTRY

30 from vllm.multimodal.cache import _I, BaseMultiModalProcessorCache

31 -from vllm.multimodal.inputs import MultiModalKwargsOptionalItems

31 from vllm.multimodal.parse import MultiModalDataItems

32 from vllm.multimodal.processing import BaseDummyInputsBuilder

33 from vllm.multimodal.processing.processor import (

35 - MultiModalPromptUpdates,

34 + MultiModalProcessingResult,

35 PlaceholderFeaturesInfo,

36 )

37 from vllm.sequence import IntermediateTensors

@@ -60,10 +59,10 @@ class VoxtralRealtimeMultiModalProcessor(VoxtralMultiModalProcessor):

59 def _maybe_apply_prompt_updates(

60 self,

61 mm_items: MultiModalDataItems,

63 - prompt_ids: list[int],

64 - mm_kwargs: MultiModalKwargsOptionalItems,

65 - mm_prompt_updates: MultiModalPromptUpdates,

62 + mm_res: MultiModalProcessingResult,

63 ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:

64 + mm_kwargs = mm_res.kwargs

65 +

66 # there are no placeholder audio tokens for streaming

67 # so we need to build the place placeholder positions manually

68

@@ -86,7 +85,7 @@ class VoxtralRealtimeMultiModalProcessor(VoxtralMultiModalProcessor):

85 * [0], # only used for length computation, so we can take dummy inputs

86 is_embed=None,

87 )

89 - return prompt_ids, {"audio": [features_info]}

88 + return mm_res.prompt_ids, {"audio": [features_info]}

89

90

91 class TimeEmbedding(torch.nn.Module):

vllm/model_executor/models/whisper.py

+16 -28

@@ -61,6 +61,7 @@ from vllm.multimodal.processing import (

61 PromptReplacement,

62 PromptUpdate,

63 )

64 +from vllm.multimodal.processing.processor import HFMultiModalInputs

65 from vllm.renderers import TokenizeParams

66 from vllm.transformers_utils.processor import cached_processor_from_config

67 from vllm.utils.jsontree import json_map_leaves

@@ -743,6 +744,9 @@ class WhisperDummyInputsBuilder(BaseDummyInputsBuilder[WhisperProcessingInfo]):

744

745

746 class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo]):

747 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

748 + return self.dummy_inputs.get_dummy_text(mm_counts)

749 +

750 def create_encoder_prompt(

751 self,

752 prompt: list[int],

@@ -754,37 +758,21 @@ class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo

758 # for encoder profiling.

759 return [0]

760

757 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

758 - return self.dummy_inputs.get_dummy_text(mm_counts)

759 -

760 - def _preprocess_hf_mm_data(

761 + def _get_hf_mm_inputs(

762 self,

762 - mm_data: Mapping[str, object],

763 - hf_processor_mm_kwargs: Mapping[str, object],

764 - ) -> tuple[Mapping[str, object], Mapping[str, object]]:

765 - feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)

766 -

767 - mm_data = dict(mm_data)

768 - mm_data["audio"] = mm_data.pop("audios")

769 -

770 - hf_processor_mm_kwargs = dict(

771 - **hf_processor_mm_kwargs,

772 - sampling_rate=feature_extractor.sampling_rate,

763 + mm_items: MultiModalDataItems,

764 + hf_kwargs: Mapping[str, object],

765 + ) -> HFMultiModalInputs:

766 + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)

767 +

768 + feature_extractor = self.info.get_feature_extractor(**hf_kwargs)

769 + return hf_inputs._replace(

770 + hf_kwargs=dict(

771 + hf_inputs.hf_kwargs,

772 + sampling_rate=feature_extractor.sampling_rate,

773 + )

774 )

775

775 - return mm_data, hf_processor_mm_kwargs

776 -

777 - def _postprocess_hf_mm_data(

778 - self,

779 - mm_data: Mapping[str, object],

780 - hf_processor_mm_kwargs: Mapping[str, object],

781 - processed_data: BatchFeature,

782 - ) -> BatchFeature:

783 - if "labels" in processed_data:

784 - processed_data["input_ids"] = processed_data.pop("labels")

785 -

786 - return processed_data

787 -

776 def _get_mm_fields_config(

777 self,

778 hf_inputs: BatchFeature,

vllm/models/dots3_note/common/processor.py

+26 -18

@@ -34,6 +34,7 @@ from vllm.multimodal.processing import (

34 PromptUpdate,

35 PromptUpdateDetails,

36 )

37 +from vllm.multimodal.processing.processor import HFMultiModalInputs

38 from vllm.transformers_utils.repo_utils import get_hf_file_to_dict

39

40 from .video import preprocess_dots3_note_video

@@ -630,26 +631,33 @@ class Dots3NoteDummyInputsBuilder(BaseDummyInputsBuilder[Dots3NoteProcessingInfo

631

632

633 class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingInfo]):

633 - def _get_hf_mm_data(

634 + def _get_hf_mm_inputs(

635 self,

636 mm_items: MultiModalDataItems,

636 - ) -> tuple[Mapping[str, object], Mapping[str, object]]:

637 - processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)

638 - if "video" not in mm_items:

639 - return processor_data, passthrough_data

640 -

641 - videos = mm_items.get_items("video", VideoProcessorItems)

642 - raw_videos: list[object] = []

643 - for index, item in enumerate(videos.data):

644 - if isinstance(item, MediaWithBytes):

645 - raw_videos.append(item.original_bytes)

646 - else:

647 - raw_videos.append(videos.get(index))

648 - processor_data = dict(processor_data)

649 - processor_data["videos"] = raw_videos

650 - return processor_data, passthrough_data

651 -

652 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:

637 + hf_kwargs: Mapping[str, object],

638 + ) -> HFMultiModalInputs:

639 + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)

640 +

641 + # The Dots3Note processor accepts "audios" instead of "audio"

642 + hf_data = hf_inputs.hf_data

643 + if "audio" in hf_data:

644 + hf_data["audios"] = hf_data.pop("audio")

645 +

646 + if "video" in mm_items:

647 + videos = mm_items.get_items("video", VideoProcessorItems)

648 +

649 + raw_videos: list[object] = []

650 + for index, item in enumerate(videos.data):

651 + if isinstance(item, MediaWithBytes):

652 + raw_videos.append(item.original_bytes)

653 + else:

654 + raw_videos.append(videos.get(index))

655 +

656 + hf_data["videos"] = raw_videos

657 +

658 + return hf_inputs

659 +

660 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

661 return self.dummy_inputs.get_dummy_text(mm_counts)

662

663 def _get_mm_fields_config(

vllm/models/inkling/common/mm_preprocess.py

+8 -8

@@ -182,20 +182,19 @@ class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo])

182 def _apply_hf_processor_main(

183 self,

184 mm_items: MultiModalDataItems,

185 - hf_processor_mm_kwargs: Mapping[str, object],

185 + hf_kwargs: Mapping[str, object],

186 ) -> BatchFeature:

187 - valid_mm_items = mm_items.select(

188 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

187 + mm_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(

188 + mm_items, hf_kwargs

189 )

190 - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

190

191 prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())

192

194 - processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)

193 + processor = self.info.get_hf_processor(**hf_kwargs)

194 tokenizer = self.info.get_tokenizer()

195

196 images = mm_data.get("images") or []

198 - audios = mm_data.get("audios") or []

197 + audios = mm_data.get("audio") or []

198 if not isinstance(images, list):

199 images = list(cast(Iterable[Any], images))

200 if not isinstance(audios, list):

@@ -235,8 +234,9 @@ class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo])

234 data["num_audio_tokens"] = torch.tensor(num_audio_tokens, dtype=torch.int64)

235

236 processed_data = BatchFeature(data=data, tensor_type=None)

238 - processed_data.update(passthrough_data)

239 - return processed_data

237 + return self._finalize_hf_mm_data(

238 + mm_data, hf_kwargs, passthrough_data, processed_data

239 + )

240

241 def _tokenize_with_placeholders(

242 self,

vllm/models/minimax_m3/common/mm_preprocess.py

+25 -25

@@ -32,6 +32,7 @@ from vllm.multimodal.processing import (

32 PromptUpdate,

33 PromptUpdateDetails,

34 )

35 +from vllm.multimodal.processing.processor import HFMultiModalInputs

36 from vllm.multimodal.video import (

37 VIDEO_LOADER_REGISTRY,

38 VideoBackend,

@@ -307,26 +308,21 @@ class MiniMaxM3VLDummyInputsBuilder(BaseDummyInputsBuilder[MiniMaxM3VLProcessing

308 class MiniMaxM3VLMultiModalProcessor(

309 BaseMultiModalProcessor[MiniMaxM3VLProcessingInfo]

310 ):

310 - def _apply_hf_processor_main(

311 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:

312 + return self.dummy_inputs.get_dummy_text(mm_counts)

313 +

314 + def _get_hf_mm_inputs(

315 self,

316 mm_items: MultiModalDataItems,

313 - hf_processor_mm_kwargs: Mapping[str, object],

314 - ) -> BatchFeature:

315 - valid_mm_items = mm_items.select(

316 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

317 - )

318 - mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

319 -

320 - if not mm_data:

321 - return BatchFeature(dict(passthrough_data))

322 -

323 - prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())

317 + hf_kwargs: Mapping[str, object],

318 + ) -> HFMultiModalInputs:

319 + hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)

320 + hf_data = hf_inputs.hf_data

321

325 - mm_data = dict(mm_data)

322 # With ``video_needs_metadata=True`` each video arrives as a

323 # ``(frames, metadata)`` tuple. Split the frames back out and forward the

324 # metadata as ``VideoMetadata`` so the processor emits timestamps.

329 - videos = cast(list | None, mm_data.get("videos"))

325 + videos = cast(list | None, hf_data.get("videos"))

326 video_metadata: list[VideoMetadata] | None = None

327 if videos:

328 frames_only = []

@@ -346,21 +342,25 @@ class MiniMaxM3VLMultiModalProcessor(

342 # stays consistent with _get_prompt_updates.

343 meta.setdefault("total_num_frames", len(frames))

344 video_metadata.append(VideoMetadata(**meta))

349 - mm_data["videos"] = frames_only

345 + hf_data["videos"] = frames_only

346

351 - # Override the video processor's default do_resize=False (set for a

352 - # pre-resized pipeline) to True for vLLM's raw-frame inputs.

353 - merged = dict(do_resize=True, **hf_processor_mm_kwargs)

354 - data = dict(text=prompt_text, **mm_data)

347 if video_metadata is not None:

356 - data["video_metadata"] = video_metadata

357 - processed_data = self.info.ctx.call_hf_processor(

358 - self.info.get_hf_processor(**hf_processor_mm_kwargs),

359 - data,

348 + hf_data["video_metadata"] = video_metadata

349 +

350 + return hf_inputs

351 +

352 + def _call_hf_processor(

353 + self,

354 + hf_data: Mapping[str, object],

355 + hf_kwargs: Mapping[str, object],

356 + ) -> BatchFeature:

357 + # Override the video processor's default for vLLM's raw-frame inputs.

358 + merged = dict(do_resize=True, **hf_kwargs)

359 + return self.info.ctx.call_hf_processor(

360 + self.info.get_hf_processor(**hf_kwargs),

361 + hf_data,

362 merged,

363 )

362 - processed_data.update(passthrough_data)

363 - return processed_data

364

365 def _get_mm_fields_config(

366 self,

vllm/multimodal/processing/context.py

+2 -8

@@ -321,12 +321,6 @@ class InputProcessingContext:

321

322 merged_kwargs = self.get_merged_mm_kwargs(kwargs)

323

324 - # vLLM needs the full untruncated sequence to keep multi-modal

325 - # placeholder tokens aligned; note that the text inputs in

326 - # call_hf_processor are just dummy text, not the original prompt.

327 - # The original prompt is already tokenized by the renderer.

328 - merged_kwargs.setdefault("truncation", False)

329 -

324 allowed_kwargs = get_allowed_kwarg_only_overrides(

325 hf_processor,

326 merged_kwargs,

@@ -426,7 +420,7 @@ class BaseProcessingInfo:

420 def get_data_parser(self) -> MultiModalDataParser:

421 """Constructs a parser to preprocess multi-modal data items

422 before passing them to

429 - [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].

423 + [`_get_hf_mm_inputs`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs].

424

425 You can support additional modalities by creating a subclass

426 of [`MultiModalDataParser`][vllm.multimodal.parse.MultiModalDataParser]

@@ -506,7 +500,7 @@ class BaseProcessingInfo:

500 """Normalize [`MultiModalDataDict`][vllm.inputs.MultiModalDataDict]

501 to [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems]

502 before passing them to

509 - [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].

503 + [`_get_hf_mm_inputs`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs].

504 """

505 mm_items = self.data_parser.parse_mm_data(mm_data)

506

vllm/multimodal/processing/processor.py

+117 -85

@@ -1008,7 +1008,20 @@ For an item `MultiModalPromptUpdates[k][i]`,

1008 _I = TypeVar("_I", bound=BaseProcessingInfo)

1009

1010

1011 -class MultiModalProcessingInfo(NamedTuple):

1011 +class HFMultiModalInputs(NamedTuple):

1012 + hf_data: dict[str, object]

1013 + """The data inputs to the HF processor."""

1014 +

1015 + hf_kwargs: Mapping[str, object]

1016 + """The non-data inputs to the HF processor."""

1017 +

1018 + passthrough_data: dict[str, object]

1019 + """The inputs to assign to the output of the HF processor,

1020 + instead of passing them to the HF processor."""

1021 +

1022 +

1023 +class MultiModalProcessingResult(NamedTuple):

1024 + prompt_ids: list[int]

1025 kwargs: MultiModalKwargsOptionalItems

1026 hashes: MultiModalHashes

1027 prompt_updates: MultiModalPromptUpdates

@@ -1137,23 +1150,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1150 ) -> Mapping[str, list[PlaceholderFeaturesInfo]]:

1151 return find_mm_placeholders(new_token_ids, mm_prompt_updates)

1152

1140 - def _get_hf_mm_data(

1141 - self,

1142 - mm_items: MultiModalDataItems,

1143 - ) -> tuple[Mapping[str, object], Mapping[str, object]]:

1144 - """Extract processor and passthrough data from multi-modal items."""

1145 - processor_data = dict[str, object]()

1146 - passthrough_data = dict[str, object]()

1147 -

1148 - for items in mm_items.values():

1149 - processor_data.update(items.get_processor_data())

1150 - passthrough_data.update(items.get_passthrough_data())

1151 -

1152 - return processor_data, passthrough_data

1153 -

1154 - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str | None:

1155 - """Get the text to pass to the HF processor alongside the multi-modal

1156 - data.

1153 + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str | None:

1154 + """Get the text to pass to the HF processor alongside the multi-modal data.

1155

1156 By default, no text is passed. If the HF processor requires that

1157 text and multi-modal items correspond to each other, you should

@@ -1162,29 +1160,53 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1160 """

1161 return None

1162

1165 - def _preprocess_hf_mm_data(

1163 + def _get_hf_mm_inputs(

1164 self,

1167 - mm_data: Mapping[str, object],

1168 - hf_processor_mm_kwargs: Mapping[str, object],

1169 - ) -> tuple[Mapping[str, object], Mapping[str, object]]:

1170 - """Pre-process the multi-modal data and HF processor keyword arguments

1171 - before they are passed to the HF processor.

1172 -

1173 - By default, both are returned as-is. If the HF processor expects the

1174 - multi-modal data under different keys than those provided by the

1175 - multi-modal items (e.g. `audio` instead of `audios`), or requires

1176 - additional keyword arguments (e.g. `sampling_rate`), you should

1177 - override this method.

1165 + mm_items: MultiModalDataItems,

1166 + hf_kwargs: Mapping[str, object],

1167 + ) -> HFMultiModalInputs:

1168 + """Extract the multi-modal data and corresponding keyword arguments

1169 + for HF processing.

1170 +

1171 + If the HF processor expects the multi-modal data under different keys

1172 + than those provided by the multi-modal items, or requires additional

1173 + keyword arguments, you should override this method.

1174 """

1179 - return mm_data, hf_processor_mm_kwargs

1175 + hf_data = dict[str, object]()

1176 + passthrough_data = dict[str, object]()

1177 +

1178 + for items in mm_items.values():

1179 + if not items:

1180 + continue

1181 +

1182 + hf_data.update(items.get_processor_data())

1183 + passthrough_data.update(items.get_passthrough_data())

1184 +

1185 + if hf_data:

1186 + prompt_text = self._get_hf_mm_text(mm_items.get_all_counts())

1187 + if prompt_text is not None:

1188 + hf_data = dict(text=prompt_text, **hf_data)

1189 +

1190 + # vLLM needs the untruncated sequence to keep placeholder

1191 + # tokens aligned. Note that the text inputs are just dummy

1192 + # text, not the original prompt. The original prompt is

1193 + # already tokenized by the renderer.

1194 + hf_kwargs = {"truncation": False, **hf_kwargs}

1195 +

1196 + # HF processors accept (images, text, videos, audio)

1197 + # so we need to remap vLLM's "audios" -> HF "audio"

1198 + if "audios" in hf_data:

1199 + hf_data["audio"] = hf_data.pop("audios")

1200 +

1201 + return HFMultiModalInputs(hf_data, hf_kwargs, passthrough_data)

1202

1203 def _postprocess_hf_mm_data(

1204 self,

1183 - mm_data: Mapping[str, object],

1205 + mm_inputs: Mapping[str, object],

1206 hf_processor_mm_kwargs: Mapping[str, object],

1207 processed_data: BatchFeature,

1208 ) -> BatchFeature:

1187 - """Post-process the output of the HF processor.

1209 + """Post-process the combined processor and passthrough data.

1210

1211 By default, the output is returned as-is. If you need to modify the

1212 output of the HF processor before it is converted into multi-modal

@@ -1192,41 +1214,51 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1214 """

1215 return processed_data

1216

1217 + def _call_hf_processor(

1218 + self,

1219 + hf_data: Mapping[str, object],

1220 + hf_kwargs: Mapping[str, object],

1221 + ) -> BatchFeature:

1222 + """Call the HF processor on the normalized multi-modal inputs."""

1223 + return self.info.ctx.call_hf_processor(

1224 + self.info.get_hf_processor(**hf_kwargs),

1225 + hf_data,

1226 + hf_kwargs,

1227 + )

1228 +

1229 + def _finalize_hf_mm_data(

1230 + self,

1231 + hf_data: Mapping[str, object],

1232 + hf_kwargs: Mapping[str, object],

1233 + passthrough_data: Mapping[str, object],

1234 + processed_data: BatchFeature | None = None,

1235 + ) -> BatchFeature:

1236 + """Merge passthrough fields and apply model-specific postprocessing."""

1237 + if processed_data is None:

1238 + from transformers.feature_extraction_utils import BatchFeature

1239 +

1240 + processed_data = BatchFeature()

1241 +

1242 + processed_data.update(passthrough_data)

1243 + return self._postprocess_hf_mm_data(hf_data, hf_kwargs, processed_data)

1244 +

1245 def _apply_hf_processor_main(

1246 self,

1247 mm_items: MultiModalDataItems,

1198 - hf_processor_mm_kwargs: Mapping[str, object],

1248 + hf_kwargs: Mapping[str, object],

1249 ) -> BatchFeature:

1250 """Apply the HF processor on the multi-modal data."""

1201 - valid_mm_items = mm_items.select(

1202 - {k for k, c in mm_items.get_all_counts().items() if c > 0}

1251 + hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(

1252 + mm_items, hf_kwargs

1253 )

1204 - processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)

1254

1206 - if processor_data:

1207 - processor_data, hf_processor_mm_kwargs = self._preprocess_hf_mm_data(

1208 - processor_data, hf_processor_mm_kwargs

1209 - )

1210 -

1211 - prompt_text = self._get_hf_processor_text(mm_items.get_all_counts())

1212 - if prompt_text is not None:

1213 - processor_data = dict(text=prompt_text, **processor_data)

1214 -

1215 - processed_data = self.info.ctx.call_hf_processor(

1216 - self.info.get_hf_processor(**hf_processor_mm_kwargs),

1217 - processor_data,

1218 - hf_processor_mm_kwargs,

1219 - )

1220 - processed_data.update(passthrough_data)

1255 + if hf_data:

1256 + processed_data = self._call_hf_processor(hf_data, hf_kwargs)

1257 else:

1222 - from transformers.feature_extraction_utils import BatchFeature

1223 -

1224 - processed_data = BatchFeature(dict(passthrough_data))

1258 + processed_data = None

1259

1226 - return self._postprocess_hf_mm_data(

1227 - processor_data,

1228 - hf_processor_mm_kwargs,

1229 - processed_data,

1260 + return self._finalize_hf_mm_data(

1261 + hf_data, hf_kwargs, passthrough_data, processed_data

1262 )

1263

1264 def _postprocess_prompt(self, prompt: list[int]) -> list[int]:

@@ -1261,6 +1293,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1293

1294 mm_missing_data = {}

1295 for modality, idxs in mm_missing_idxs.items():

1296 + if not idxs:

1297 + continue

1298 +

1299 missing_modality_data = []

1300 for idx in idxs:

1301 data = mm_data_items[modality][idx]

@@ -1271,6 +1306,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1306 )

1307 else:

1308 missing_modality_data.append(data)

1309 +

1310 mm_missing_data[modality] = missing_modality_data

1311

1312 mm_missing_items = self.info.parse_mm_data(mm_missing_data, validate=False)

@@ -1342,11 +1378,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1378 self,

1379 inputs: ProcessorInputs,

1380 timing_ctx: TimingContext,

1345 - ) -> MultiModalProcessingInfo:

1381 + ) -> MultiModalProcessingResult:

1382 with timing_ctx.record("apply_hf_processor"):

1383 mm_processed_data = self._apply_hf_processor_main(

1384 mm_items=inputs.mm_data_items,

1349 - hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,

1385 + hf_kwargs=inputs.hf_processor_mm_kwargs,

1386 )

1387

1388 mm_kwargs = MultiModalKwargsItems.from_hf_inputs(

@@ -1369,26 +1405,28 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1405 mm_kwargs,

1406 )

1407

1372 - mm_info = MultiModalProcessingInfo(

1408 + return MultiModalProcessingResult(

1409 + prompt_ids=self._postprocess_prompt(inputs.prompt),

1410 kwargs=mm_kwargs,

1411 hashes=mm_hashes,

1412 prompt_updates=mm_prompt_updates,

1413 )

1414

1378 - return mm_info

1379 -

1415 def _cached_apply_hf_processor(

1416 self,

1417 inputs: ProcessorInputs,

1418 timing_ctx: TimingContext,

1384 - ) -> MultiModalProcessingInfo:

1419 + ) -> MultiModalProcessingResult:

1420 """Apply the HF processor on the full prompt text,

1421 caching the results and reusing cached results.

1422 """

1423 cache = self.cache

1424 + has_passthrough_data = any(

1425 + len(items.get_passthrough_data()) > 0

1426 + for items in inputs.mm_data_items.values()

1427 + )

1428

1390 - _, passthrough_data = self._get_hf_mm_data(inputs.mm_data_items)

1391 - if cache is None or passthrough_data:

1429 + if cache is None or has_passthrough_data:

1430 return self._apply_hf_processor(inputs, timing_ctx)

1431

1432 with timing_ctx.record("get_mm_hashes"):

@@ -1410,7 +1448,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1448 with timing_ctx.record("apply_hf_processor"):

1449 mm_missing_processed_data = self._apply_hf_processor_main(

1450 mm_items=mm_missing_data_items,

1413 - hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,

1451 + hf_kwargs=inputs.hf_processor_mm_kwargs,

1452 )

1453

1454 mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs(

@@ -1435,14 +1473,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1473 mm_missing_prompt_updates=mm_missing_prompt_updates,

1474 )

1475

1438 - mm_info = MultiModalProcessingInfo(

1476 + return MultiModalProcessingResult(

1477 + prompt_ids=self._postprocess_prompt(inputs.prompt),

1478 kwargs=mm_kwargs,

1479 hashes=mm_hashes,

1480 prompt_updates=mm_prompt_updates,

1481 )

1482

1444 - return mm_info

1445 -

1483 def _apply_token_matches(

1484 self,

1485 prompt: list[int],

@@ -1670,17 +1707,15 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1707 def _maybe_apply_prompt_updates(

1708 self,

1709 mm_items: MultiModalDataItems,

1673 - prompt_ids: list[int],

1674 - mm_kwargs: MultiModalKwargsOptionalItems,

1675 - mm_prompt_updates: MultiModalPromptUpdates,

1710 + mm_res: MultiModalProcessingResult,

1711 ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:

1712 mm_item_counts = mm_items.get_all_counts()

1678 - self._validate_mm_kwargs(mm_kwargs, mm_item_counts)

1679 - self._validate_mm_updates(mm_prompt_updates, mm_item_counts)

1713 + self._validate_mm_kwargs(mm_res.kwargs, mm_item_counts)

1714 + self._validate_mm_updates(mm_res.prompt_updates, mm_item_counts)

1715

1716 prompt_ids, mm_placeholders = self._apply_prompt_updates(

1682 - prompt_ids,

1683 - mm_prompt_updates,

1717 + mm_res.prompt_ids,

1718 + mm_res.prompt_updates,

1719 )

1720

1721 self._validate_mm_placeholders(mm_placeholders, mm_item_counts)

@@ -1705,15 +1740,12 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1740 4. Extract information about the placeholder tokens from the

1741 processed token IDs.

1742 """

1708 - prompt_ids = self._postprocess_prompt(inputs.prompt)

1709 - mm_info = self._cached_apply_hf_processor(inputs, timing_ctx)

1743 + mm_res = self._cached_apply_hf_processor(inputs, timing_ctx)

1744

1745 with timing_ctx.record("apply_prompt_updates"):

1746 prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(

1747 mm_items=inputs.mm_data_items,

1714 - prompt_ids=prompt_ids,

1715 - mm_kwargs=mm_info.kwargs,

1716 - mm_prompt_updates=mm_info.prompt_updates,

1748 + mm_res=mm_res,

1749 )

1750

1751 mm_placeholder_ranges = {

@@ -1723,8 +1755,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1755

1756 return mm_input(

1757 prompt_token_ids=prompt_ids,

1726 - mm_kwargs=mm_info.kwargs,

1727 - mm_hashes=mm_info.hashes,

1758 + mm_kwargs=mm_res.kwargs,

1759 + mm_hashes=mm_res.hashes,

1760 mm_placeholders=mm_placeholder_ranges,

1761 )

1762

[MM] Further cleanup _apply_hf_processor_main (#53610)