b00f475f09 · tduncan/Nvidia-Optimization-Stack-VLLM

docs/contributing/model/multimodal.md

-7

@@ -456,13 +456,11 @@ return a schema of the tensors outputted by the HF processor that are related to

456 prompt: str,

457 mm_data: Mapping[str, object],

458 mm_kwargs: Mapping[str, object],

459 - tok_kwargs: Mapping[str, object],

459 ) -> BatchFeature:

460 processed_outputs = super()._call_hf_processor(

461 prompt=prompt,

462 mm_data=mm_data,

463 mm_kwargs=mm_kwargs,

465 - tok_kwargs=tok_kwargs,

464 )

465

466 pixel_values = processed_outputs.get("pixel_values")

@@ -479,11 +477,6 @@ return a schema of the tensors outputted by the HF processor that are related to

477 return processed_outputs

478 ```

479

482 - !!! note

483 - The `_call_hf_processor` method specifies both `mm_kwargs` and `tok_kwargs` for

484 - processing. `mm_kwargs` is used to both initialize and call the huggingface

485 - processor, whereas `tok_kwargs` is only used to call the huggingface processor.

486 -

480 Since `pixel_values` is now a list with one tensor per image, we can override

481 [_get_mm_fields_config][vllm.multimodal.processing.BaseMultiModalProcessor._get_mm_fields_config] as follows:

482

tests/models/multimodal/processing/test_audioflamingo3.py

+1 -1

@@ -107,7 +107,7 @@ def test_audio_chunk_counting(mock_ctx):

107 mm_data = {"audio": [audio_1, audio_2]}

108 prompt = "<|user|>Listen.<|end|>"

109

110 - processed = processor._call_hf_processor(prompt, mm_data, {}, {})

110 + processed = processor._call_hf_processor(prompt, mm_data, {})

111

112 chunk_counts = processed["chunk_counts"]

113

tests/models/multimodal/processing/test_common.py

+6 -62

@@ -145,10 +145,10 @@ def get_transformers_backend_model_ids_to_test():

145 )

146

147

148 -def get_text_token_prompts(

148 +def get_token_prompt(

149 processor: BaseMultiModalProcessor,

150 mm_data: MultiModalDataDict,

151 -):

151 +) -> list[int]:

152 dummy_inputs = processor.dummy_inputs

153 tokenizer: TokenizerLike = processor.info.get_tokenizer()

154 model_config = processor.info.ctx.model_config

@@ -178,21 +178,10 @@ def get_text_token_prompts(

178 mm_options={},

179 )

180

181 - text_prompt: str | None

182 - token_prompt: list[int]

183 - if isinstance(inputs.prompt, list):

184 - text_prompt = None

185 - token_prompt = inputs.prompt

186 - elif isinstance(inputs.prompt, str):

187 - text_prompt = inputs.prompt

188 - token_prompt = tokenizer.encode(

189 - text_prompt,

190 - **processor.info.get_default_tok_params().get_encode_kwargs(),

191 - )

192 - else:

181 + if not isinstance(inputs.prompt, list):

182 raise TypeError(type(inputs.prompt))

183

195 - return text_prompt, token_prompt

184 + return inputs.prompt

185

186

187 def random_vision_chunk(

@@ -358,7 +347,7 @@ def _test_processing_correctness_one(

347 ):

348 model_type = model_config.hf_config.model_type

349

361 - text_prompt, token_prompt = get_text_token_prompts(baseline_processor, mm_data)

350 + token_prompt = get_token_prompt(baseline_processor, mm_data)

351 mm_items = baseline_processor.info.parse_mm_data(mm_data)

352 ignore_mm_keys = _IGNORE_MM_KEYS.get(model_type, set[str]())

353

@@ -381,55 +370,10 @@ def _test_processing_correctness_one(

370 msg=(

371 f"Failed ({batch_idx=}, {hit_rate=}, "

372 f"{num_batches=}, {simplify_rate=}, "

384 - f"{text_prompt=}, {token_prompt=}, {mm_data=})"

373 + f"{token_prompt=}, {mm_data=})"

374 ),

375 )

376

388 - if text_prompt is not None:

389 - baseline_text_result = baseline_processor(

390 - text_prompt,

391 - mm_items=mm_items,

392 - hf_processor_mm_kwargs={},

393 - )

394 - cached_text_result = cached_processor(

395 - text_prompt,

396 - mm_items=mm_items,

397 - hf_processor_mm_kwargs={},

398 - )

399 -

400 - _assert_inputs_equal(

401 - baseline_text_result,

402 - cached_text_result,

403 - ignore_mm_keys=ignore_mm_keys,

404 - msg=(

405 - f"Failed ({batch_idx=}, {hit_rate=}, "

406 - f"{num_batches=}, {simplify_rate=}, "

407 - f"{text_prompt=}, {token_prompt=}, {mm_data=})"

408 - ),

409 - )

410 -

411 - _assert_inputs_equal(

412 - baseline_text_result,

413 - baseline_tokenized_result,

414 - ignore_mm_keys=ignore_mm_keys,

415 - msg=(

416 - f"Failed ({batch_idx=}, {hit_rate=}, "

417 - f"{num_batches=}, {simplify_rate=}, "

418 - f"{text_prompt=}, {token_prompt=}, {mm_data=})"

419 - ),

420 - )

421 -

422 - _assert_inputs_equal(

423 - cached_text_result,

424 - cached_tokenized_result,

425 - ignore_mm_keys=ignore_mm_keys,

426 - msg=(

427 - f"Failed ({batch_idx=}, {hit_rate=}, "

428 - f"{num_batches=}, {simplify_rate=}, "

429 - f"{text_prompt=}, {token_prompt=}, {mm_data=})"

430 - ),

431 - )

432 -

377

378 @pytest.mark.parametrize("model_id", get_model_ids_to_test())

379 @pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])

tests/models/multimodal/processing/test_llava_next.py

+17 -2

@@ -11,6 +11,7 @@ from pqdm.threads import pqdm

11 from vllm.multimodal import MULTIMODAL_REGISTRY

12 from vllm.multimodal.parse import ImageSize

13 from vllm.multimodal.processing import BaseMultiModalProcessor

14 +from vllm.tokenizers.hf import maybe_make_thread_pool

15

16 from ...utils import build_model_context

17

@@ -144,7 +145,14 @@ def test_processor_prompt_replacements_regression(model_id, num_imgs):

145 mm_processor_kwargs=None,

146 limit_mm_per_prompt={"image": num_imgs},

147 )

147 - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

148 +

149 + # Avoid tokenizer already borrowed error

150 + maybe_make_thread_pool(ctx.tokenizer)

151 +

152 + processor = MULTIMODAL_REGISTRY.create_processor(

153 + ctx.model_config,

154 + tokenizer=ctx.tokenizer,

155 + )

156

157 image_ratios = [

158 (171, 152),

@@ -177,7 +185,14 @@ def test_processor_prompt_replacements_all(model_id, num_imgs):

185 mm_processor_kwargs=None,

186 limit_mm_per_prompt={"image": num_imgs},

187 )

180 - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

188 +

189 + # Avoid tokenizer already borrowed error

190 + maybe_make_thread_pool(ctx.tokenizer)

191 +

192 + processor = MULTIMODAL_REGISTRY.create_processor(

193 + ctx.model_config,

194 + tokenizer=ctx.tokenizer,

195 + )

196

197 seen_aspect_ratios = set[float]()

198 image_sizes = list[ImageSize]()

tests/models/multimodal/processing/test_llava_onevision.py

+18 -2

@@ -11,6 +11,7 @@ from pqdm.threads import pqdm

11 from vllm.multimodal import MULTIMODAL_REGISTRY

12 from vllm.multimodal.parse import ImageSize

13 from vllm.multimodal.processing import BaseMultiModalProcessor

14 +from vllm.tokenizers.hf import maybe_make_thread_pool

15

16 from ...utils import build_model_context

17

@@ -42,7 +43,15 @@ def test_processor_max_tokens(model_id):

43 mm_processor_kwargs=None,

44 limit_mm_per_prompt={"image": 1},

45 )

45 - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

46 +

47 + # Avoid tokenizer already borrowed error

48 + maybe_make_thread_pool(ctx.tokenizer)

49 +

50 + processor = MULTIMODAL_REGISTRY.create_processor(

51 + ctx.model_config,

52 + tokenizer=ctx.tokenizer,

53 + )

54 +

55 info = processor.info

56

57 seen_aspect_ratios = set[float]()

@@ -142,7 +151,14 @@ def test_processor_prompt_replacements_regression(model_id, num_imgs):

151 mm_processor_kwargs=None,

152 limit_mm_per_prompt={"image": num_imgs},

153 )

145 - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)

154 +

155 + # Avoid tokenizer already borrowed error

156 + maybe_make_thread_pool(ctx.tokenizer)

157 +

158 + processor = MULTIMODAL_REGISTRY.create_processor(

159 + ctx.model_config,

160 + tokenizer=ctx.tokenizer,

161 + )

162

163 image_ratios = [

164 (171, 152),

tests/models/multimodal/processing/test_moss_audio.py

+1 -3

@@ -35,12 +35,10 @@ from vllm.sequence import IntermediateTensors

35

36

37 class _Tokenizer:

38 - def encode(self, text, add_special_tokens=False):

39 - del add_special_tokens

38 + def encode(self, text, **kwargs):

39 return [ord(char) for char in text]

40

41 def decode(self, token_ids, **kwargs):

43 - del kwargs

42 return "".join(chr(token_id) for token_id in token_ids)

43

44 def batch_decode(self, batch_token_ids, **kwargs):

tests/models/multimodal/processing/test_openvla.py

-5

@@ -185,11 +185,6 @@ def test_openvla_prompt_update_inserts_image_tokens_after_bos() -> None:

185 image = Image.new("RGB", (640, 480), color=(255, 255, 255))

186 mm_items = MultiModalDataItems({"image": ImageProcessorItems([image])})

187

188 - assert (

189 - processor._hf_processor_applies_updates("In: test\nOut:", mm_items, {}, {})

190 - is False

191 - )

192 -

188 prompt_update = processor._get_prompt_updates(mm_items, {}, {})[0]

189 resolved = prompt_update.resolve(0)

190 content = resolved.content

tests/models/multimodal/processing/test_tensor_schema.py

+3 -3

@@ -37,7 +37,7 @@ from vllm.utils.torch_utils import set_default_torch_dtype

37 from ....utils import create_new_process_for_each_test

38 from ...registry import HF_EXAMPLE_MODELS

39 from ...utils import dummy_hf_overrides

40 -from .test_common import get_model_ids_to_test, get_text_token_prompts

40 +from .test_common import get_model_ids_to_test, get_token_prompt

41

42 ImageInput = list[Image.Image]

43 VideoInput: TypeAlias = (

@@ -107,10 +107,10 @@ def create_batched_mm_kwargs(

107 }

108

109 # video metadata will be added back to the resized video data here.

110 - text_prompt, token_prompt = get_text_token_prompts(processor, resized_mm_data)

110 + token_prompt = get_token_prompt(processor, resized_mm_data)

111

112 mm_kwargs = processor(

113 - prompt=token_prompt if text_prompt is None else text_prompt,

113 + prompt=token_prompt,

114 mm_items=processor.info.parse_mm_data(resized_mm_data),

115 hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,

116 )["mm_kwargs"].require_data()

vllm/model_executor/models/audioflamingo3.py

-2

@@ -382,7 +382,6 @@ class AudioFlamingo3MultiModalProcessor(

382 prompt: str,

383 mm_data: Mapping[str, object],

384 mm_kwargs: Mapping[str, Any],

385 - tok_kwargs: Mapping[str, object],

385 ) -> BatchFeature:

386 processor_mm_data = dict(mm_data)

387 audios = processor_mm_data.pop("audios", None)

@@ -393,7 +392,6 @@ class AudioFlamingo3MultiModalProcessor(

392 prompt=prompt,

393 mm_data=processor_mm_data,

394 mm_kwargs=mm_kwargs,

396 - tok_kwargs=tok_kwargs,

395 )

396

397 if "input_features_mask" in outputs:

vllm/model_executor/models/bagel.py

-9

@@ -273,15 +273,6 @@ class BagelDummyInputsBuilder(BaseDummyInputsBuilder[BagelProcessingInfo]):

273 class BagelMultiModalProcessor(BaseMultiModalProcessor[BagelProcessingInfo]):

274 """Multimodal processor for BAGEL model."""

275

276 - def _hf_processor_applies_updates(

277 - self,

278 - prompt_text: str,

279 - mm_items: MultiModalDataItems,

280 - hf_processor_mm_kwargs: Mapping[str, object],

281 - tokenization_kwargs: Mapping[str, object],

282 - ) -> bool:

283 - return False

284 -

276 def _get_prompt_updates(

277 self,

278 mm_items: MultiModalDataItems,

vllm/model_executor/models/blip2.py

-2

@@ -472,7 +472,6 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):

472 prompt: str,

473 mm_data: Mapping[str, object],

474 mm_kwargs: Mapping[str, object],

475 - tok_kwargs: Mapping[str, object],

475 ) -> BatchFeature:

476 if not mm_data:

477 # HF processor always adds placeholders even when there's no image

@@ -484,7 +483,6 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):

483 prompt=prompt,

484 mm_data=mm_data,

485 mm_kwargs=mm_kwargs,

487 - tok_kwargs=tok_kwargs,

486 )

487

488 def _get_mm_fields_config(

vllm/model_executor/models/chameleon.py

-2

@@ -140,7 +140,6 @@ class ChameleonMultiModalProcessor(BaseMultiModalProcessor[ChameleonProcessingIn

140 prompt: str,

141 mm_data: Mapping[str, object],

142 mm_kwargs: Mapping[str, object],

143 - tok_kwargs: Mapping[str, object],

143 ) -> BatchFeature:

144 if not mm_data:

145 prompt_ids = self.info.get_tokenizer().encode(prompt)

@@ -151,7 +150,6 @@ class ChameleonMultiModalProcessor(BaseMultiModalProcessor[ChameleonProcessingIn

150 prompt=prompt,

151 mm_data=mm_data,

152 mm_kwargs=mm_kwargs,

154 - tok_kwargs=tok_kwargs,

153 )

154

155 def _apply_hf_processor_tokens_only(

vllm/model_executor/models/cheers.py

+1 -11

@@ -497,18 +497,8 @@ class CheersMultiModalProcessor(BaseMultiModalProcessor[CheersProcessingInfo]):

497 prompt: str,

498 mm_data: Mapping[str, object],

499 mm_kwargs: Mapping[str, object],

500 - tok_kwargs: Mapping[str, object],

500 ) -> BatchFeature:

502 - return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)

503 -

504 - def _hf_processor_applies_updates(

505 - self,

506 - prompt_text: str,

507 - mm_items: MultiModalDataItems,

508 - hf_processor_mm_kwargs: Mapping[str, object],

509 - tokenization_kwargs: Mapping[str, object],

510 - ) -> bool:

511 - return False

501 + return super()._call_hf_processor(prompt, mm_data, mm_kwargs)

502

503 def _get_prompt_updates(

504 self,

vllm/model_executor/models/clip.py

+7 -30

@@ -208,40 +208,17 @@ class CLIPMultiModalProcessor(BaseMultiModalProcessor[CLIPProcessingInfo]):

208 timing_ctx: TimingContext,

209 ) -> MultiModalInput:

210 if inputs.mm_data_items:

211 - if isinstance(inputs.prompt, str):

212 - if len(inputs.prompt) > 0:

213 - raise ValueError(

214 - "CLIP accepts text-only or image-only inputs, not both! "

215 - "You must pass an image with an empty text prompt."

216 - )

211 + special_tokens = self.info.get_tokenizer().all_special_ids

212 + if all(tok in special_tokens for tok in inputs.prompt):

213 + inputs.prompt = []

214 else:

218 - special_tokens = self.info.get_tokenizer().all_special_ids

219 - if all(tok in special_tokens for tok in inputs.prompt):

220 - inputs.prompt = []

221 - else:

222 - raise ValueError(

223 - "CLIP accepts text-only or image-only inputs, not both! "

224 - "You must pass an image with an empty token prompt."

225 - )

226 -

227 - # For multi-modal data, the prompt after processing should

228 - # only contain the dummy image tokens

229 - inputs.tokenization_kwargs = {

230 - **inputs.tokenization_kwargs,

231 - "add_special_tokens": False,

232 - }

215 + raise ValueError(

216 + "CLIP accepts text-only or image-only inputs, not both! "

217 + "You must pass an image with an empty token prompt."

218 + )

219

220 return super().apply(inputs, timing_ctx)

221

236 - def _hf_processor_applies_updates(

237 - self,

238 - prompt_text: str,

239 - mm_items: MultiModalDataItems,

240 - hf_processor_mm_kwargs: Mapping[str, object],

241 - tokenization_kwargs: Mapping[str, object],

242 - ) -> bool:

243 - return False

244 -

222 def _get_mm_fields_config(

223 self,

224 hf_inputs: BatchFeature,

vllm/model_executor/models/cohere2_vision.py

-2

@@ -227,13 +227,11 @@ class Cohere2VisionMultiModalProcessor(

227 prompt: str,

228 mm_data: Mapping[str, object],

229 mm_kwargs: Mapping[str, object],

230 - tok_kwargs: Mapping[str, object],

230 ) -> BatchFeature:

231 processed_outputs = super()._call_hf_processor(

232 prompt,

233 mm_data,

234 mm_kwargs,

236 - tok_kwargs,

235 )

236

237 # Ensure num_patches is available for proper tensor splitting

vllm/model_executor/models/cohere_asr.py

+2 -4

@@ -1936,9 +1936,9 @@ class CohereASRMultiModalProcessor(EncDecMultiModalProcessor[CohereASRProcessing

1936

1937 def create_encoder_prompt(

1938 self,

1939 - prompt: str | list[int],

1939 + prompt: list[int],

1940 mm_items: MultiModalDataItems,

1941 - ) -> str | list[int]:

1941 + ) -> list[int]:

1942 return [0]

1943

1944 def _call_hf_processor(

@@ -1946,7 +1946,6 @@ class CohereASRMultiModalProcessor(EncDecMultiModalProcessor[CohereASRProcessing

1946 prompt: str,

1947 mm_data: Mapping[str, object],

1948 mm_kwargs: Mapping[str, object],

1949 - tok_kwargs: Mapping[str, object],

1949 ):

1950 if mm_data:

1951 feature_extractor = self.info.get_feature_extractor(**mm_kwargs)

@@ -1959,7 +1958,6 @@ class CohereASRMultiModalProcessor(EncDecMultiModalProcessor[CohereASRProcessing

1958 prompt=prompt,

1959 mm_data=mm_data,

1960 mm_kwargs=mm_kwargs,

1962 - tok_kwargs=tok_kwargs,

1961 )

1962 if "labels" in processed_outputs:

1963 processed_outputs["input_ids"] = processed_outputs.pop("labels")

vllm/model_executor/models/colmodernvbert.py

-11

@@ -159,14 +159,12 @@ class ColModernVBertMultiModalProcessor(

159 prompt: str,

160 mm_data: Mapping[str, object],

161 mm_kwargs: Mapping[str, object],

162 - tok_kwargs: Mapping[str, object],

162 ) -> BatchFeature:

163 tokenizer = self.info.get_tokenizer()

164 assert isinstance(tokenizer, HfTokenizer)

165 text_encoding = tokenizer(

166 prompt,

167 return_tensors="pt",

169 - **tok_kwargs,

168 )

169 result = BatchFeature(data=dict(text_encoding))

170

@@ -187,15 +185,6 @@ class ColModernVBertMultiModalProcessor(

185

186 return result

187

190 - def _hf_processor_applies_updates(

191 - self,

192 - prompt_text: str,

193 - mm_items: MultiModalDataItems,

194 - hf_processor_mm_kwargs: Mapping[str, object],

195 - tokenization_kwargs: Mapping[str, object],

196 - ) -> bool:

197 - return False

198 -

188 def _get_mm_fields_config(

189 self,

190 hf_inputs: BatchFeature,

vllm/model_executor/models/colpali.py

+6 -8

@@ -64,7 +64,6 @@ class ColPaliMultiModalProcessor(PaliGemmaMultiModalProcessor):

64 prompt: str,

65 mm_data: Mapping[str, object],

66 mm_kwargs: Mapping[str, object],

67 - tok_kwargs: Mapping[str, object],

67 ) -> BatchFeature:

68 if mm_data:

69 # The ColPali tokenizer_config.json ships with a small default

@@ -72,13 +71,12 @@ class ColPaliMultiModalProcessor(PaliGemmaMultiModalProcessor):

71 # by PaliGemmaProcessor, causing a token-count mismatch.

72 # vLLM enforces its own max_model_len, so we disable HF

73 # truncation to keep all image + text tokens intact.

75 - tok_kwargs = dict(tok_kwargs, truncation=False)

76 - return super()._call_hf_processor(

77 - prompt=prompt,

78 - mm_data=mm_data,

79 - mm_kwargs=mm_kwargs,

80 - tok_kwargs=tok_kwargs,

81 - )

74 + return self.info.ctx.call_hf_processor(

75 + self.info.get_hf_processor(**mm_kwargs),

76 + dict(text=prompt, **mm_data),

77 + dict(**mm_kwargs, truncation=False),

78 + )

79 + return super()._call_hf_processor(prompt, mm_data, mm_kwargs)

80

81

82 @default_pooling_type(seq_pooling_type="CLS", tok_pooling_type="ALL")

vllm/model_executor/models/deepseek_ocr.py

-1

@@ -292,7 +292,6 @@ class DeepseekOCRMultiModalProcessor(

292 prompt: str,

293 mm_data: Mapping[str, object],

294 mm_kwargs: Mapping[str, object],

295 - tok_kwargs: Mapping[str, object],

295 ) -> BatchFeature:

296 if mm_data:

297 processed_outputs = self.info.ctx.call_hf_processor(

vllm/model_executor/models/deepseek_ocr2.py

-1

@@ -163,7 +163,6 @@ class DeepseekOCR2MultiModalProcessor(

163 prompt: str,

164 mm_data: Mapping[str, object],

165 mm_kwargs: Mapping[str, object],

166 - tok_kwargs: Mapping[str, object],

166 ) -> BatchFeature:

167 if mm_data:

168 processed_outputs = self.info.ctx.call_hf_processor(

vllm/model_executor/models/deepseek_vl2.py

-2

@@ -241,7 +241,6 @@ class DeepseekVL2MultiModalProcessor(

241 prompt: str,

242 mm_data: Mapping[str, object],

243 mm_kwargs: Mapping[str, object],

244 - tok_kwargs: Mapping[str, object],

244 ) -> BatchFeature:

245 if not mm_data:

246 tokenizer = self.info.get_tokenizer()

@@ -252,7 +251,6 @@ class DeepseekVL2MultiModalProcessor(

251 prompt=prompt,

252 mm_data=mm_data,

253 mm_kwargs=mm_kwargs,

255 - tok_kwargs=tok_kwargs,

254 )

255

256 processed_outputs["num_patches"] = (

vllm/model_executor/models/ernie45_vl.py

+1 -2

@@ -1115,7 +1115,6 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin

1115 prompt: str,

1116 mm_data: Mapping[str, object],

1117 mm_kwargs: Mapping[str, object],

1118 - tok_kwargs: Mapping[str, object],

1118 ) -> BatchFeature:

1119 # when the prompt is not empty but the multimodal data is empty,

1120 # directly invoke the tokenizer.

@@ -1151,7 +1150,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin

1150 processor_output = self.info.ctx.call_hf_processor(

1151 hf_processor,

1152 dict(text=[prompt], images=mm_data["images"], videos=mm_data["videos"]),

1154 - dict(**mm_kwargs, **tok_kwargs),

1153 + mm_kwargs,

1154 )

1155

1156 # Divide the processor_output into two modalities: image and video.

vllm/model_executor/models/fireredasr2.py

-2

@@ -233,7 +233,6 @@ class FireRedASR2MultiModalProcessor(

233 prompt: str,

234 mm_data: Mapping[str, object],

235 mm_kwargs: Mapping[str, object],

236 - tok_kwargs: Mapping[str, object],

236 ) -> BatchFeature:

237 if mm_data:

238 feature_extractor = self.info.get_feature_extractor(**mm_kwargs)

@@ -246,7 +245,6 @@ class FireRedASR2MultiModalProcessor(

245 prompt=prompt,

246 mm_data=mm_data,

247 mm_kwargs=mm_kwargs,

249 - tok_kwargs=tok_kwargs,

248 )

249 if "labels" in processed_outputs:

250 processed_outputs["input_ids"] = processed_outputs.pop("labels")

vllm/model_executor/models/fireredlid.py

+2 -4

@@ -445,9 +445,9 @@ class FireRedLIDMultiModalProcessor(

445 ):

446 def create_encoder_prompt(

447 self,

448 - prompt: str | list[int],

448 + prompt: list[int],

449 mm_items: MultiModalDataItems,

450 - ) -> str | list[int]:

450 + ) -> list[int]:

451 # Dummy encoder prompt for profiling (encoder only processes audio).

452 return [0]

453

@@ -456,7 +456,6 @@ class FireRedLIDMultiModalProcessor(

456 prompt: str,

457 mm_data: Mapping[str, object],

458 mm_kwargs: Mapping[str, object],

459 - tok_kwargs: Mapping[str, object],

459 ) -> BatchFeature:

460 if mm_data:

461 feature_extractor = self.info.get_feature_extractor(**mm_kwargs)

@@ -469,7 +468,6 @@ class FireRedLIDMultiModalProcessor(

468 prompt=prompt,

469 mm_data=mm_data,

470 mm_kwargs=mm_kwargs,

472 - tok_kwargs=tok_kwargs,

471 )

472 if "labels" in processed_outputs:

473 processed_outputs["input_ids"] = processed_outputs.pop("labels")

vllm/model_executor/models/funasr.py

-2

@@ -752,7 +752,6 @@ class FunASRMultiModalProcessor(BaseMultiModalProcessor[FunASRProcessingInfo]):

752 prompt: str,

753 mm_data: Mapping[str, object],

754 mm_kwargs: Mapping[str, object],

755 - tok_kwargs: Mapping[str, object],

755 ) -> BatchFeature:

756 if mm_data:

757 feature_extractor = self.info.get_feature_extractor(**mm_kwargs)

@@ -765,7 +764,6 @@ class FunASRMultiModalProcessor(BaseMultiModalProcessor[FunASRProcessingInfo]):

764 prompt=prompt,

765 mm_data=mm_data,

766 mm_kwargs=mm_kwargs,

768 - tok_kwargs=tok_kwargs,

767 )

768 if "labels" in processed_outputs:

769 processed_outputs["input_ids"] = processed_outputs.pop("labels")

vllm/model_executor/models/funaudiochat.py

+1 -11

@@ -617,10 +617,9 @@ class FunAudioChatMultiModalProcessor(

617 prompt: str,

618 mm_data: Mapping[str, object],

619 mm_kwargs: Mapping[str, object],

620 - tok_kwargs: Mapping[str, object],

620 ) -> BatchFeature:

621 tokenizer = self.info.get_tokenizer()

623 - input_ids = torch.tensor([tokenizer.encode(prompt, **tok_kwargs)])

622 + input_ids = torch.tensor([tokenizer.encode(prompt)])

623

624 audios = mm_data.get("audios", [])

625 if not audios:

@@ -679,15 +678,6 @@ class FunAudioChatMultiModalProcessor(

678

679 return BatchFeature({"input_ids": input_ids, **mm_inputs})

680

682 - def _hf_processor_applies_updates(

683 - self,

684 - prompt_text: str,

685 - mm_items: MultiModalDataItems,

686 - hf_processor_mm_kwargs: Mapping[str, object],

687 - tokenization_kwargs: Mapping[str, object],

688 - ) -> bool:

689 - return False

690 -

681 def _get_mm_fields_config(

682 self,

683 hf_inputs: BatchFeature,

vllm/model_executor/models/gemma3_mm.py

-2

@@ -266,13 +266,11 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):

266 prompt: str,

267 mm_data: Mapping[str, object],

268 mm_kwargs: Mapping[str, object],

269 - tok_kwargs: Mapping[str, object],

269 ) -> BatchFeature:

270 processed_outputs = super()._call_hf_processor(

271 prompt,

272 mm_data,

273 mm_kwargs,

275 - tok_kwargs,

274 )

275

276 # HF processor pops the `num_crops` kwarg, which is needed by vLLM

vllm/model_executor/models/gemma3n_mm.py

-2

@@ -262,7 +262,6 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])

262 prompt: str,

263 mm_data: Mapping[str, object],

264 mm_kwargs: Mapping[str, object],

265 - tok_kwargs: Mapping[str, object],

265 ) -> BatchFeature:

266 # HF Transformers audio processor no longer accepts `audios` key.

267 # We pop `audios` and replace it with `audio` key to suppress

@@ -273,7 +272,6 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])

272 prompt,

273 mm_data,

274 mm_kwargs,

276 - tok_kwargs,

275 )

276

277 if "input_features" in processed_outputs:

vllm/model_executor/models/gemma4_mm.py

-22

@@ -560,31 +560,11 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):

560

561

562 class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):

563 - def _apply_hf_processor_text_only(

564 - self,

565 - prompt_text: str,

566 - tokenization_kwargs: Mapping[str, object],

567 - ) -> list[int]:

568 - # Bypass the HF processor and tokenize directly. The HF

569 - # processor expands multimodal placeholders (<|video|>, etc.)

570 - # via get_text_with_replacements, which raises StopIteration

571 - # when the prompt contains placeholders without matching data.

572 - # The text-only path only needs token IDs, so the tokenizer

573 - # alone is sufficient.

574 - processor = self.info.get_hf_processor()

575 - text_inputs = processor.tokenizer([prompt_text], **tokenization_kwargs)

576 - input_ids = text_inputs["input_ids"]

577 - if not isinstance(input_ids, list):

578 - input_ids = input_ids.tolist()

579 - (prompt_ids,) = input_ids

580 - return prompt_ids

581 -

563 def _call_hf_processor(

564 self,

565 prompt: str,

566 mm_data: Mapping[str, object],

567 mm_kwargs: Mapping[str, object],

587 - tok_kwargs: Mapping[str, object],

568 ) -> BatchFeature:

569 merged_kwargs = self.info.ctx.get_merged_mm_kwargs(mm_kwargs)

570 val, is_top_level_max_soft_tokens = _get_max_soft_tokens(merged_kwargs)

@@ -640,7 +620,6 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):

620 prompt=dummy_prompt,

621 mm_data={"images": frames},

622 mm_kwargs=video_mm_kwargs,

643 - tok_kwargs=tok_kwargs,

623 )

624

625 # Remap HF key name

@@ -741,7 +720,6 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):

720 prompt,

721 mm_data,

722 patched_mm_kwargs,

744 - tok_kwargs,

723 )

724

725 # HF uses 'image_position_ids'; vLLM uses 'pixel_position_ids'.

vllm/model_executor/models/glm4_1v.py

-4

@@ -1581,7 +1581,6 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):

1581 prompt: str,

1582 mm_data: Mapping[str, object],

1583 mm_kwargs: Mapping[str, object],

1584 - tok_kwargs: Mapping[str, object],

1584 ) -> BatchFeature:

1585 mm_data = dict(mm_data)

1586 if not mm_data:

@@ -1602,7 +1601,6 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):

1601 prompt=prompt,

1602 mm_data=prepared_data,

1603 mm_kwargs=prepared_kwargs,

1605 - tok_kwargs=tok_kwargs,

1604 )

1605

1606 if (

@@ -1631,7 +1629,6 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):

1629 prompt="<|begin_of_video|><|video|><|end_of_video|>",

1630 mm_data=video_mm_data,

1631 mm_kwargs=video_mm_kwargs,

1634 - tok_kwargs=tok_kwargs,

1632 )

1633 input_ids = video_outputs.pop("input_ids")

1634 if swap_video_frame_tokens:

@@ -1659,7 +1656,6 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):

1656 prompt=prompt,

1657 mm_data=mm_data,

1658 mm_kwargs=mm_kwargs,

1662 - tok_kwargs=tok_kwargs,

1659 )

1660 if swap_video_frame_tokens:

1661 input_ids = processed_outputs["input_ids"]

vllm/model_executor/models/glm4v.py

-9

@@ -466,15 +466,6 @@ class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]):

466

467

468 class GLM4VMultiModalProcessor(BaseMultiModalProcessor[GLM4VProcessingInfo]):

469 - def _hf_processor_applies_updates(

470 - self,

471 - prompt_text: str,

472 - mm_items: MultiModalDataItems,

473 - hf_processor_mm_kwargs: Mapping[str, object],

474 - tokenization_kwargs: Mapping[str, object],

475 - ) -> bool:

476 - return False

477 -

469 def _get_mm_fields_config(

470 self,

471 hf_inputs: BatchFeature,

vllm/model_executor/models/glmasr.py

-2

@@ -755,7 +755,6 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])

755 prompt: str,

756 mm_data: dict[str, object],

757 mm_kwargs: Mapping[str, Any],

758 - tok_kwargs: Mapping[str, object],

758 ) -> BatchFeature:

759 # Normalize input: handle deprecated key and list conversion.

760 if "audios" in mm_data:

@@ -782,7 +781,6 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])

781 prompt=prompt,

782 mm_data=mm_data,

783 mm_kwargs=mm_kwargs,

785 - tok_kwargs=tok_kwargs,

784 )

785

786 # Postprocess: rename mask and add chunk counts

vllm/model_executor/models/granite_speech.py

-2

@@ -185,7 +185,6 @@ class GraniteSpeechMultiModalProcessor(

185 prompt: str,

186 mm_data: Mapping[str, object],

187 mm_kwargs: Mapping[str, object],

188 - tok_kwargs: Mapping[str, object],

188 ) -> BatchFeature:

189 mm_data = dict(mm_data)

190 audios = mm_data.pop("audios", [])

@@ -198,7 +197,6 @@ class GraniteSpeechMultiModalProcessor(

197 prompt=prompt,

198 mm_data=mm_data,

199 mm_kwargs=mm_kwargs,

201 - tok_kwargs=tok_kwargs,

200 )

201

202 if "audio" in mm_data:

vllm/model_executor/models/hunyuan_vision.py

+1 -2

@@ -736,7 +736,6 @@ class HunYuanVLMultiModalProcessor(BaseMultiModalProcessor[HunYuanVLProcessingIn

736 prompt: str,

737 mm_data: Mapping[str, object],

738 mm_kwargs: Mapping[str, object],

739 - tok_kwargs: Mapping[str, object],

739 ) -> BatchFeature:

740 hf_processor = self.info.get_hf_processor(**mm_kwargs)

741 # HunYuanVLProcessor requires image placeholders wrapped with start/end tokens.

@@ -751,7 +750,7 @@ class HunYuanVLMultiModalProcessor(BaseMultiModalProcessor[HunYuanVLProcessingIn

750 return self.info.ctx.call_hf_processor(

751 hf_processor,

752 dict(text=prompt, **mm_data),

754 - dict(**mm_kwargs, **tok_kwargs),

753 + mm_kwargs,

754 )

755

756 def _get_prompt_updates(

vllm/model_executor/models/hyperclovax_vision.py

-10

@@ -195,7 +195,6 @@ class HCXVisionMultiModalProcessor(BaseMultiModalProcessor[HCXVisionProcessingIn

195 prompt: str,

196 mm_data: Mapping[str, object],

197 mm_kwargs: Mapping[str, object],

198 - tok_kwargs: Mapping[str, object],

198 ) -> BatchFeature:

199 for video_idx, video_arr in enumerate(mm_data.get("videos", [])):

200 if video_arr.dtype != np.uint8:

@@ -263,15 +262,6 @@ class HCXVisionMultiModalProcessor(BaseMultiModalProcessor[HCXVisionProcessingIn

262

263 return processed_outputs

264

266 - def _hf_processor_applies_updates(

267 - self,

268 - prompt_text: str,

269 - mm_items: MultiModalDataItems,

270 - hf_processor_mm_kwargs: Mapping[str, object],

271 - tokenization_kwargs: Mapping[str, object],

272 - ) -> bool:

273 - return False

274 -

265 def _get_prompt_updates(

266 self,

267 mm_items: MultiModalDataItems,

vllm/model_executor/models/hyperclovax_vision_v2.py

+9 -23

@@ -209,11 +209,16 @@ class HCXVisionV2DummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionV2Processing

209 )

210 dummy_mm_items = self.info.parse_mm_data(dummy_mm_data, validate=False)

211

212 + tokenizer = self.info.get_tokenizer()

213 + prompt = tokenizer.encode(

214 + prompt_text,

215 + **self.info.default_tok_params.get_encode_kwargs(),

216 + )

217 +

218 return ProcessorInputs(

213 - prompt=prompt_text,

219 + prompt=prompt,

220 mm_data_items=dummy_mm_items,

221 hf_processor_mm_kwargs=mm_processor_kwargs or {},

216 - tokenization_kwargs={"truncation": False},

222 )

223

224 def get_dummy_mm_data(

@@ -261,7 +266,6 @@ class HCXVisionV2MultiModalProcessor(

266 prompt: str,

267 mm_data: Mapping[str, object],

268 mm_kwargs: Mapping[str, object],

264 - tok_kwargs: Mapping[str, object],

269 ) -> BatchFeature:

270 images = mm_data.get("images")

271 videos = mm_data.get("videos")

@@ -271,8 +275,7 @@ class HCXVisionV2MultiModalProcessor(

275

276 # Build data dict for HF processor (images/videos only)

277 # NOTE: We pass the prompt as-is without token normalization.

274 - # Token expansion is handled by vLLM via _get_prompt_updates since

275 - # _hf_processor_applies_updates returns False.

278 + # Token expansion is handled by vLLM via _get_prompt_updates.

279 data: dict[str, object] = dict(

280 text=prompt,

281 images=images,

@@ -282,28 +285,11 @@ class HCXVisionV2MultiModalProcessor(

285 processed_outputs = self.info.ctx.call_hf_processor(

286 hf_processor=hf_processor,

287 data=data,

285 - kwargs=dict(**mm_kwargs, **tok_kwargs),

288 + kwargs=mm_kwargs,

289 )

290

291 return processed_outputs

292

290 - def _hf_processor_applies_updates(

291 - self,

292 - prompt_text: str,

293 - mm_items: MultiModalDataItems,

294 - hf_processor_mm_kwargs: Mapping[str, object],

295 - tokenization_kwargs: Mapping[str, object],

296 - ) -> bool:

297 - # Match BaseMultiModalProcessor behavior:

298 - # - raw multimodal inputs: HF processor applies updates

299 - # - embedding inputs: vLLM applies updates

300 - return super()._hf_processor_applies_updates(

301 - prompt_text,

302 - mm_items,

303 - hf_processor_mm_kwargs,

304 - tokenization_kwargs,

305 - )

306 -

293 def _get_prompt_updates(

294 self,

295 mm_items: MultiModalDataItems,

vllm/model_executor/models/idefics3.py

-2

@@ -244,7 +244,6 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo

244 prompt: str,

245 mm_data: Mapping[str, object],

246 mm_kwargs: Mapping[str, object],

247 - tok_kwargs: Mapping[str, object],

247 ) -> BatchFeature:

248 # Text-only input not supported in composite processor

249 if not (images := mm_data.get("images", [])):

@@ -260,7 +259,6 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo

259 prompt,

260 mm_data,

261 mm_kwargs,

263 - tok_kwargs,

262 )

263

264 mm_items = self.info.parse_mm_data({"image": images}, validate=False)

vllm/model_executor/models/interns1.py

+1 -4

@@ -337,7 +337,6 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo

337 prompt: str,

338 mm_data: Mapping[str, object],

339 mm_kwargs: Mapping[str, object],

340 - tok_kwargs: Mapping[str, object],

340 ) -> BatchFeature:

341 mm_data = dict(mm_data)

342 videos = mm_data.pop("videos", [])

@@ -364,7 +363,6 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo

363 prompt=hf_processor.image_token,

364 mm_data={"images": image},

365 mm_kwargs=mm_kwargs,

367 - tok_kwargs=tok_kwargs,

366 )

367 image_pixel_values.append(processed_outputs.pop("pixel_values"))

368

@@ -387,7 +385,6 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo

385 prompt=hf_processor.video_token,

386 mm_data={"videos": video},

387 mm_kwargs=mm_kwargs,

390 - tok_kwargs=tok_kwargs,

388 )

389 video_pixel_values.append(processed_outputs.pop("pixel_values"))

390

@@ -406,7 +403,7 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo

403

404 prompt = re.sub("<image_placeholder>", hf_processor.image_token, prompt)

405 prompt = re.sub("<video_placeholder>", hf_processor.video_token, prompt)

409 - text_outputs = tokenizer(prompt, **tok_kwargs, return_tensors="pt")

406 + text_outputs = tokenizer(prompt, return_tensors="pt")

407

408 return BatchFeature({**text_outputs, **image_outputs, **video_outputs})

409

vllm/model_executor/models/internvl.py

+1 -6

@@ -225,13 +225,11 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):

225 prompt: str,

226 mm_data: Mapping[str, object],

227 mm_kwargs: Mapping[str, object],

228 - tok_kwargs: Mapping[str, object],

228 ) -> BatchFeature:

229 processed_outputs = super()._call_hf_processor(

230 prompt=prompt,

231 mm_data=mm_data,

232 mm_kwargs=mm_kwargs,

234 - tok_kwargs=tok_kwargs,

233 )

234

235 hf_processor = self.info.get_hf_processor(**mm_kwargs)

@@ -462,11 +460,8 @@ class InternVLMultiModalProcessor(

460 prompt: str,

461 mm_data: Mapping[str, object],

462 mm_kwargs: Mapping[str, object],

465 - tok_kwargs: Mapping[str, object],

463 ) -> BatchFeature:

467 - processed_outputs = super()._call_hf_processor(

468 - prompt, mm_data, mm_kwargs, tok_kwargs

469 - )

464 + processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)

465

466 hf_processor = self.info.get_hf_processor(**mm_kwargs)

467 if (video_token_id := hf_processor.ctx_video_token_id) is not None:

vllm/model_executor/models/jina_vl.py

+1 -2

@@ -59,7 +59,6 @@ class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor):

59 prompt: str,

60 mm_data: Mapping[str, object],

61 mm_kwargs: Mapping[str, object],

62 - tok_kwargs: Mapping[str, object],

62 ) -> BatchFeature:

63 # NOTE: We should reverse the order of the mm_data because the

64 # query prompt is placed after the document prompt in the score

@@ -67,7 +66,7 @@ class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor):

66 # stored in the opposite order (query first, then document).

67 for _, value in mm_data.items():

68 value.reverse()

70 - return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)

69 + return super()._call_hf_processor(prompt, mm_data, mm_kwargs)

70

71

72 @MULTIMODAL_REGISTRY.register_processor(

vllm/model_executor/models/kanana_v.py

-1

@@ -466,7 +466,6 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])

466 prompt: str,

467 mm_data: Mapping[str, object],

468 mm_kwargs: Mapping[str, object],

469 - tok_kwargs: Mapping[str, object],

469 ) -> BatchFeature:

470 """Run the underlying HF processor on text and image data."""

471 # Text-only input is handled as a special case here.

vllm/model_executor/models/keye.py

+1 -2

@@ -1167,11 +1167,10 @@ class KeyeMultiModalProcessor(BaseMultiModalProcessor[KeyeProcessingInfo]):

1167 prompt: str,

1168 mm_data: Mapping[str, object],

1169 mm_kwargs: Mapping[str, object],

1170 - tok_kwargs: Mapping[str, object],

1170 ) -> BatchFeature:

1171 # Override to use the text path instead of token path to use the

1172 # video-specific logic in processing_keye.py

1174 - return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)

1173 + return super()._call_hf_processor(prompt, mm_data, mm_kwargs)

1174

1175 def _get_prompt_updates(

1176 self,

vllm/model_executor/models/keye_vl1_5.py

+1 -2

@@ -383,11 +383,10 @@ class KeyeVL1_5MultiModalProcessor(BaseMultiModalProcessor[KeyeVL1_5ProcessingIn

383 prompt: str,

384 mm_data: Mapping[str, object],

385 mm_kwargs: Mapping[str, object],

386 - tok_kwargs: Mapping[str, object],

386 ) -> BatchFeature:

387 # Override to use the text path instead of token path to use the

388 # video-specific logic in processing_keye.py

390 - return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)

389 + return super()._call_hf_processor(prompt, mm_data, mm_kwargs)

390

391 def _get_prompt_updates(

392 self,

vllm/model_executor/models/kimi_audio.py

+1 -2

@@ -231,7 +231,6 @@ class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingIn

231 prompt: str,

232 mm_data: Mapping[str, object],

233 mm_kwargs: Mapping[str, object],

234 - tok_kwargs: Mapping[str, object],

234 ) -> BatchFeature:

235 """Call the HuggingFace processor."""

236 # Convert mm_data format: {'audios': [...]} -> {'audio': ...}

@@ -256,7 +255,7 @@ class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingIn

255 return self.info.ctx.call_hf_processor(

256 self.info.get_hf_processor(**mm_kwargs),

257 dict(text=prompt, **mm_data),

259 - dict(**mm_kwargs, **tok_kwargs),

258 + mm_kwargs,

259 )

260

261 def _get_mm_fields_config(

vllm/model_executor/models/kimi_k25.py

+1 -2

@@ -263,11 +263,10 @@ class KimiK25MultiModalProcessor(BaseMultiModalProcessor[KimiK25ProcessingInfo])

263 prompt: str,

264 mm_data: Mapping[str, object],

265 mm_kwargs: Mapping[str, object],

266 - tok_kwargs: Mapping[str, object],

266 ) -> BatchFeature:

267 # Override to use the text path instead of token path because vision chunk

268 # is not considered

270 - return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)

269 + return super()._call_hf_processor(prompt, mm_data, mm_kwargs)

270

271 def _get_prompt_updates(

272 self,

vllm/model_executor/models/lfm2_vl.py

-2

@@ -398,7 +398,6 @@ class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):

398 prompt: str,

399 mm_data: Mapping[str, object],

400 mm_kwargs: Mapping[str, object],

401 - tok_kwargs: Mapping[str, object],

401 ) -> BatchFeature:

402 # Text-only input not supported in composite processor

403 if not (images := mm_data.get("images", [])):

@@ -412,7 +411,6 @@ class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):

411 prompt,

412 mm_data,

413 mm_kwargs,

415 - tok_kwargs,

414 )

415

416 mm_items = self.info.parse_mm_data({"image": images}, validate=False)

vllm/model_executor/models/lightonocr.py

-2

@@ -44,13 +44,11 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingIn

44 prompt: str,

45 mm_data: Mapping[str, object],

46 mm_kwargs: Mapping[str, object],

47 - tok_kwargs: Mapping[str, object],

47 ) -> BatchFeature:

48 processed_outputs = super()._call_hf_processor(

49 prompt=prompt,

50 mm_data=mm_data,

51 mm_kwargs=mm_kwargs,

53 - tok_kwargs=tok_kwargs,

52 )

53

54 # NOTE: LightOnOCR does not use break/end tokens, so we remove them here.

vllm/model_executor/models/llava.py

-2

@@ -326,13 +326,11 @@ class PixtralHFMultiModalProcessor(BaseMultiModalProcessor[PixtralHFProcessingIn

326 prompt: str,

327 mm_data: Mapping[str, object],

328 mm_kwargs: Mapping[str, object],

329 - tok_kwargs: Mapping[str, object],

329 ) -> BatchFeature:

330 processed_outputs = super()._call_hf_processor(

331 prompt=prompt,

332 mm_data=mm_data,

333 mm_kwargs=mm_kwargs,

335 - tok_kwargs=tok_kwargs,

334 )

335

336 pixel_values = processed_outputs.get("pixel_values")

vllm/model_executor/models/llava_onevision.py

-21

@@ -326,7 +326,6 @@ class LlavaOnevisionMultiModalProcessor(

326 prompt: str,

327 mm_data: Mapping[str, object],

328 mm_kwargs: Mapping[str, object],

329 - tok_kwargs: Mapping[str, object],

329 ) -> BatchFeature:

330 mm_data = dict(mm_data)

331 videos = mm_data.pop("videos", [])

@@ -337,7 +336,6 @@ class LlavaOnevisionMultiModalProcessor(

336 prompt=prompt,

337 mm_data=mm_data,

338 mm_kwargs=mm_kwargs,

340 - tok_kwargs=tok_kwargs,

339 )

340

341 # LLaVA-OneVision processor doesn't support multiple videos

@@ -352,7 +350,6 @@ class LlavaOnevisionMultiModalProcessor(

350 prompt=prompt,

351 mm_data={},

352 mm_kwargs=mm_kwargs,

355 - tok_kwargs=tok_kwargs,

353 )

354

355 images = mm_data.pop("images", [])

@@ -362,7 +359,6 @@ class LlavaOnevisionMultiModalProcessor(

359 prompt=image_token * len(images),

360 mm_data={"images": images},

361 mm_kwargs=mm_kwargs,

365 - tok_kwargs=tok_kwargs,

362 )

363 image_outputs = {

364 k: v

@@ -378,7 +374,6 @@ class LlavaOnevisionMultiModalProcessor(

374 prompt=video_token,

375 mm_data={"videos": video},

376 mm_kwargs=mm_kwargs,

381 - tok_kwargs=tok_kwargs,

377 )

378

379 pixel_values_videos.append(item_outputs["pixel_values_videos"][0])

@@ -392,22 +387,6 @@ class LlavaOnevisionMultiModalProcessor(

387 )

388 return BatchFeature(combined_outputs)

389

395 - def _hf_processor_applies_updates(

396 - self,

397 - prompt_text: str,

398 - mm_items: MultiModalDataItems,

399 - hf_processor_mm_kwargs: Mapping[str, object],

400 - tokenization_kwargs: Mapping[str, object],

401 - ) -> bool:

402 - base_result = super()._hf_processor_applies_updates(

403 - prompt_text=prompt_text,

404 - mm_items=mm_items,

405 - hf_processor_mm_kwargs=hf_processor_mm_kwargs,

406 - tokenization_kwargs=tokenization_kwargs,

407 - )

408 -

409 - return base_result and mm_items.get_count("video", strict=False) == 0

410 -

390 def _get_prompt_updates(

391 self,

392 mm_items: MultiModalDataItems,

vllm/model_executor/models/llava_onevision2.py

+1 -7

@@ -1552,16 +1552,13 @@ class LlavaOnevision2MultiModalProcessor(

1552 prompt: str,

1553 mm_data: Mapping[str, object],

1554 mm_kwargs: Mapping[str, object],

1555 - tok_kwargs: Mapping[str, object],

1555 ) -> BatchFeature:

1556 # The wrapped OV2 processor is a bare custom class without the standard

1557 # ProcessorMixin ``_merge_kwargs`` machinery, so vLLM's default path

1558 # fails; overriding this method routes the base class to call us

1559 # directly.

1560 hf_processor = self.info.get_hf_processor(**mm_kwargs)

1562 - merged_kwargs = self.info.ctx.get_merged_mm_kwargs(

1563 - dict(**mm_kwargs, **tok_kwargs)

1564 - )

1561 + merged_kwargs = self.info.ctx.get_merged_mm_kwargs(mm_kwargs)

1562 merged_kwargs.setdefault("return_tensors", "pt")

1563 call_kwargs = {

1564 k: v

@@ -1638,7 +1635,6 @@ class LlavaOnevision2MultiModalProcessor(

1635 prompt=prompt,

1636 mm_data=mm_data,

1637 mm_kwargs={**mm_kwargs, "video_backend": "codec"},

1641 - tok_kwargs=tok_kwargs,

1638 )

1639 data = dict(output)

1640 return BatchFeature(

@@ -1736,7 +1732,6 @@ class LlavaOnevision2MultiModalProcessor(

1732 prompt=new_prompt,

1733 mm_data=merged_mm_data,

1734 mm_kwargs=mm_kwargs,

1739 - tok_kwargs=tok_kwargs,

1735 )

1736 data = dict(output)

1737

@@ -1803,7 +1798,6 @@ class LlavaOnevision2MultiModalProcessor(

1798 prompt=prompt,

1799 mm_data=mm_data,

1800 mm_kwargs=mm_kwargs,

1806 - tok_kwargs=tok_kwargs,

1801 )

1802

1803 def _rename_codec_outputs_to_video(

vllm/model_executor/models/midashenglm.py

-2

@@ -588,7 +588,6 @@ class MiDashengLMMultiModalProcessor(

588 prompt: str,

589 mm_data: Mapping[str, object],

590 mm_kwargs: Mapping[str, Any],

591 - tok_kwargs: Mapping[str, object],

591 ) -> BatchFeature:

592 audios = mm_data.pop("audios", [])

593

@@ -622,7 +621,6 @@ class MiDashengLMMultiModalProcessor(

621 prompt=prompt,

622 mm_data=mm_data,

623 mm_kwargs=mm_kwargs,

625 - tok_kwargs=tok_kwargs,

624 )

625

626 def _get_mm_fields_config(

vllm/model_executor/models/mimo_v2_omni.py

+1 -2

@@ -917,7 +917,6 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing

917 prompt: str,

918 mm_data: Mapping[str, object],

919 mm_kwargs: Mapping[str, object],

920 - tok_kwargs: Mapping[str, object],

920 ) -> BatchFeature:

921 """Convert numpy video arrays to (TCHW, timestamps) tuples for MiMo.

922 Also remap 'audios' → 'audio' since MiMoOmniProcessor.__call__ uses

@@ -1000,7 +999,7 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing

999

1000 mm_data = {**mm_data, "videos": converted}

1001

1003 - return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)

1002 + return super()._call_hf_processor(prompt, mm_data, mm_kwargs)

1003

1004 def _get_prompt_updates(

1005 self,

vllm/model_executor/models/minicpmo.py

+2 -5

@@ -436,7 +436,6 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing

436 self,

437 mm_data: Mapping[str, object],

438 mm_kwargs: Mapping[str, object],

439 - tok_kwargs: Mapping[str, object],

439 ) -> Mapping[str, NestedTensors]:

440 if (audios := mm_data.get("audios")) is None:

441 return {}

@@ -453,7 +452,6 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing

452 prompts=[self.info.audio_pattern] * len(parsed_audios),

453 mm_data={"audios": [[audio] for audio in parsed_audios]},

454 mm_kwargs={**mm_kwargs, "chunk_input": True},

456 - tok_kwargs=tok_kwargs,

455 out_keys={"audio_features", "audio_feature_lens"},

456 )

457

@@ -486,11 +484,10 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing

484 self,

485 mm_data: Mapping[str, object],

486 mm_kwargs: Mapping[str, object],

489 - tok_kwargs: Mapping[str, object],

487 ) -> Mapping[str, NestedTensors]:

488 return {

492 - **super().process_mm_inputs(mm_data, mm_kwargs, tok_kwargs),

493 - **self.process_audios(mm_data, mm_kwargs, tok_kwargs),

489 + **super().process_mm_inputs(mm_data, mm_kwargs),

490 + **self.process_audios(mm_data, mm_kwargs),

491 }

492

493 def _get_prompt_updates(

vllm/model_executor/models/minicpmv.py

+4 -22

@@ -869,7 +869,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

869 self,

870 mm_data: Mapping[str, object],

871 mm_kwargs: Mapping[str, object],

872 - tok_kwargs: Mapping[str, object],

872 ) -> Mapping[str, NestedTensors]:

873 if (images := mm_data.get("images")) is None:

874 return {}

@@ -886,7 +885,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

885 prompts=[self.info.image_pattern] * len(parsed_images),

886 mm_data={"images": [[image] for image in parsed_images]},

887 mm_kwargs=mm_kwargs,

889 - tok_kwargs=tok_kwargs,

888 out_keys={"pixel_values", "image_sizes", "tgt_sizes"},

889 )

890

@@ -896,7 +894,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

894 self,

895 mm_data: Mapping[str, object],

896 mm_kwargs: Mapping[str, object],

899 - tok_kwargs: Mapping[str, object],

897 ) -> Mapping[str, NestedTensors]:

898 if (videos := mm_data.get("videos")) is None:

899 return {}

@@ -918,7 +915,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

915 **mm_kwargs,

916 "max_slice_nums": self.info.get_video_max_slice_num(),

917 },

921 - tok_kwargs=tok_kwargs,

918 out_keys={"pixel_values", "image_sizes", "tgt_sizes"},

919 )

920

@@ -930,11 +926,10 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

926 self,

927 mm_data: Mapping[str, object],

928 mm_kwargs: Mapping[str, object],

933 - tok_kwargs: Mapping[str, object],

929 ) -> Mapping[str, NestedTensors]:

930 return {

936 - **self.process_images(mm_data, mm_kwargs, tok_kwargs),

937 - **self.process_videos(mm_data, mm_kwargs, tok_kwargs),

931 + **self.process_images(mm_data, mm_kwargs),

932 + **self.process_videos(mm_data, mm_kwargs),

933 }

934

935 def _apply_prompt_updates(

@@ -993,7 +988,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

988 prompts: list[str],

989 mm_data: Mapping[str, Sequence[object]],

990 mm_kwargs: Mapping[str, object],

996 - tok_kwargs: Mapping[str, object],

991 *,

992 out_keys: set[str],

993 ) -> dict[str, NestedTensors]:

@@ -1003,7 +997,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

997 prompt=prompts, # type: ignore

998 mm_data=mm_data,

999 mm_kwargs=mm_kwargs,

1006 - tok_kwargs=tok_kwargs,

1000 )

1001 else:

1002 inputs = defaultdict[str, list[torch.Tensor]](list)

@@ -1013,7 +1006,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

1006 prompt=prompt,

1007 mm_data={k: v[i] for k, v in mm_data.items()},

1008 mm_kwargs=mm_kwargs,

1016 - tok_kwargs=tok_kwargs,

1009 )

1010

1011 for k, v in inputs_one.items():

@@ -1027,12 +1019,11 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

1019 prompt: str,

1020 mm_data: Mapping[str, object],

1021 mm_kwargs: Mapping[str, object],

1030 - tok_kwargs: Mapping[str, object],

1022 ) -> BatchFeature:

1023 tokenizer = self.info.get_tokenizer()

1024

1034 - input_ids = torch.tensor([tokenizer.encode(prompt, **tok_kwargs)])

1035 - mm_inputs = self.process_mm_inputs(mm_data, mm_kwargs, tok_kwargs)

1025 + input_ids = torch.tensor([tokenizer.encode(prompt)])

1026 + mm_inputs = self.process_mm_inputs(mm_data, mm_kwargs)

1027

1028 return BatchFeature(

1029 {

@@ -1041,15 +1032,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

1032 }

1033 )

1034

1044 - def _hf_processor_applies_updates(

1045 - self,

1046 - prompt_text: str,

1047 - mm_items: MultiModalDataItems,

1048 - hf_processor_mm_kwargs: Mapping[str, object],

1049 - tokenization_kwargs: Mapping[str, object],

1050 - ) -> bool:

1051 - return False

1052 -

1035 def _get_prompt_updates(

1036 self,

1037 mm_items: MultiModalDataItems,

vllm/model_executor/models/minicpmv4_6.py

-2

@@ -154,7 +154,6 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):

154 self,

155 mm_data: Mapping[str, object],

156 mm_kwargs: Mapping[str, object],

157 - tok_kwargs: Mapping[str, object],

157 ) -> Mapping[str, NestedTensors]:

158 if (images := mm_data.get("images")) is None:

159 return {}

@@ -221,7 +220,6 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):

220 self,

221 mm_data: Mapping[str, object],

222 mm_kwargs: Mapping[str, object],

224 - tok_kwargs: Mapping[str, object],

223 ) -> Mapping[str, NestedTensors]:

224 if (videos := mm_data.get("videos")) is None:

225 return {}

vllm/model_executor/models/mistral3.py

-2

@@ -238,13 +238,11 @@ class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo

238 prompt: str,

239 mm_data: Mapping[str, object],

240 mm_kwargs: Mapping[str, object],

241 - tok_kwargs: Mapping[str, object],

241 ) -> BatchFeature:

242 processed_outputs = super()._call_hf_processor(

243 prompt=prompt,

244 mm_data=mm_data,

245 mm_kwargs=mm_kwargs,

247 - tok_kwargs=tok_kwargs,

246 )

247

248 pixel_values = processed_outputs.get("pixel_values")

vllm/model_executor/models/mllama4.py

-2

@@ -594,13 +594,11 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo])

594 prompt: str,

595 mm_data: Mapping[str, object],

596 mm_kwargs: Mapping[str, object],

597 - tok_kwargs: Mapping[str, object],

597 ) -> BatchFeature:

598 processed_outputs = super()._call_hf_processor(

599 prompt=prompt,

600 mm_data=mm_data,

601 mm_kwargs=mm_kwargs,

603 - tok_kwargs=tok_kwargs,

602 )

603

604 processor = self.info.get_hf_processor(**mm_kwargs)

vllm/model_executor/models/molmo.py

+1 -2

@@ -1128,13 +1128,12 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):

1128 prompt: str,

1129 mm_data: Mapping[str, object],

1130 mm_kwargs: Mapping[str, object],

1131 - tok_kwargs: Mapping[str, object],

1131 ) -> BatchFeature:

1132 hf_processor = self.info.get_hf_processor(**mm_kwargs)

1133 processed_outputs = self.info.ctx.call_hf_processor(

1134 hf_processor.process,

1135 dict(text=prompt, **mm_data),

1137 - dict(**mm_kwargs, **tok_kwargs),

1136 + mm_kwargs,

1137 )

1138

1139 tokenizer = hf_processor.tokenizer

vllm/model_executor/models/molmo2.py

+2 -3

@@ -1948,7 +1948,6 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):

1948 prompt: str,

1949 mm_data: Mapping[str, object],

1950 mm_kwargs: Mapping[str, object],

1951 - tok_kwargs: Mapping[str, object],

1951 ) -> BatchFeature:

1952 mm_data = dict(mm_data)

1953

@@ -2008,7 +2007,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):

2007 video_outputs = self.info.ctx.call_hf_processor(

2008 patched_call,

2009 dict(text=VIDEO_PROMPT, **video_mm_data),

2011 - dict(**video_mm_kwargs, **tok_kwargs),

2010 + video_mm_kwargs,

2011 )

2012

2013 input_ids = video_outputs.pop("input_ids")

@@ -2058,7 +2057,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):

2057 processed_outputs = self.info.ctx.call_hf_processor(

2058 patched_call,

2059 dict(text=prompt, **mm_data),

2061 - dict(**mm_kwargs, **tok_kwargs),

2060 + mm_kwargs,

2061 )

2062

2063 if (images := mm_data.get("images")) is not None:

vllm/model_executor/models/moondream3.py

+1 -14

@@ -947,11 +947,10 @@ class Moondream3MultiModalProcessor(BaseMultiModalProcessor[Moondream3Processing

947 prompt: str,

948 mm_data: Mapping[str, object],

949 mm_kwargs: Mapping[str, object],

950 - tok_kwargs: Mapping[str, object],

950 ) -> BatchFeature:

951 # Moondream3's processor handles images directly rather than exposing a

952 # separate `image_processor`, so keep the cache path on text+MM calls.

954 - return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)

953 + return super()._call_hf_processor(prompt, mm_data, mm_kwargs)

954

955 @cached_property

956 def bos_image_placeholder_tokens(self) -> list[int]:

@@ -977,18 +976,6 @@ class Moondream3MultiModalProcessor(BaseMultiModalProcessor[Moondream3Processing

976 "tilings": MultiModalFieldConfig.batched("image", keep_on_cpu=True),

977 }

978

980 - def _hf_processor_applies_updates(

981 - self,

982 - prompt_text: str,

983 - mm_items: MultiModalDataItems,

984 - hf_processor_mm_kwargs: Mapping[str, object],

985 - tokenization_kwargs: Mapping[str, object],

986 - ) -> bool:

987 - # Moondream3 HF processor does NOT expand placeholder tokens.

988 - # vLLM expands BOS + <image> so the whole HF image prefix is marked

989 - # bidirectional by the multimodal prefix-LM mask.

990 - return False

991 -

979 def _get_prompt_updates(

980 self,

981 mm_items: MultiModalDataItems,

vllm/model_executor/models/moss_audio.py

+35 -7

@@ -49,6 +49,7 @@ from vllm.multimodal.processing import (

49 BaseDummyInputsBuilder,

50 BaseMultiModalProcessor,

51 BaseProcessingInfo,

52 + InputProcessingContext,

53 PromptReplacement,

54 PromptUpdate,

55 PromptUpdateDetails,

@@ -102,6 +103,34 @@ MOSS_AUDIO_PROCESSOR_CONFIG_KEYS = {

103 "enable_time_marker",

104 "mel_config",

105 }

106 +MOSS_AUDIO_PLACEHOLDER_TOKENS = (

107 + MOSS_AUDIO_BOS_TOKEN,

108 + MOSS_AUDIO_TOKEN,

109 + MOSS_AUDIO_EOS_TOKEN,

110 +)

111 +

112 +

113 +def _ensure_moss_audio_placeholder_tokens(tokenizer: object) -> None:

114 + """Register the audio placeholder tokens if the tokenizer lacks them.

115 +

116 + The MOSS-Audio hub tokenizer does not define the audio placeholder

117 + tokens (the reference HF processor patches them in at runtime). Left

118 + as plain text, they merge with adjacent tokens under BPE, so prompts

119 + no longer contain a stable token sequence for prompt updates to match.

120 + """

121 + convert_tokens_to_ids = getattr(tokenizer, "convert_tokens_to_ids", None)

122 + add_tokens = getattr(tokenizer, "add_tokens", None)

123 + if convert_tokens_to_ids is None or add_tokens is None:

124 + return

125 +

126 + unk_token_id = getattr(tokenizer, "unk_token_id", None)

127 + missing_tokens = [

128 + token

129 + for token in MOSS_AUDIO_PLACEHOLDER_TOKENS

130 + if convert_tokens_to_ids(token) in (None, unk_token_id)

131 + ]

132 + if missing_tokens:

133 + add_tokens(missing_tokens, special_tokens=True)

134

135

136 class MossAudioAudioInputs(TensorSchema):

@@ -1157,6 +1186,11 @@ class MossAudioProcessor:

1186

1187

1188 class MossAudioProcessingInfo(BaseProcessingInfo):

1189 + def __init__(self, ctx: InputProcessingContext) -> None:

1190 + super().__init__(ctx)

1191 + if ctx.tokenizer is not None:

1192 + _ensure_moss_audio_placeholder_tokens(ctx.tokenizer)

1193 +

1194 def get_hf_config(self) -> MossAudioConfig:

1195 config = self.ctx.get_hf_config()

1196 if isinstance(config, MossAudioConfig):

@@ -1306,7 +1340,6 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn

1340 prompt: str,

1341 mm_data: Mapping[str, object],

1342 mm_kwargs: Mapping[str, object],

1309 - tok_kwargs: Mapping[str, object],

1343 ) -> BatchFeature:

1344 mm_data = dict(mm_data)

1345 audios = mm_data.pop("audios", [])

@@ -1314,15 +1347,10 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn

1347 mm_data["audio"] = audios

1348 mm_kwargs = dict(mm_kwargs)

1349 processor_kwargs = _filter_moss_audio_processor_config(mm_kwargs)

1317 - tok_kwargs = {

1318 - key: value

1319 - for key, value in tok_kwargs.items()

1320 - if key not in MOSS_AUDIO_PROCESSOR_CONFIG_KEYS

1321 - }

1350 return self.info.ctx.call_hf_processor(

1351 self.info.get_hf_processor(**processor_kwargs),

1352 dict(text=prompt, **mm_data),

1325 - dict(**tok_kwargs),

1353 + {},

1354 )

1355

1356 def _get_mm_fields_config(

vllm/model_executor/models/moss_transcribe_diarize.py

+2 -15

@@ -466,33 +466,20 @@ class MossTranscribeDiarizeMultiModalProcessor(

466 prompt: str,

467 mm_data: Mapping[str, object],

468 mm_kwargs: Mapping[str, object],

469 - tok_kwargs: Mapping[str, object],

469 ) -> BatchFeature:

470 tokenizer = self.info.get_tokenizer()

471 audios = _get_audios_from_mm_data(mm_data)

472 if not audios:

474 - input_ids = tokenizer.encode(

475 - prompt,

476 - add_special_tokens=tok_kwargs.get("add_special_tokens", False),

477 - )

473 + input_ids = tokenizer.encode(prompt, add_special_tokens=False)

474 return BatchFeature({"input_ids": [input_ids]}, tensor_type="pt")

475

476 processed = self.info.ctx.call_hf_processor(

477 self.info.get_hf_processor(**mm_kwargs),

478 dict(text=prompt, audio=audios),

483 - dict(**mm_kwargs, **tok_kwargs),

479 + mm_kwargs,

480 )

481 return _add_vllm_audio_metadata(processed, len(audios))

482

487 - def _hf_processor_applies_updates(

488 - self,

489 - prompt_text: str,

490 - mm_items: MultiModalDataItems,

491 - hf_processor_mm_kwargs: Mapping[str, object],

492 - tokenization_kwargs: Mapping[str, object],

493 - ) -> bool:

494 - return mm_items.get_count("audio", strict=False) > 0

495 -

483 def _get_mm_fields_config(

484 self,

485 hf_inputs: BatchFeature,

vllm/model_executor/models/muse_glimmer.py

-13

@@ -262,23 +262,11 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing

262 class MuseGlimmerMultiModalProcessor(

263 BaseMultiModalProcessor[MuseGlimmerProcessingInfo]

264 ):

265 - def _apply_hf_processor_text_only(

266 - self,

267 - prompt_text: str,

268 - tokenization_kwargs: Mapping[str, object],

269 - ) -> list[int]:

270 - tokenizer = self.info.get_tokenizer()

271 - return tokenizer.encode(

272 - prompt_text,

273 - **{"add_special_tokens": False, **tokenization_kwargs},

274 - )

275 -

265 def _call_hf_processor(

266 self,

267 prompt: str,

268 mm_data: Mapping[str, object],

269 mm_kwargs: Mapping[str, object],

281 - tok_kwargs: Mapping[str, object],

270 ) -> BatchFeature:

271 processor = self.info.get_hf_processor(**mm_kwargs)

272 tokenizer = processor.tokenizer

@@ -366,7 +354,6 @@ class MuseGlimmerMultiModalProcessor(

354 video_pixel_values=video_pixels,

355 video_feature_sizes=torch.tensor(video_sizes),

356 )

369 - del tok_kwargs

357 return BatchFeature(data=data, tensor_type=None)

358

359 def _get_mm_fields_config(

vllm/model_executor/models/nano_nemotron_vl.py

+2 -8

@@ -366,14 +366,13 @@ class NanoNemotronVLMultiModalProcessor(

366 prompt: str,

367 mm_data: Mapping[str, object],

368 mm_kwargs: Mapping[str, object],

369 - tok_kwargs: Mapping[str, object],

369 ) -> BatchFeature:

370 """

371 Bypass `call_hf_processor_mm_only` by no-op overriding`_call_hf_processor`,

372 so it chooses this path:

373 `type(self)._call_hf_processor != BaseMultiModalProcessor._call_hf_processor`

374 """

376 - return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)

375 + return super()._call_hf_processor(prompt, mm_data, mm_kwargs)

376

377 def _get_image_fields_config(self, hf_inputs: BatchFeature):

378 if self.info.is_dynamic_tiler:

@@ -714,10 +713,8 @@ class NanoNemotronVLMultiModalProcessor(

713 if not audio_items:

714 return super().apply(inputs, timing_ctx)

715

717 - prompt = inputs.prompt

716 tokenizer = self.info.get_tokenizer()

719 - if not isinstance(prompt, str):

720 - prompt = tokenizer.decode(prompt, skip_special_tokens=False)

717 + prompt = tokenizer.decode(inputs.prompt, skip_special_tokens=False)

718

719 # Inject AUDIO_CONTEXT only after <video> tokens whose video

720 # actually contained an audio stream (preserving video-audio pairing).

@@ -733,9 +730,6 @@ class NanoNemotronVLMultiModalProcessor(

730

731 inputs.prompt = tokenizer.encode(prompt, add_special_tokens=False)

732

736 - if inputs.tokenization_kwargs is None:

737 - inputs.tokenization_kwargs = {}

738 -

733 # Bypass the cached path: the HF processor must receive the

734 # prompt (with injected <so_embedding>) and the audio data

735 # together so it can perform audio-token replacement natively.

vllm/model_executor/models/nemotron_parse.py

+3 -6

@@ -406,9 +406,9 @@ class NemotronParseMultiModalProcessor(

406 ):

407 def create_encoder_prompt(

408 self,

409 - prompt: str | list[int],

409 + prompt: list[int],

410 mm_items: MultiModalDataItems,

411 - ) -> str | list[int]:

411 + ) -> list[int]:

412 return [0]

413

414 def _call_hf_processor(

@@ -416,12 +416,9 @@ class NemotronParseMultiModalProcessor(

416 prompt: str,

417 mm_data: Mapping[str, object],

418 mm_kwargs: Mapping[str, object],

419 - tok_kwargs: Mapping[str, object],

419 ) -> BatchFeature:

420 if mm_data:

422 - processed_outputs = super()._call_hf_processor(

423 - prompt, mm_data, mm_kwargs, tok_kwargs

424 - )

421 + processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)

422 else:

423 hf_processor = self.info.get_hf_processor()

424 tokenizer = hf_processor.tokenizer

vllm/model_executor/models/opencua.py

-10

@@ -141,16 +141,6 @@ class OpenCUAMultiModalProcessor(BaseMultiModalProcessor[OpenCUAProcessingInfo])

141 self.info.get_hf_config().vision_config.spatial_merge_size

142 )(hf_inputs)

143

144 - def _hf_processor_applies_updates(

145 - self,

146 - prompt_text: str,

147 - mm_items: MultiModalDataItems,

148 - hf_processor_mm_kwargs: Mapping[str, object],

149 - tokenization_kwargs: Mapping[str, object],

150 - ) -> bool:

151 - """vLLM이 prompt 업데이트를 처리하도록 False 반환."""

152 - return False

153 -

144 def _get_prompt_updates(

145 self,

146 mm_items: MultiModalDataItems,

vllm/model_executor/models/openvla.py

-9

@@ -327,15 +327,6 @@ class OpenVLAMultiModalProcessor(BaseMultiModalProcessor[OpenVLAProcessingInfo])

327 ) -> Mapping[str, MultiModalFieldConfig]:

328 return dict(pixel_values=MultiModalFieldConfig.batched("image"))

329

330 - def _hf_processor_applies_updates(

331 - self,

332 - prompt_text: str,

333 - mm_items: MultiModalDataItems,

334 - hf_processor_mm_kwargs: Mapping[str, object],

335 - tokenization_kwargs: Mapping[str, object],

336 - ) -> bool:

337 - return False

338 -

330 def _get_prompt_updates(

331 self,

332 mm_items: MultiModalDataItems,

vllm/model_executor/models/ovis.py

-2

@@ -346,7 +346,6 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):

346 prompt: str,

347 mm_data: Mapping[str, object],

348 mm_kwargs: Mapping[str, object],

349 - tok_kwargs: Mapping[str, object],

349 ) -> BatchFeature:

350 if not mm_data:

351 # Avoid warning from HF logger for text-only input

@@ -358,7 +357,6 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):

357 prompt=prompt,

358 mm_data=mm_data,

359 mm_kwargs=mm_kwargs,

361 - tok_kwargs=tok_kwargs,

360 )

361

362 hf_processor = self.info.get_hf_processor()

vllm/model_executor/models/ovis2_5.py

-2

@@ -337,7 +337,6 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo])

337 prompt: str,

338 mm_data: Mapping[str, object],

339 mm_kwargs: Mapping[str, object],

340 - tok_kwargs: Mapping[str, object],

340 ) -> BatchFeature:

341 if not mm_data:

342 # Avoid warning from HF logger for text-only input

@@ -349,7 +348,6 @@ class Ovis2_5MultiModalProcessor(BaseMultiModalProcessor[Ovis2_5ProcessingInfo])

348 prompt=prompt,

349 mm_data=mm_data,

350 mm_kwargs=mm_kwargs,

352 - tok_kwargs=tok_kwargs,

351 )

352 hf_processor = self.info.get_hf_processor()

353

vllm/model_executor/models/paddleocr_vl.py

+1 -2

@@ -249,7 +249,6 @@ class PaddleOCRVLMultiModalProcessor(

249 prompt: str,

250 mm_data: Mapping[str, object],

251 mm_kwargs: Mapping[str, object],

252 - tok_kwargs: Mapping[str, object],

252 ) -> BatchFeature:

253 if mm_data:

254 final_mm_kwargs = dict(mm_kwargs or {})

@@ -259,7 +258,7 @@ class PaddleOCRVLMultiModalProcessor(

258 processed_outputs = self.info.ctx.call_hf_processor(

259 self.info.get_hf_processor(**final_mm_kwargs),

260 dict(text=prompt, **mm_data),

262 - dict(**mm_kwargs, **tok_kwargs),

261 + mm_kwargs,

262 )

263 num_patches_per_image = processed_outputs["image_grid_thw"].prod(-1)

264 processed_outputs["pixel_values"] = processed_outputs["pixel_values"].split(

vllm/model_executor/models/paligemma.py

-2

@@ -157,7 +157,6 @@ class PaliGemmaMultiModalProcessor(BaseMultiModalProcessor[PaliGemmaProcessingIn

157 prompt: str,

158 mm_data: Mapping[str, object],

159 mm_kwargs: Mapping[str, object],

160 - tok_kwargs: Mapping[str, object],

160 ) -> BatchFeature:

161 tokenizer = self.info.get_tokenizer()

162 if not mm_data:

@@ -168,7 +167,6 @@ class PaliGemmaMultiModalProcessor(BaseMultiModalProcessor[PaliGemmaProcessingIn

167 prompt=prompt,

168 mm_data=mm_data,

169 mm_kwargs=mm_kwargs,

171 - tok_kwargs=tok_kwargs,

170 )

171

172 def _get_mm_fields_config(

vllm/model_executor/models/phi3v.py

+2 -8

@@ -401,13 +401,11 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):

401 prompt: str,

402 mm_data: Mapping[str, object],

403 mm_kwargs: Mapping[str, object],

404 - tok_kwargs: Mapping[str, object],

404 ) -> BatchFeature:

405 processed_outputs = super()._call_hf_processor(

406 prompt=prompt,

407 mm_data=mm_data,

408 mm_kwargs=mm_kwargs,

410 - tok_kwargs=tok_kwargs,

409 )

410

411 input_ids = processed_outputs["input_ids"]

@@ -497,12 +495,8 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):

495 if len(token_ids) and token_ids[0] == tokenizer.bos_token_id:

496 token_ids = token_ids[1:]

497 text = tokenizer.decode(token_ids)

500 - for special_tokens in tokenizer.special_tokens_map.values():

501 - if isinstance(special_tokens, str):

502 - text = text.replace(f"{special_tokens} ", special_tokens)

503 - elif isinstance(special_tokens, list):

504 - for special_token in special_tokens:

505 - text = text.replace(f"{special_token} ", special_token)

498 + for special_token in tokenizer.all_special_tokens:

499 + text = text.replace(f"{special_token} ", special_token)

500 # perform hf behavior

501 # https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/64f88b6/processing_phi3_v.py#L407

502 pattern = r"<\|image_\d+\|>"

vllm/model_executor/models/phi4mm.py

+1 -4

@@ -873,7 +873,6 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):

873 prompt: str,

874 mm_data: Mapping[str, object],

875 mm_kwargs: Mapping[str, object],

876 - tok_kwargs: Mapping[str, object],

876 ) -> BatchFeature:

877 if not mm_data:

878 prompt_ids = self.info.get_tokenizer().encode(prompt)

@@ -884,9 +883,7 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):

883 if audio_data := mm_data.get("audios", []):

884 mm_data["audios"] = [(data, sr) for data in audio_data]

885

887 - processed_outputs = super()._call_hf_processor(

888 - prompt, mm_data, mm_kwargs, tok_kwargs

889 - )

886 + processed_outputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)

887

888 hf_processor = self.info.get_hf_processor(**mm_kwargs)

889 num_img_tokens = [

vllm/model_executor/models/phi4siglip.py

-16

@@ -147,13 +147,11 @@ class Phi4SiglipMultiModalProcessor(

147 prompt: str,

148 mm_data: Mapping[str, object],

149 mm_kwargs: Mapping[str, object],

150 - tok_kwargs: Mapping[str, object],

150 ) -> BatchFeature:

151 processed = super()._call_hf_processor(

152 prompt=prompt,

153 mm_data=mm_data,

154 mm_kwargs=mm_kwargs,

156 - tok_kwargs=tok_kwargs,

155 )

156

157 # The HF processor's tokenizer_image_token() replaces the "<image>"

@@ -170,20 +168,6 @@ class Phi4SiglipMultiModalProcessor(

168

169 return processed

170

173 - def _hf_processor_applies_updates(

174 - self,

175 - prompt_text: str,

176 - mm_items: MultiModalDataItems,

177 - hf_processor_mm_kwargs: Mapping[str, object],

178 - tokenization_kwargs: Mapping[str, object],

179 - ) -> bool:

180 - # The HF processor replaces "<image>" with a single -200 placeholder

181 - # but does NOT expand it into N vision-encoder tokens. Since we also

182 - # re-tokenize the prompt (see _call_hf_processor), prompt updates are

183 - # never applied by the HF processor — vLLM handles the expansion via

184 - # _apply_prompt_updates.

185 - return False

186 -

171 def _get_mm_fields_config(

172 self,

173 hf_inputs: BatchFeature,

vllm/model_executor/models/pixtral.py

+4 -6

@@ -225,14 +225,12 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])

225 prompt: str,

226 mm_data: Mapping[str, object],

227 mm_kwargs: Mapping[str, object],

228 - tok_kwargs: Mapping[str, object],

228 ) -> BatchFeature:

230 - outputs = super()._call_hf_processor(

231 - prompt=prompt,

232 - mm_data=mm_data,

233 - mm_kwargs=mm_kwargs,

229 + outputs = self.info.ctx.call_hf_processor(

230 + self.info.get_hf_processor(**mm_kwargs),

231 + dict(text=prompt, **mm_data),

232 # Avoid padding issue

235 - tok_kwargs={**tok_kwargs, "return_tensors": None},

233 + dict(**mm_kwargs, return_tensors=None),

234 )

235

236 # Missing batch dimension

vllm/model_executor/models/qwen2_5_omni_thinker.py

+1 -38

@@ -504,7 +504,6 @@ class Qwen2_5OmniThinkerMultiModalProcessor(

504 prompt: str,

505 mm_data: Mapping[str, object],

506 mm_kwargs: Mapping[str, object],

507 - tok_kwargs: Mapping[str, object],

507 ) -> BatchFeature:

508 mm_data = dict(mm_data)

509 audios = mm_data.pop("audios", [])

@@ -552,7 +551,6 @@ class Qwen2_5OmniThinkerMultiModalProcessor(

551 prompt=prompt,

552 mm_data=mm_data,

553 mm_kwargs=mm_kwargs,

555 - tok_kwargs=tok_kwargs,

554 )

555

556 input_features = hf_inputs.pop("input_features", None)

@@ -887,44 +885,10 @@ class Qwen2_5OmniThinkerMultiModalProcessor(

885 ),

886 ]

887

890 - def _apply_hf_processor_main(

891 - self,

892 - prompt: str | list[int],

893 - mm_items: MultiModalDataItems,

894 - hf_processor_mm_kwargs: Mapping[str, object],

895 - tokenization_kwargs: Mapping[str, object],

896 - *,

897 - enable_hf_prompt_update: bool,

898 - ) -> tuple[list[int], BatchFeature, bool]:

899 - """

900 - Qwen2.5-Omni reimplements this function to handle text only.

901 - """

902 - if isinstance(prompt, str):

903 - if enable_hf_prompt_update:

904 - return self._apply_hf_processor_text_mm(

905 - prompt_text=prompt,

906 - mm_items=mm_items,

907 - hf_processor_mm_kwargs=hf_processor_mm_kwargs,

908 - tokenization_kwargs=tokenization_kwargs,

909 - )

910 - tokenizer = self.info.get_tokenizer()

911 - prompt_ids = tokenizer.encode(prompt)

912 - else:

913 - prompt_ids = self._apply_hf_processor_tokens_only(prompt)

914 -

915 - mm_processed_data = self._apply_hf_processor_mm_only(

916 - mm_items=mm_items,

917 - hf_processor_mm_kwargs=hf_processor_mm_kwargs,

918 - tokenization_kwargs=tokenization_kwargs,

919 - )

920 -

921 - return prompt_ids, mm_processed_data, False

922 -

888 def _apply_hf_processor_mm_only(

889 self,

890 mm_items: MultiModalDataItems,

891 hf_processor_mm_kwargs: Mapping[str, object],

927 - tokenization_kwargs: Mapping[str, object],

892 ) -> BatchFeature:

893 """

894 Qwen2.5-Omni reimplements this function to handle `use_audio_in_video`.

@@ -939,11 +903,10 @@ class Qwen2_5OmniThinkerMultiModalProcessor(

903 )

904 mm_counts["audio"] -= mm_counts["video"]

905

942 - _, mm_processed_data, _ = self._apply_hf_processor_text_mm(

906 + _, mm_processed_data = self._apply_hf_processor_text_mm(

907 prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),

908 mm_items=mm_items,

909 hf_processor_mm_kwargs=hf_processor_mm_kwargs,

946 - tokenization_kwargs=tokenization_kwargs,

910 )

911

912 return mm_processed_data

vllm/model_executor/models/qwen2_5_vl.py

+1 -2

@@ -1159,11 +1159,10 @@ class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):

1159 prompt: str,

1160 mm_data: Mapping[str, object],

1161 mm_kwargs: Mapping[str, object],

1162 - tok_kwargs: Mapping[str, object],

1162 ) -> BatchFeature:

1163 # Override to use the text path instead of token path to use the

1164 # video-specific logic in processing_qwen2_5_vl.py

1166 - return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)

1165 + return super()._call_hf_processor(prompt, mm_data, mm_kwargs)

1166

1167 def _get_prompt_updates(

1168 self,

vllm/model_executor/models/qwen2_audio.py

-2

@@ -240,7 +240,6 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessing

240 prompt: str,

241 mm_data: Mapping[str, object],

242 mm_kwargs: Mapping[str, Any],

243 - tok_kwargs: Mapping[str, object],

243 ) -> BatchFeature:

244 # NOTE - we rename audios -> audio in mm data because transformers has

245 # deprecated audios for the qwen2audio processor and will remove

@@ -265,7 +264,6 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessing

264 prompt=prompt,

265 mm_data=mm_data,

266 mm_kwargs=mm_kwargs,

268 - tok_kwargs=tok_kwargs,

267 )

268

269 def _get_mm_fields_config(

vllm/model_executor/models/qwen3_omni_moe_thinker.py

+1 -4

@@ -1200,7 +1200,6 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(

1200 prompt: str,

1201 mm_data: Mapping[str, object],

1202 mm_kwargs: Mapping[str, object],

1203 - tok_kwargs: Mapping[str, object],

1203 ) -> BatchFeature:

1204 mm_data = dict(mm_data)

1205 audios = mm_data.pop("audios", [])

@@ -1230,7 +1229,6 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(

1229 # released and Transformers version update:

1230 # https://github.com/huggingface/transformers/pull/41473

1231 mm_kwargs = dict(mm_kwargs)

1233 - tok_kwargs = dict(tok_kwargs)

1232 mm_kwargs["audio_kwargs"] = dict(mm_kwargs.get("audio_kwargs") or {})

1233 mm_kwargs["text_kwargs"] = dict(mm_kwargs.get("text_kwargs") or {})

1234 elif mm_kwargs.get("use_audio_in_video") and mm_data.get("videos"):

@@ -1245,7 +1243,6 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(

1243 prompt=prompt,

1244 mm_data=mm_data,

1245 mm_kwargs=mm_kwargs,

1248 - tok_kwargs=tok_kwargs,

1246 )

1247

1248 if (

@@ -1279,7 +1276,7 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(

1276 mm_kwargs: MultiModalKwargsItems,

1277 mm_prompt_updates: MultiModalPromptUpdates,

1278 is_update_applied: bool,

1282 - ) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:

1279 + ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:

1280 """

1281 Qwen3-Omni reimplements this function to handle `use_audio_in_video`.

1282 """

vllm/model_executor/models/qwen3_vl.py

-3

@@ -1285,7 +1285,6 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])

1285 prompt: str,

1286 mm_data: Mapping[str, object],

1287 mm_kwargs: Mapping[str, object],

1288 - tok_kwargs: Mapping[str, object],

1288 ) -> BatchFeature:

1289 mm_data = dict(mm_data)

1290

@@ -1370,7 +1369,6 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])

1369 prompt="<|vision_start|><|video_pad|><|vision_end|>",

1370 mm_data=video_mm_data,

1371 mm_kwargs=video_mm_kwargs,

1373 - tok_kwargs=tok_kwargs,

1372 )

1373

1374 # Discard HF output input_ids — we use get_video_repl below

@@ -1432,7 +1430,6 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])

1430 prompt=prompt,

1431 mm_data=mm_data,

1432 mm_kwargs=non_video_mm_kwargs,

1435 - tok_kwargs=tok_kwargs,

1433 )

1434

1435 # Replace each placeholder with pre-computed video tokens.

vllm/model_executor/models/siglip.py

+7 -30

@@ -191,40 +191,17 @@ class SiglipMultiModalProcessor(BaseMultiModalProcessor[SiglipProcessingInfo]):

191 timing_ctx: TimingContext,

192 ) -> MultiModalInput:

193 if inputs.mm_data_items:

194 - if isinstance(inputs.prompt, str):

195 - if len(inputs.prompt) > 0:

196 - raise ValueError(

197 - "SigLIP accepts text-only or image-only inputs, not both! "

198 - "You must pass an image with an empty text prompt."

199 - )

194 + special_tokens = self.info.get_tokenizer().all_special_ids

195 + if all(tok in special_tokens for tok in inputs.prompt):

196 + inputs.prompt = []

197 else:

201 - special_tokens = self.info.get_tokenizer().all_special_ids

202 - if all(tok in special_tokens for tok in inputs.prompt):

203 - inputs.prompt = []

204 - else:

205 - raise ValueError(

206 - "SigLIP accepts text-only or image-only inputs, not both! "

207 - "You must pass an image with an empty token prompt."

208 - )

209 -

210 - # For multi-modal data, the prompt after processing should

211 - # only contain the dummy image tokens

212 - inputs.tokenization_kwargs = {

213 - **inputs.tokenization_kwargs,

214 - "add_special_tokens": False,

215 - }

198 + raise ValueError(

199 + "SigLIP accepts text-only or image-only inputs, not both! "

200 + "You must pass an image with an empty token prompt."

201 + )

202

203 return super().apply(inputs, timing_ctx)

204

219 - def _hf_processor_applies_updates(

220 - self,

221 - prompt_text: str,

222 - mm_items: MultiModalDataItems,

223 - hf_processor_mm_kwargs: Mapping[str, object],

224 - tokenization_kwargs: Mapping[str, object],

225 - ) -> bool:

226 - return False

227 -

205 def _get_mm_fields_config(

206 self,

207 hf_inputs: BatchFeature,

vllm/model_executor/models/transformers/multimodal.py

+16 -63

@@ -350,24 +350,21 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]

350 prompt: str,

351 mm_data: Mapping[str, object],

352 mm_kwargs: Mapping[str, object],

353 - tok_kwargs: Mapping[str, object],

353 ) -> "BatchFeature":

354 """Run the HF processor and unpad inputs."""

355 try:

357 - hf_inputs = super()._call_hf_processor(

358 - prompt, mm_data, mm_kwargs, tok_kwargs

359 - )

356 + hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)

357 except ValueError:

358 if any(mm_data.values()):

359 raise

360 # Some processors reject a prompt holding placeholders with

361 # no data to go with them, so tokenize it without them

365 - prompt_ids = self.info.get_tokenizer().encode(prompt, **tok_kwargs)

362 + prompt_ids = self.info.get_tokenizer().encode(prompt)

363 hf_inputs = transformers.BatchFeature(

364 dict(input_ids=[prompt_ids]), tensor_type="pt"

365 )

366 self._unpad_images(hf_inputs)

370 - self._unpad_audios(hf_inputs, mm_data, mm_kwargs, tok_kwargs)

367 + self._unpad_audios(hf_inputs, mm_data, mm_kwargs)

368 return hf_inputs

369

370 def _unpad_audios(

@@ -375,7 +372,6 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]

372 hf_inputs: "BatchFeature",

373 mm_data: Mapping[str, object],

374 mm_kwargs: Mapping[str, object],

378 - tok_kwargs: Mapping[str, object],

375 ) -> None:

376 """Replace the audio fields with each audio processed on its own.

377

@@ -395,7 +391,7 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]

391 dict(

392 text=self.dummy_inputs.get_dummy_text({"audio": 1}), audio=[audio]

393 ),

398 - dict(**mm_kwargs, **tok_kwargs),

394 + mm_kwargs,

395 )

396 for audio in audios

397 ]

@@ -459,13 +455,13 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):

455 prompt: str,

456 mm_data: Mapping[str, object],

457 mm_kwargs: Mapping[str, object],

462 - tok_kwargs: Mapping[str, object],

458 ) -> "BatchFeature":

459 """Ask which modality owns each token of the expanded prompt, which is the

460 only marking of the tokens an expansion adds around an item."""

461 if any(mm_data.values()):

462 mm_data = {**mm_data, "return_mm_token_type_ids": True}

468 - return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)

463 +

464 + return super()._call_hf_processor(prompt, mm_data, mm_kwargs)

465

466 def _get_mm_token_ids(self, modality: str) -> list[int]:

467 """Token ids marking where `modality` sits in the prompt, which for some

@@ -613,29 +609,26 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):

609 prompt = inputs.prompt

610 mm_items = inputs.mm_data_items

611 hf_processor_mm_kwargs = inputs.hf_processor_mm_kwargs

616 - tokenization_kwargs = inputs.tokenization_kwargs

612

613 with timing_ctx.record("apply_hf_processor"):

614 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)

620 - if not isinstance(prompt, str):

621 - # HF processors only accept text, and the decoded string already

622 - # contains any special tokens, so don't let them be added again

623 - prompt = hf_processor.decode(prompt)

624 - tokenization_kwargs = {

625 - **tokenization_kwargs,

626 - "add_special_tokens": False,

627 - }

615 + prompt = hf_processor.decode(prompt)

616

617 # Bypass cached processor and always apply to the full set of mm inputs

618 # NOTE: we can't just set caching=False because base class method

619 # transforms outputs to `MultiModalKwargs` which is not going to

620 # work for Transformers. The vision path has logic tied to

621 # `mm_tokens_per_modality` in _apply_vision()

634 - prompt_ids, processed_data, _ = self._apply_hf_processor_text_mm(

622 + prompt_ids, processed_data = self._apply_hf_processor_text_mm(

623 prompt_text=prompt,

624 mm_items=mm_items,

637 - hf_processor_mm_kwargs=hf_processor_mm_kwargs,

638 - tokenization_kwargs=tokenization_kwargs,

625 + hf_processor_mm_kwargs={

626 + **hf_processor_mm_kwargs,

627 + # HF processors only accept text, and the decoded string already

628 + # contains any special tokens, so don't let them be added again

629 + # (`_call_hf_processor` disables `add_special_tokens`).

630 + "add_special_tokens": False,

631 + },

632 )

633

634 # Use overrides if provided; fallback to data-dependent hashing.

@@ -731,45 +724,6 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):

724 )

725 return updates

726

734 - def _apply_hf_processor_main(

735 - self,

736 - prompt: str | list[int],

737 - mm_items: MultiModalDataItems,

738 - hf_processor_mm_kwargs: Mapping[str, object],

739 - tokenization_kwargs: Mapping[str, object],

740 - *,

741 - enable_hf_prompt_update: bool,

742 - ) -> tuple[list[int], "BatchFeature", bool]:

743 - """Tokenize the prompt unexpanded, leaving the expansion for vLLM to splice

744 - in, whatever `enable_hf_prompt_update` asks for.

745 -

746 - This differs from `super()` only for a text prompt with

747 - `enable_hf_prompt_update`, where `super()` would keep the expanded token

748 - ids the HF processor returns. Both routes give the same ids, so forcing

749 - this one costs an extra processor call and buys the guarantee that a

750 - request bypassing the cache is identical to a cached one.

751 -

752 - A text prompt only arrives with `enable_hf_prompt_update` when there is

753 - no multi-modal processor cache. Requests carrying pre-computed

754 - embeddings bypass the cache too, but `--enable-mm-embeds` is unsupported

755 - here, so they fail validation shortly afterwards.

756 - """

757 - if isinstance(prompt, str) and enable_hf_prompt_update:

758 - logger.warning_once(

759 - "Disabling the multi-modal processor cache is extra slow with the "

760 - "Transformers modeling backend: the prompt is still tokenized "

761 - "unexpanded and the expansion spliced in, to keep the token ids "

762 - "identical to what the cache produces, which costs an extra HF "

763 - "processor call per request."

764 - )

765 - return super()._apply_hf_processor_main(

766 - prompt,

767 - mm_items,

768 - hf_processor_mm_kwargs,

769 - tokenization_kwargs,

770 - enable_hf_prompt_update=False,

771 - )

772 -

727 def _get_num_image_patches(

728 self,

729 hf_inputs: "BatchFeature",

@@ -824,13 +778,12 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):

778 prompt: str,

779 mm_data: Mapping[str, object],

780 mm_kwargs: Mapping[str, object],

827 - tok_kwargs: Mapping[str, object],

781 ) -> "BatchFeature":

782 """Ask for the replacement each placeholder expands to, and record it as

783 per-item fields: its token ids, and the tokens or patches behind them."""

784 if has_mm_data := any(mm_data.values()):

785 mm_data = {**mm_data, "return_text_replacement_offsets": True}

833 - hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)

786 + hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)

787 # Drop the inputs the model would reject

788 hf_inputs.pop("mm_token_type_ids", None)

789 hf_inputs.pop("token_type_ids", None)

vllm/model_executor/models/ultravox.py

-7

@@ -195,7 +195,6 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo

195 prompt: str,

196 mm_data: Mapping[str, object],

197 mm_kwargs: Mapping[str, object],

198 - tok_kwargs: Mapping[str, object],

198 ) -> BatchFeature:

199 # Text-only input not supported in composite processor

200 if not mm_data.get("audios", []):

@@ -218,16 +217,10 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo

217

218 item_processor_data = dict(**mm_data, audios=audios)

219

221 - # some tokenizer kwargs are incompatible with UltravoxProcessor

222 - tok_kwargs.pop("add_special_tokens", None)

223 - tok_kwargs.pop("padding", None)

224 - tok_kwargs.pop("truncation", None)

225 -

220 output = super()._call_hf_processor(

221 prompt=prompt,

222 mm_data=item_processor_data,

223 mm_kwargs=mm_kwargs,

230 - tok_kwargs=tok_kwargs,

224 )

225 output["audio_features"] = output.pop("audio_values")

226

vllm/model_executor/models/voxtral.py

+4 -6

@@ -225,7 +225,6 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])

225 prompt: str,

226 mm_data: Mapping[str, object],

227 mm_kwargs: Mapping[str, object],

228 - tok_kwargs: Mapping[str, object],

228 ) -> BatchFeature:

229 mm_data = dict(mm_data)

230 audios = mm_data.pop("audios", [])

@@ -234,12 +233,11 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])

233 # MistralCommonVoxtralProcessor accepts "audio"

234 mm_data["audio"] = audios

235

237 - outputs = super()._call_hf_processor(

238 - prompt=prompt,

239 - mm_data=mm_data,

240 - mm_kwargs=mm_kwargs,

236 + outputs = self.info.ctx.call_hf_processor(

237 + self.info.get_hf_processor(**mm_kwargs),

238 + dict(text=prompt, **mm_data),

239 # Avoid padding issue

242 - tok_kwargs={**tok_kwargs, "return_tensors": None},

240 + dict(**mm_kwargs, return_tensors=None),

241 )

242

243 # Missing batch dimension

vllm/model_executor/models/whisper.py

+2 -12

@@ -747,9 +747,9 @@ class WhisperDummyInputsBuilder(BaseDummyInputsBuilder[WhisperProcessingInfo]):

747 class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo]):

748 def create_encoder_prompt(

749 self,

750 - prompt: str | list[int],

750 + prompt: list[int],

751 mm_items: MultiModalDataItems,

752 - ) -> str | list[int]:

752 + ) -> list[int]:

753 # Strictly speaking, whisper encoder only accept audio features.

754 # We create a dummy encoder prompt here which will be padded to

755 # num_audio_tokens. So that we can create dummy data from this

@@ -761,7 +761,6 @@ class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo

761 prompt: str,

762 mm_data: Mapping[str, object],

763 mm_kwargs: Mapping[str, object],

764 - tok_kwargs: Mapping[str, object],

764 ) -> BatchFeature:

765 if mm_data:

766 feature_extractor = self.info.get_feature_extractor(**mm_kwargs)

@@ -770,19 +769,10 @@ class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo

769 **mm_kwargs,

770 sampling_rate=feature_extractor.sampling_rate,

771 )

773 - # The HF WhisperProcessor passes **kwargs to both the tokenizer

774 - # and the feature extractor. Text-tokenizer kwargs like

775 - # `truncation` and `max_length` must be removed when audio data

776 - # is present, otherwise the feature extractor interprets

777 - # `max_length` as raw audio samples and truncates the audio.

778 - tok_kwargs = {

779 - k: v for k, v in tok_kwargs.items() if k not in ("truncation", "max_length")

780 - }

772 processed_outputs = super()._call_hf_processor(

773 prompt=prompt,

774 mm_data=mm_data,

775 mm_kwargs=mm_kwargs,

785 - tok_kwargs=tok_kwargs,

776 )

777 if "labels" in processed_outputs:

778 processed_outputs["input_ids"] = processed_outputs.pop("labels")

vllm/models/dots3_note/common/processor.py

+1 -11

@@ -654,18 +654,8 @@ class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingIn

654 prompt: str,

655 mm_data: Mapping[str, object],

656 mm_kwargs: Mapping[str, object],

657 - tok_kwargs: Mapping[str, object],

657 ) -> BatchFeature:

659 - return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)

660 -

661 - def _hf_processor_applies_updates(

662 - self,

663 - prompt_text: str,

664 - mm_items: MultiModalDataItems,

665 - hf_processor_mm_kwargs: Mapping[str, object],

666 - tokenization_kwargs: Mapping[str, object],

667 - ) -> bool:

668 - return False

658 + return super()._call_hf_processor(prompt, mm_data, mm_kwargs)

659

660 def _get_mm_fields_config(

661 self,

vllm/models/inkling/common/mm_preprocess.py

+3 -15

@@ -22,7 +22,7 @@ from vllm.multimodal.inputs import (

22 MultiModalFieldConfig,

23 MultiModalKwargsItems,

24 )

25 -from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser

25 +from vllm.multimodal.parse import MultiModalDataParser

26 from vllm.multimodal.processing import (

27 BaseDummyInputsBuilder,

28 BaseMultiModalProcessor,

@@ -179,21 +179,11 @@ class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):

179

180

181 class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo]):

182 - def _hf_processor_applies_updates(

183 - self,

184 - prompt_text: str,

185 - mm_items: MultiModalDataItems,

186 - hf_processor_mm_kwargs: Mapping[str, object],

187 - tokenization_kwargs: Mapping[str, object],

188 - ) -> bool:

189 - return False

190 -

182 def _call_hf_processor(

183 self,

184 prompt: str,

185 mm_data: Mapping[str, object],

186 mm_kwargs: Mapping[str, object],

196 - tok_kwargs: Mapping[str, object],

187 ) -> BatchFeature:

188 # Inkling is not a standard HF processor (no fused text+mm call), so we run

189 # the vendored extractors ourselves and tokenize the text separately.

@@ -275,10 +265,8 @@ class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo])

265 ids.extend(tokenizer.encode(chunk, add_special_tokens=False))

266

267 # Reconcile against the declared media counts only when media is

278 - # present. With no media items -- e.g. the base text-only tokenization

279 - # probe (``_apply_hf_processor_text_only``), which calls this via

280 - # ``_call_hf_processor`` with empty ``mm_data`` -- emit the markers

281 - # verbatim; the marker<->item correspondence is enforced later by

268 + # present. With no media items, emit the markers verbatim; the

269 + # marker<->item correspondence is enforced later by

270 # ``_get_prompt_updates`` once the media features are available.

271 if num_images or num_audios:

272 # Fail clearly on a placeholder/media-count mismatch instead of

vllm/models/kimi_k3/common/mm_preprocess.py

+1 -11

@@ -236,23 +236,13 @@ class KimiK3MultiModalProcessor(BaseMultiModalProcessor[KimiK3ProcessingInfo]):

236 prompt: str,

237 mm_data: Mapping[str, object],

238 mm_kwargs: Mapping[str, object],

239 - tok_kwargs: Mapping[str, object],

239 ) -> BatchFeature:

240 # Override so the base always routes through the text+mm path

241 # (`KimiK3Processor.__call__`). Otherwise the mm-only fast path calls

242 # the checkpoint image processor directly with bare PIL images, but it

243 # requires `{"type": "image", "image": PIL}` media dicts that only our

244 # wrapper builds.

246 - return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)

247 -

248 - def _hf_processor_applies_updates(

249 - self,

250 - prompt_text: str,

251 - mm_items: MultiModalDataItems,

252 - hf_processor_mm_kwargs: Mapping[str, object],

253 - tokenization_kwargs: Mapping[str, object],

254 - ) -> bool:

255 - return False

245 + return super()._call_hf_processor(prompt, mm_data, mm_kwargs)

246

247 def _get_mm_fields_config(

248 self,

vllm/models/minimax_m3/common/mm_preprocess.py

+1 -2

@@ -311,7 +311,6 @@ class MiniMaxM3VLMultiModalProcessor(

311 prompt: str,

312 mm_data: Mapping[str, object],

313 mm_kwargs: Mapping[str, object],

314 - tok_kwargs: Mapping[str, object],

314 ) -> BatchFeature:

315 mm_data = dict(mm_data)

316 # With ``video_needs_metadata=True`` each video arrives as a

@@ -341,7 +340,7 @@ class MiniMaxM3VLMultiModalProcessor(

340

341 # Override the video processor's default do_resize=False (set for a

342 # pre-resized pipeline) to True for vLLM's raw-frame inputs.

344 - merged = dict(do_resize=True, **mm_kwargs, **tok_kwargs)

343 + merged = dict(do_resize=True, **mm_kwargs)

344 data = dict(text=prompt, **mm_data)

345 if video_metadata is not None:

346 data["video_metadata"] = video_metadata

vllm/multimodal/processing/dummy_inputs.py

+13 -3

@@ -83,12 +83,22 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):

83 dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)

84 dummy_mm_items = self.info.parse_mm_data(dummy_mm_data, validate=False)

85

86 - tokenization_kwargs = {"truncation": False}

86 + tokenizer = self.info.ctx.tokenizer

87 + dummy_prompt: list[int]

88 + if tokenizer is None:

89 + # Tokenizer-less models (e.g. `skip_tokenizer_init=True`) only

90 + # accept embeddings and have an empty dummy text, so there are no

91 + # prompt tokens.

92 + dummy_prompt = []

93 + else:

94 + dummy_prompt = tokenizer.encode(

95 + dummy_text,

96 + **self.info.default_tok_params.get_encode_kwargs(),

97 + )

98

99 return ProcessorInputs(

89 - prompt=dummy_text,

100 + prompt=dummy_prompt,

101 mm_data_items=dummy_mm_items,

91 - tokenization_kwargs=tokenization_kwargs,

102 )

103

104 def _get_dummy_audios(

vllm/multimodal/processing/inputs.py

+1 -2

@@ -17,11 +17,10 @@ class ProcessorInputs:

17 [`vllm.multimodal.processing.BaseMultiModalProcessor.apply`][].

18 """

19

20 - prompt: str | list[int]

20 + prompt: list[int]

21 mm_data_items: MultiModalDataItems

22 mm_uuid_items: MultiModalUUIDItems | None = None

23 hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)

24 - tokenization_kwargs: Mapping[str, object] = field(default_factory=dict)

24

25 def get_mm_hashes(

26 self,

vllm/multimodal/processing/processor.py

+23 -103

@@ -38,12 +38,7 @@ from ..inputs import (

38 MultiModalKwargsOptionalItems,

39 PlaceholderRange,

40 )

41 -from ..parse import (

42 - DictEmbeddingItems,

43 - EmbeddingItems,

44 - MultiModalDataItems,

45 - MultiModalUUIDItems,

46 -)

41 +from ..parse import MultiModalDataItems, MultiModalUUIDItems

42 from .context import BaseProcessingInfo, TimingContext

43 from .dummy_inputs import BaseDummyInputsBuilder

44 from .inputs import ProcessorInputs

@@ -1199,11 +1194,18 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1194

1195 def __call__(

1196 self,

1202 - prompt: str,

1197 + prompt: str | list[int],

1198 mm_items: MultiModalDataItems,

1199 mm_uuid_items: MultiModalUUIDItems | None = None,

1200 hf_processor_mm_kwargs: Mapping[str, object] | None = None,

1201 ) -> MultiModalInput:

1202 + if isinstance(prompt, str):

1203 + tokenizer = self.info.get_tokenizer()

1204 + prompt = tokenizer.encode(

1205 + prompt,

1206 + **self.info.default_tok_params.get_encode_kwargs(),

1207 + )

1208 +

1209 processor_inputs = ProcessorInputs(

1210 prompt,

1211 mm_items,

@@ -1307,7 +1309,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1309 # This refers to the data to be passed to HF processor.

1310 mm_data: Mapping[str, object],

1311 mm_kwargs: Mapping[str, object],

1310 - tok_kwargs: Mapping[str, object],

1312 ) -> BatchFeature:

1313 """

1314 Call the HF processor on the prompt text and

@@ -1316,26 +1317,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1317 return self.info.ctx.call_hf_processor(

1318 self.info.get_hf_processor(**mm_kwargs),

1319 dict(text=prompt, **mm_data),

1319 - dict(**mm_kwargs, **tok_kwargs),

1320 - )

1321 -

1322 - def _hf_processor_applies_updates(

1323 - self,

1324 - prompt_text: str,

1325 - mm_items: MultiModalDataItems,

1326 - hf_processor_mm_kwargs: Mapping[str, object],

1327 - tokenization_kwargs: Mapping[str, object],

1328 - ) -> bool:

1329 - """

1330 - Return whether the HF processor applies prompt updates.

1331 -

1332 - For most HF processors, this should be `True` when multi-modal

1333 - data items are passed, but `False` when multi-modal embeddings

1334 - are passed.

1335 - """

1336 - return not any(

1337 - isinstance(items, (EmbeddingItems, DictEmbeddingItems))

1338 - for items in mm_items.values()

1320 + mm_kwargs,

1321 )

1322

1323 def _apply_hf_processor_text_mm(

@@ -1343,13 +1325,10 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1325 prompt_text: str,

1326 mm_items: MultiModalDataItems,

1327 hf_processor_mm_kwargs: Mapping[str, object],

1346 - tokenization_kwargs: Mapping[str, object],

1347 - ) -> tuple[list[int], BatchFeature, bool]:

1328 + ) -> tuple[list[int], BatchFeature]:

1329 """

1330 Apply the HF processor on the prompt text and multi-modal data

1331 together.

1351 -

1352 - In addition, return whether prompt updates have been applied.

1332 """

1333 valid_mm_items = mm_items.select(

1334 {k for k, c in mm_items.get_all_counts().items() if c > 0}

@@ -1360,7 +1339,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1339 prompt=prompt_text,

1340 mm_data=processor_data,

1341 mm_kwargs=hf_processor_mm_kwargs,

1363 - tok_kwargs=tokenization_kwargs,

1342 )

1343 processed_data.update(passthrough_data)

1344

@@ -1370,35 +1348,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1348

1349 (prompt_ids,) = input_ids

1350

1373 - is_update_applied = self._hf_processor_applies_updates(

1374 - prompt_text=prompt_text,

1375 - mm_items=mm_items,

1376 - hf_processor_mm_kwargs=hf_processor_mm_kwargs,

1377 - tokenization_kwargs=tokenization_kwargs,

1378 - )

1379 -

1380 - return prompt_ids, processed_data, is_update_applied

1381 -

1382 - def _apply_hf_processor_text_only(

1383 - self,

1384 - prompt_text: str,

1385 - tokenization_kwargs: Mapping[str, object],

1386 - ) -> list[int]:

1387 - """

1388 - Apply the HF processor on the prompt text only.

1389 -

1390 - Since HF processor requires that text and multi-modal items

1391 - correspond to each other, we create dummy multi-modal items

1392 - to go along with the text.

1393 - """

1394 - prompt_ids, _, _ = self._apply_hf_processor_text_mm(

1395 - prompt_text=prompt_text,

1396 - mm_items=MultiModalDataItems({}),

1397 - hf_processor_mm_kwargs={},

1398 - tokenization_kwargs=tokenization_kwargs,

1399 - )

1400 -

1401 - return prompt_ids

1351 + return prompt_ids, processed_data

1352

1353 def _apply_hf_processor_tokens_only(

1354 self,

@@ -1410,10 +1360,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1360 Most HF processors accept prompt text but not prompt tokens.

1361 If the HF processor adds or removes tokens that are not related to

1362 multi-modal data, you should override this method so it is consistent

1413 - with the output of

1414 - [`_apply_hf_processor_text_only`][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_text_only]

1415 - on the

1416 - corresponding text.

1363 + with the tokenization of the corresponding text.

1364 """

1365 return prompt_tokens

1366

@@ -1421,7 +1368,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1368 self,

1369 mm_items: MultiModalDataItems,

1370 hf_processor_mm_kwargs: Mapping[str, object],

1424 - tokenization_kwargs: Mapping[str, object],

1371 ) -> BatchFeature:

1372 """

1373 Apply the HF processor on the multi-modal data only.

@@ -1435,11 +1381,10 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1381 if type(self)._call_hf_processor != BaseMultiModalProcessor._call_hf_processor:

1382 mm_counts = mm_items.get_all_counts()

1383

1438 - _, mm_processed_data, _ = self._apply_hf_processor_text_mm(

1384 + _, mm_processed_data = self._apply_hf_processor_text_mm(

1385 prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),

1386 mm_items=mm_items,

1387 hf_processor_mm_kwargs=hf_processor_mm_kwargs,

1442 - tokenization_kwargs=tokenization_kwargs,

1388 )

1389

1390 return mm_processed_data

@@ -1455,7 +1400,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1400 self.info.get_hf_processor(**hf_processor_mm_kwargs),

1401 ),

1402 processor_data,

1458 - dict(**hf_processor_mm_kwargs, **tokenization_kwargs),

1403 + hf_processor_mm_kwargs,

1404 )

1405 processed_data.update(passthrough_data)

1406

@@ -1463,41 +1408,22 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1408

1409 def _apply_hf_processor_main(

1410 self,

1466 - prompt: str | list[int],

1411 + prompt: list[int],

1412 mm_items: MultiModalDataItems,

1413 hf_processor_mm_kwargs: Mapping[str, object],

1469 - tokenization_kwargs: Mapping[str, object],

1470 - *,

1471 - enable_hf_prompt_update: bool,

1414 ) -> tuple[list[int], BatchFeature, bool]:

1415 """

1474 - Apply the HF processor on the prompt text and multi-modal data.

1416 + Apply the HF processor on the prompt tokens and multi-modal data.

1417

1418 In addition, return whether prompt updates have been applied

1477 - (for most HF processors, this should be `True`).

1478 -

1479 - Note:

1480 - If `enable_hf_prompt_update=False`, we use HF processor

1481 - to perform prompt updates if available; HF processor requires

1482 - that the prompt corresponds to multi-modal items.

1419 + (`False` here, since for token prompts the updates are normally

1420 + applied separately via `_maybe_apply_prompt_updates`).

1421 """

1484 - if isinstance(prompt, str):

1485 - if enable_hf_prompt_update:

1486 - return self._apply_hf_processor_text_mm(

1487 - prompt_text=prompt,

1488 - mm_items=mm_items,

1489 - hf_processor_mm_kwargs=hf_processor_mm_kwargs,

1490 - tokenization_kwargs=tokenization_kwargs,

1491 - )

1492 -

1493 - prompt_ids = self._apply_hf_processor_text_only(prompt, tokenization_kwargs)

1494 - else:

1495 - prompt_ids = self._apply_hf_processor_tokens_only(prompt)

1422 + prompt_ids = self._apply_hf_processor_tokens_only(prompt)

1423

1424 mm_processed_data = self._apply_hf_processor_mm_only(

1425 mm_items=mm_items,

1426 hf_processor_mm_kwargs=hf_processor_mm_kwargs,

1500 - tokenization_kwargs=tokenization_kwargs,

1427 )

1428

1429 return prompt_ids, mm_processed_data, False

@@ -1615,8 +1541,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1541 prompt=inputs.prompt,

1542 mm_items=inputs.mm_data_items,

1543 hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,

1618 - tokenization_kwargs=inputs.tokenization_kwargs,

1619 - enable_hf_prompt_update=True,

1544 )

1545

1546 mm_kwargs = MultiModalKwargsItems.from_hf_inputs(

@@ -1687,8 +1611,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1611 prompt=inputs.prompt,

1612 mm_items=mm_missing_data_items,

1613 hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,

1690 - tokenization_kwargs=inputs.tokenization_kwargs,

1691 - enable_hf_prompt_update=False,

1614 )

1615

1616 mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs(

@@ -1938,7 +1860,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1860 is_update_applied,

1861 ) = self._cached_apply_hf_processor(inputs, timing_ctx)

1862

1941 - # NOTE: tokenization_kwargs are not required to init processor

1863 with timing_ctx.record("apply_prompt_updates"):

1864 prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(

1865 mm_items=inputs.mm_data_items,

@@ -1967,9 +1888,9 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):

1888 @abstractmethod

1889 def create_encoder_prompt(

1890 self,

1970 - prompt: str | list[int],

1891 + prompt: list[int],

1892 mm_items: MultiModalDataItems,

1972 - ) -> str | list[int]:

1893 + ) -> list[int]:

1894 """

1895 Create input prompt for the encoder. HF processor will be applied on

1896 this prompt during profiling and generation.

@@ -1986,7 +1907,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):

1907

1908 def _get_enc_dec_inputs(

1909 self,

1989 - prompt: str | list[int],

1910 + prompt: list[int],

1911 mm_items: MultiModalDataItems,

1912 encoder_inputs: MultiModalInput,

1913 ):

@@ -2028,7 +1949,6 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):

1949 inputs.mm_data_items,

1950 inputs.mm_uuid_items,

1951 hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,

2031 - tokenization_kwargs=inputs.tokenization_kwargs,

1952 )

1953

1954 encoder_inputs = super().apply(encoder_processor_inputs, timing_ctx)

[MM] Remove text components from ProcessorInputs (#53093)