@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d8cf819a9a

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 03:30 UTC

[Core] [Bugfix] [Multimodal] Fix multimodal profiling and generation for SFT/PTQed models (#20058)

Signed-off-by: Kyle Sayers <kylesayrs@gmail.com>

Kyle Sayers committed Jun 30, 2025 at 13:26 UTC d8cf819a9a337a578b7dfc7642617921cc468c17

41 files changed +207 -38

docs/contributing/model/multimodal.md

@@ -538,11 +538,13 @@ return a schema of the tensors outputted by the HF processor that are related to

538 prompt: str,

539 mm_data: Mapping[str, object],

540 mm_kwargs: Mapping[str, object],

541 + tok_kwargs: Mapping[str, object],

542 ) -> BatchFeature:

543 processed_outputs = super()._call_hf_processor(

544 prompt=prompt,

545 mm_data=mm_data,

546 mm_kwargs=mm_kwargs,

547 + tok_kwargs=tok_kwargs,

548 )

549

550 image_patches = processed_outputs.get("image_patches")

@@ -566,6 +568,11 @@ return a schema of the tensors outputted by the HF processor that are related to

568 Our [actual code](gh-file:vllm/model_executor/models/fuyu.py) has special handling

569 for text-only inputs to prevent unnecessary warnings from HF processor.

570

571 + !!! note

572 + The `_call_hf_processor` method specifies both `mm_kwargs` and `tok_kwargs` for

573 + processing. `mm_kwargs` is used to both initialize and call the huggingface

574 + processor, whereas `tok_kwargs` is only used to call the huggingface processor.

575 +

576 This lets us override [_get_mm_fields_config][vllm.multimodal.processing.BaseMultiModalProcessor._get_mm_fields_config] as follows:

577

578 ```python

tests/multimodal/test_processing.py

@@ -1086,6 +1086,7 @@ def test_hf_processor_kwargs(model_id, call_kwargs, expected_kwargs):

1086 prompt="",

1087 mm_data={},

1088 mm_kwargs=call_kwargs,

1089 + tok_kwargs={},

1090 )

1091

1092 assert out_kwargs == expected_kwargs

vllm/entrypoints/llm.py

@@ -481,6 +481,13 @@ class LLM:

481 # Use default sampling params.

482 sampling_params = self.get_default_sampling_params()

483

484 + tokenization_kwargs: dict[str, Any] = {}

485 + truncate_prompt_tokens = None

486 + if isinstance(sampling_params, SamplingParams):

487 + truncate_prompt_tokens = sampling_params.truncate_prompt_tokens

488 + _validate_truncation_size(self.llm_engine.model_config.max_model_len,

489 + truncate_prompt_tokens, tokenization_kwargs)

490 +

491 self._validate_and_add_requests(

492 prompts=parsed_prompts,

493 params=sampling_params,

@@ -488,6 +495,7 @@ class LLM:

495 lora_request=lora_request,

496 prompt_adapter_request=prompt_adapter_request,

497 guided_options=guided_options_request,

498 + tokenization_kwargs=tokenization_kwargs,

499 priority=priority,

500 )

501

vllm/entrypoints/utils.py

@@ -171,6 +171,10 @@ def _validate_truncation_size(

171 tokenization_kwargs["truncation"] = True

172 tokenization_kwargs["max_length"] = truncate_prompt_tokens

173

174 + else:

175 + if tokenization_kwargs is not None:

176 + tokenization_kwargs["truncation"] = False

177 +

178 return truncate_prompt_tokens

179

180

vllm/inputs/preprocess.py

+22 -7

@@ -265,7 +265,8 @@ class InputPreprocessor:

265 prompt: Union[str, list[int]],

266 mm_data: MultiModalDataDict,

267 mm_processor_kwargs: Optional[Mapping[str, object]],

268 - lora_request: Optional[LoRARequest],

268 + tokenization_kwargs: Optional[dict[str, Any]] = None,

269 + lora_request: Optional[LoRARequest] = None,

270 return_mm_hashes: bool = False,

271 ) -> MultiModalInputs:

272 """

@@ -280,15 +281,19 @@ class InputPreprocessor:

281 if mm_processor_kwargs is None:

282 mm_processor_kwargs = {}

283

283 - return mm_processor.apply(prompt, mm_data, mm_processor_kwargs,

284 - return_mm_hashes)

284 + return mm_processor.apply(prompt,

285 + mm_data,

286 + hf_processor_mm_kwargs=mm_processor_kwargs,

287 + tokenization_kwargs=tokenization_kwargs,

288 + return_mm_hashes=return_mm_hashes)

289

290 async def _process_multimodal_async(

291 self,

292 prompt: Union[str, list[int]],

293 mm_data: MultiModalDataDict,

294 mm_processor_kwargs: Optional[Mapping[str, object]],

291 - lora_request: Optional[LoRARequest],

295 + tokenization_kwargs: Optional[dict[str, Any]] = None,

296 + lora_request: Optional[LoRARequest] = None,

297 return_mm_hashes: bool = False,

298 ) -> MultiModalInputs:

299 """

@@ -302,8 +307,11 @@ class InputPreprocessor:

307 if mm_processor_kwargs is None:

308 mm_processor_kwargs = {}

309

305 - return mm_processor.apply(prompt, mm_data, mm_processor_kwargs,

306 - return_mm_hashes)

310 + return mm_processor.apply(prompt,

311 + mm_data,

312 + hf_processor_mm_kwargs=mm_processor_kwargs,

313 + tokenization_kwargs=tokenization_kwargs,

314 + return_mm_hashes=return_mm_hashes)

315

316 def _process_embeds(

317 self,

@@ -338,6 +346,7 @@ class InputPreprocessor:

346 def _process_tokens(

347 self,

348 parsed_content: TokensPrompt,

349 + tokenization_kwargs: Optional[dict[str, Any]] = None,

350 lora_request: Optional[LoRARequest] = None,

351 return_mm_hashes: bool = False,

352 ) -> Union[TokenInputs, MultiModalInputs]:

@@ -350,6 +359,7 @@ class InputPreprocessor:

359 prompt_token_ids,

360 multi_modal_data,

361 parsed_content.get("mm_processor_kwargs"),

362 + tokenization_kwargs=tokenization_kwargs,

363 lora_request=lora_request,

364 return_mm_hashes=return_mm_hashes,

365 )

@@ -367,6 +377,7 @@ class InputPreprocessor:

377 async def _process_tokens_async(

378 self,

379 parsed_content: TokensPrompt,

380 + tokenization_kwargs: Optional[dict[str, Any]] = None,

381 lora_request: Optional[LoRARequest] = None,

382 return_mm_hashes: bool = False,

383 ) -> Union[TokenInputs, MultiModalInputs]:

@@ -379,6 +390,7 @@ class InputPreprocessor:

390 prompt_token_ids,

391 multi_modal_data,

392 parsed_content.get("mm_processor_kwargs"),

393 + tokenization_kwargs=tokenization_kwargs,

394 lora_request=lora_request,

395 return_mm_hashes=return_mm_hashes,

396 )

@@ -408,6 +420,7 @@ class InputPreprocessor:

420 prompt_text,

421 multi_modal_data,

422 parsed_content.get("mm_processor_kwargs"),

423 + tokenization_kwargs=tokenization_kwargs,

424 lora_request=lora_request,

425 return_mm_hashes=return_mm_hashes,

426 )

@@ -442,6 +455,7 @@ class InputPreprocessor:

455 prompt_text,

456 multi_modal_data,

457 parsed_content.get("mm_processor_kwargs"),

458 + tokenization_kwargs=tokenization_kwargs,

459 lora_request=lora_request,

460 return_mm_hashes=return_mm_hashes,

461 )

@@ -860,7 +874,8 @@ class InputPreprocessor:

874 "returned until they are supported on vLLM V1.")

875 # Encoder-decoder model requires special mapping of

876 # input prompts to encoder & decoder

863 - return self._process_encoder_decoder_prompt(prompt)

877 + return self._process_encoder_decoder_prompt(

878 + prompt, tokenization_kwargs)

879

880 if is_explicit_encoder_decoder_prompt(prompt):

881 raise ValueError("Cannot pass encoder-decoder prompt "

vllm/model_executor/models/aya_vision.py

@@ -185,11 +185,13 @@ class AyaVisionMultiModalProcessor(

185 prompt: str,

186 mm_data: Mapping[str, object],

187 mm_kwargs: Mapping[str, object],

188 + tok_kwargs: Mapping[str, object],

189 ) -> BatchFeature:

190 processed_outputs = super()._call_hf_processor(

191 prompt,

192 mm_data,

193 mm_kwargs,

194 + tok_kwargs,

195 )

196 hf_processor = self.info.get_hf_processor(**mm_kwargs)

197 image_processor = hf_processor.image_processor

vllm/model_executor/models/blip2.py

@@ -454,6 +454,7 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):

454 prompt: str,

455 mm_data: Mapping[str, object],

456 mm_kwargs: Mapping[str, object],

457 + tok_kwargs: Mapping[str, object],

458 ) -> BatchFeature:

459 if not mm_data:

460 # HF processor always adds placeholders even when there's no image

@@ -465,6 +466,7 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):

466 prompt=prompt,

467 mm_data=mm_data,

468 mm_kwargs=mm_kwargs,

469 + tok_kwargs=tok_kwargs,

470 )

471

472 def _get_mm_fields_config(

vllm/model_executor/models/chameleon.py

@@ -107,6 +107,7 @@ class ChameleonMultiModalProcessor(

107 prompt: str,

108 mm_data: Mapping[str, object],

109 mm_kwargs: Mapping[str, object],

110 + tok_kwargs: Mapping[str, object],

111 ) -> BatchFeature:

112 if not mm_data:

113 prompt_ids = self.info.get_tokenizer().encode(prompt)

@@ -117,6 +118,7 @@ class ChameleonMultiModalProcessor(

118 prompt=prompt,

119 mm_data=mm_data,

120 mm_kwargs=mm_kwargs,

121 + tok_kwargs=tok_kwargs,

122 )

123

124 def _apply_hf_processor_tokens_only(

vllm/model_executor/models/deepseek_vl2.py

+5 -1

@@ -204,12 +204,13 @@ class DeepseekVL2MultiModalProcessor(

204 prompt: str,

205 mm_data: Mapping[str, object],

206 mm_kwargs: Mapping[str, object],

207 + tok_kwargs: Mapping[str, object],

208 ) -> BatchFeature:

209 if mm_data:

210 processed_outputs = self.info.ctx.call_hf_processor(

211 self.info.get_hf_processor(**mm_kwargs),

212 dict(prompt=prompt, **mm_data),

212 - mm_kwargs,

213 + dict(**mm_kwargs, **tok_kwargs),

214 )

215 pixel_values = processed_outputs["pixel_values"]

216 # split pixel values into patches corresponding to each image

@@ -278,6 +279,7 @@ class DeepseekVL2MultiModalProcessor(

279 prompt: Union[str, list[int]],

280 mm_data_items: MultiModalDataItems,

281 hf_processor_mm_kwargs: Mapping[str, object],

282 + tokenization_kwargs: Mapping[str, object],

283 *,

284 return_mm_hashes: bool,

285 ) -> tuple[list[int], MultiModalKwargs, Optional[MultiModalHashes], bool]:

@@ -290,6 +292,7 @@ class DeepseekVL2MultiModalProcessor(

292 prompt=prompt,

293 mm_data_items=mm_data_items,

294 hf_processor_mm_kwargs=hf_processor_mm_kwargs,

295 + tokenization_kwargs=tokenization_kwargs,

296 return_mm_hashes=return_mm_hashes,

297 )

298

@@ -297,6 +300,7 @@ class DeepseekVL2MultiModalProcessor(

300 prompt=prompt,

301 mm_data_items=mm_data_items,

302 hf_processor_mm_kwargs=hf_processor_mm_kwargs,

303 + tokenization_kwargs=tokenization_kwargs,

304 return_mm_hashes=return_mm_hashes,

305 )

306

vllm/model_executor/models/florence2.py

+3 -1

@@ -794,6 +794,7 @@ class Florence2MultiModalProcessor(

794 prompt_text: str,

795 mm_items: MultiModalDataItems,

796 hf_processor_mm_kwargs: Mapping[str, object],

797 + tokenization_kwargs: Mapping[str, object],

798 ) -> bool:

799 return False

800

@@ -828,10 +829,11 @@ class Florence2MultiModalProcessor(

829 prompt: str,

830 mm_data: Mapping[str, object],

831 mm_kwargs: Mapping[str, object],

832 + tok_kwargs: Mapping[str, object],

833 ) -> BatchFeature:

834 if mm_data:

835 processed_outputs = super()._call_hf_processor(

834 - prompt, mm_data, mm_kwargs)

836 + prompt, mm_data, mm_kwargs, tok_kwargs)

837 else:

838 hf_processor = self.info.get_hf_processor()

839 tokenizer = hf_processor.tokenizer

vllm/model_executor/models/fuyu.py

@@ -153,6 +153,7 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):

153 prompt: str,

154 mm_data: Mapping[str, object],

155 mm_kwargs: Mapping[str, object],

156 + tok_kwargs: Mapping[str, object],

157 ) -> BatchFeature:

158 if not mm_data:

159 # Avoid warning from HF logger for text-only input

@@ -164,6 +165,7 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):

165 prompt=prompt,

166 mm_data=mm_data,

167 mm_kwargs=mm_kwargs,

168 + tok_kwargs=tok_kwargs,

169 )

170

171 image_patches = processed_outputs.get("image_patches")

vllm/model_executor/models/gemma3_mm.py

@@ -259,11 +259,13 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):

259 prompt: str,

260 mm_data: Mapping[str, object],

261 mm_kwargs: Mapping[str, object],

262 + tok_kwargs: Mapping[str, object],

263 ) -> BatchFeature:

264 processed_outputs = super()._call_hf_processor(

265 prompt,

266 mm_data,

267 mm_kwargs,

268 + tok_kwargs,

269 )

270

271 # HF processor pops the `num_crops` kwarg, which is needed by vLLM

vllm/model_executor/models/glm4v.py

@@ -481,6 +481,7 @@ class GLM4VMultiModalProcessor(BaseMultiModalProcessor[GLM4VProcessingInfo]):

481 prompt_text: str,

482 mm_items: MultiModalDataItems,

483 hf_processor_mm_kwargs: Mapping[str, object],

484 + tokenization_kwargs: Mapping[str, object],

485 ) -> bool:

486 return False

487

vllm/model_executor/models/granite_speech.py

@@ -141,6 +141,7 @@ class GraniteSpeechMultiModalProcessor(

141 prompt: str,

142 mm_data: Mapping[str, object],

143 mm_kwargs: Mapping[str, object],

144 + tok_kwargs: Mapping[str, object],

145 ) -> BatchFeature:

146 mm_data = dict(mm_data)

147 audios = mm_data.pop("audios", [])

@@ -153,6 +154,7 @@ class GraniteSpeechMultiModalProcessor(

154 prompt=prompt,

155 mm_data=mm_data,

156 mm_kwargs=mm_kwargs,

157 + tok_kwargs=tok_kwargs,

158 )

159

160 if "audio" in mm_data:

vllm/model_executor/models/h2ovl.py

@@ -490,6 +490,7 @@ class H2OVLMultiModalProcessor(

490 prompt: Union[str, list[int]],

491 mm_data_items: MultiModalDataItems,

492 hf_processor_mm_kwargs: Mapping[str, object],

493 + tokenization_kwargs: Mapping[str, object],

494 *,

495 return_mm_hashes: bool,

496 ) -> tuple[list[int], MultiModalKwargs, Optional[MultiModalHashes], bool]:

@@ -502,6 +503,7 @@ class H2OVLMultiModalProcessor(

503 prompt=prompt,

504 mm_data_items=mm_data_items,

505 hf_processor_mm_kwargs=hf_processor_mm_kwargs,

506 + tokenization_kwargs=tokenization_kwargs,

507 return_mm_hashes=return_mm_hashes,

508 )

509

@@ -509,6 +511,7 @@ class H2OVLMultiModalProcessor(

511 prompt=prompt,

512 mm_data_items=mm_data_items,

513 hf_processor_mm_kwargs=hf_processor_mm_kwargs,

514 + tokenization_kwargs=tokenization_kwargs,

515 return_mm_hashes=return_mm_hashes,

516 )

517

vllm/model_executor/models/idefics3.py

@@ -326,6 +326,7 @@ class Idefics3MultiModalProcessor(

326 prompt: str,

327 mm_data: Mapping[str, object],

328 mm_kwargs: Mapping[str, object],

329 + tok_kwargs: Mapping[str, object],

330 ) -> BatchFeature:

331 # Text-only input not supported in composite processor

332 if not (images := mm_data.get("images", [])):

@@ -337,6 +338,7 @@ class Idefics3MultiModalProcessor(

338 prompt,

339 mm_data,

340 mm_kwargs,

341 + tok_kwargs,

342 )

343

344 parsed_images = (self._get_data_parser().parse_mm_data({

vllm/model_executor/models/internvl.py

+4 -1

@@ -758,11 +758,13 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):

758 prompt: str,

759 mm_data: Mapping[str, object],

760 mm_kwargs: Mapping[str, object],

761 + tok_kwargs: Mapping[str, object],

762 ) -> Mapping[str, NestedTensors]:

763 processed_outputs = super()._call_hf_processor(

764 prompt=prompt,

765 mm_data=mm_data,

766 mm_kwargs=mm_kwargs,

767 + tok_kwargs=tok_kwargs,

768 )

769

770 hf_processor = self.info.get_hf_processor(**mm_kwargs)

@@ -941,9 +943,10 @@ class InternVLMultiModalProcessor(

943 prompt: str,

944 mm_data: Mapping[str, object],

945 mm_kwargs: Mapping[str, object],

946 + tok_kwargs: Mapping[str, object],

947 ) -> Mapping[str, NestedTensors]:

948 processed_outputs = super()._call_hf_processor(prompt, mm_data,

946 - mm_kwargs)

949 + mm_kwargs, tok_kwargs)

950

951 hf_processor = self.info.get_hf_processor(**mm_kwargs)

952 if self.info.supports_video and (

vllm/model_executor/models/llava.py

+4 -1

@@ -296,11 +296,13 @@ class PixtralHFMultiModalProcessor(

296 prompt: str,

297 mm_data: Mapping[str, object],

298 mm_kwargs: Mapping[str, object],

299 + tok_kwargs: Mapping[str, object],

300 ) -> BatchFeature:

301 processed_outputs = super()._call_hf_processor(

302 prompt=prompt,

303 mm_data=mm_data,

304 mm_kwargs=mm_kwargs,

305 + tok_kwargs=tok_kwargs,

306 )

307

308 pixel_values = processed_outputs.get("pixel_values")

@@ -797,6 +799,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):

799 prompt: Union[str, list[int]],

800 mm_data: MultiModalDataDict,

801 hf_processor_mm_kwargs: Mapping[str, object],

802 + tokenization_kwargs: Optional[Mapping[str, object]] = None,

803 return_mm_hashes: bool = False,

804 ) -> MultiModalInputs:

805 hf_config = self.info.get_hf_config()

@@ -809,7 +812,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):

812 )

813

814 result = super().apply(prompt, mm_data, hf_processor_mm_kwargs,

812 - return_mm_hashes)

815 + tokenization_kwargs, return_mm_hashes)

816

817 mm_items = self._to_mm_items(mm_data)

818 mm_item_counts = mm_items.get_all_counts()

vllm/model_executor/models/llava_onevision.py

@@ -286,6 +286,7 @@ class LlavaOnevisionMultiModalProcessor(

286 prompt: str,

287 mm_data: Mapping[str, object],

288 mm_kwargs: Mapping[str, object],

289 + tok_kwargs: Mapping[str, object],

290 ) -> BatchFeature:

291 mm_data = dict(mm_data)

292 videos = mm_data.pop("videos", [])

@@ -296,6 +297,7 @@ class LlavaOnevisionMultiModalProcessor(

297 prompt=prompt,

298 mm_data=mm_data,

299 mm_kwargs=mm_kwargs,

300 + tok_kwargs=tok_kwargs,

301 )

302

303 # LLaVA-OneVision processor doesn't support multiple videos

@@ -310,6 +312,7 @@ class LlavaOnevisionMultiModalProcessor(

312 prompt=prompt,

313 mm_data={},

314 mm_kwargs=mm_kwargs,

315 + tok_kwargs=tok_kwargs,

316 )

317

318 images = mm_data.pop("images", [])

@@ -319,6 +322,7 @@ class LlavaOnevisionMultiModalProcessor(

322 prompt=image_token * len(images),

323 mm_data={"images": images},

324 mm_kwargs=mm_kwargs,

325 + tok_kwargs=tok_kwargs,

326 )

327 image_outputs = {

328 k: v

@@ -334,6 +338,7 @@ class LlavaOnevisionMultiModalProcessor(

338 prompt=video_token,

339 mm_data={"videos": video},

340 mm_kwargs=mm_kwargs,

341 + tok_kwargs=tok_kwargs,

342 )

343

344 pixel_values_videos.append(item_outputs["pixel_values_videos"][0])

@@ -352,11 +357,13 @@ class LlavaOnevisionMultiModalProcessor(

357 prompt_text: str,

358 mm_items: MultiModalDataItems,

359 hf_processor_mm_kwargs: Mapping[str, object],

360 + tokenization_kwargs: Mapping[str, object],

361 ) -> bool:

362 base_result = super()._hf_processor_applies_updates(

363 prompt_text=prompt_text,

364 mm_items=mm_items,

365 hf_processor_mm_kwargs=hf_processor_mm_kwargs,

366 + tokenization_kwargs=tokenization_kwargs,

367 )

368

369 return base_result and mm_items.get_count("video", strict=False) == 0

vllm/model_executor/models/minicpmo.py

+6 -4

@@ -260,6 +260,7 @@ class MiniCPMOMultiModalProcessor(

260 self,

261 mm_data: Mapping[str, object],

262 mm_kwargs: Mapping[str, object],

263 + tok_kwargs: Mapping[str, object],

264 ) -> Mapping[str, NestedTensors]:

265 if (audios := mm_data.get("audios")) is None:

266 return {}

@@ -276,9 +277,9 @@ class MiniCPMOMultiModalProcessor(

277 prompts=[self.info.audio_pattern] * len(parsed_audios),

278 mm_data={"audios": [[audio] for audio in parsed_audios]},

279 mm_kwargs={

279 - **mm_kwargs,

280 - "chunk_input": True,

280 + **mm_kwargs, "chunk_input": True

281 },

282 + tok_kwargs=tok_kwargs,

283 out_keys={"audio_features", "audio_feature_lens"},

284 )

285

@@ -302,10 +303,11 @@ class MiniCPMOMultiModalProcessor(

303 self,

304 mm_data: Mapping[str, object],

305 mm_kwargs: Mapping[str, object],

306 + tok_kwargs: Mapping[str, object],

307 ) -> Mapping[str, NestedTensors]:

308 return {

307 - **super().process_mm_inputs(mm_data, mm_kwargs),

308 - **self.process_audios(mm_data, mm_kwargs),

309 + **super().process_mm_inputs(mm_data, mm_kwargs, tok_kwargs),

310 + **self.process_audios(mm_data, mm_kwargs, tok_kwargs),

311 }

312

313 def _get_prompt_updates(

vllm/model_executor/models/minicpmv.py

+14 -4

@@ -534,6 +534,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

534 self,

535 mm_data: Mapping[str, object],

536 mm_kwargs: Mapping[str, object],

537 + tok_kwargs: Mapping[str, object],

538 ) -> Mapping[str, NestedTensors]:

539 if (images := mm_data.get("images")) is None:

540 return {}

@@ -550,6 +551,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

551 prompts=[self.info.image_pattern] * len(parsed_images),

552 mm_data={"images": [[image] for image in parsed_images]},

553 mm_kwargs=mm_kwargs,

554 + tok_kwargs=tok_kwargs,

555 out_keys={"pixel_values", "image_sizes", "tgt_sizes"},

556 )

557

@@ -563,6 +565,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

565 self,

566 mm_data: Mapping[str, object],

567 mm_kwargs: Mapping[str, object],

568 + tok_kwargs: Mapping[str, object],

569 ) -> Mapping[str, NestedTensors]:

570 if (videos := mm_data.get("videos")) is None:

571 return {}

@@ -586,6 +589,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

589 "max_slice_nums":

590 self.info.get_video_max_slice_num(),

591 },

592 + tok_kwargs=tok_kwargs,

593 out_keys={"pixel_values", "image_sizes", "tgt_sizes"},

594 )

595

@@ -601,10 +605,11 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

605 self,

606 mm_data: Mapping[str, object],

607 mm_kwargs: Mapping[str, object],

608 + tok_kwargs: Mapping[str, object],

609 ) -> Mapping[str, NestedTensors]:

610 return {

606 - **self.process_images(mm_data, mm_kwargs),

607 - **self.process_videos(mm_data, mm_kwargs),

611 + **self.process_images(mm_data, mm_kwargs, tok_kwargs),

612 + **self.process_videos(mm_data, mm_kwargs, tok_kwargs),

613 }

614

615 def _base_call_hf_processor(

@@ -612,6 +617,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

617 prompts: list[str],

618 mm_data: Mapping[str, Sequence[object]],

619 mm_kwargs: Mapping[str, object],

620 + tok_kwargs: Mapping[str, object],

621 *,

622 out_keys: set[str],

623 ) -> dict[str, NestedTensors]:

@@ -621,6 +627,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

627 prompt=prompts, # type: ignore

628 mm_data=mm_data,

629 mm_kwargs=mm_kwargs,

630 + tok_kwargs=tok_kwargs,

631 )

632 else:

633 inputs = defaultdict[str, list[torch.Tensor]](list)

@@ -633,6 +640,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

640 for k, v in mm_data.items()

641 },

642 mm_kwargs=mm_kwargs,

643 + tok_kwargs=tok_kwargs,

644 )

645

646 for k, v in inputs_one.items():

@@ -646,11 +654,12 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

654 prompt: str,

655 mm_data: Mapping[str, object],

656 mm_kwargs: Mapping[str, object],

657 + tok_kwargs: Mapping[str, object],

658 ) -> BatchFeature:

659 tokenizer = self.info.get_tokenizer()

660

652 - input_ids = torch.tensor([tokenizer.encode(prompt)])

653 - mm_inputs = self.process_mm_inputs(mm_data, mm_kwargs)

661 + input_ids = torch.tensor([tokenizer.encode(prompt, **tok_kwargs)])

662 + mm_inputs = self.process_mm_inputs(mm_data, mm_kwargs, tok_kwargs)

663

664 return BatchFeature({

665 "input_ids": input_ids,

@@ -662,6 +671,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

671 prompt_text: str,

672 mm_items: MultiModalDataItems,

673 hf_processor_mm_kwargs: Mapping[str, object],

674 + tokenization_kwargs: Mapping[str, object],

675 ) -> bool:

676 return False

677

vllm/model_executor/models/minimax_vl_01.py

@@ -113,11 +113,13 @@ class MiniMaxVL01MultiModalProcessor(

113 prompt: str,

114 mm_data: Mapping[str, object],

115 mm_kwargs: Mapping[str, object],

116 + tok_kwargs: Mapping[str, object],

117 ) -> BatchFeature:

118 processed_outputs = super()._call_hf_processor(

119 prompt=prompt,

120 mm_data=mm_data,

121 mm_kwargs=mm_kwargs,

122 + tok_kwargs=tok_kwargs,

123 )

124

125 pixel_values = processed_outputs.get("pixel_values")

vllm/model_executor/models/mistral3.py

@@ -228,11 +228,13 @@ class Mistral3MultiModalProcessor(

228 prompt: str,

229 mm_data: Mapping[str, object],

230 mm_kwargs: Mapping[str, object],

231 + tok_kwargs: Mapping[str, object],

232 ) -> BatchFeature:

233 processed_outputs = super()._call_hf_processor(

234 prompt=prompt,

235 mm_data=mm_data,

236 mm_kwargs=mm_kwargs,

237 + tok_kwargs=tok_kwargs,

238 )

239

240 pixel_values = processed_outputs.get("pixel_values")

vllm/model_executor/models/mllama.py

+4 -2

@@ -166,10 +166,11 @@ class MllamaMultiModalProcessor(EncDecMultiModalProcessor[MllamaProcessingInfo]

166 prompt: Union[str, list[int]],

167 mm_data: MultiModalDataDict,

168 hf_processor_mm_kwargs: Mapping[str, object],

169 + tokenization_kwargs: Optional[Mapping[str, object]] = None,

170 return_mm_hashes: bool = False,

171 ) -> MultiModalEncDecInputs:

172 mm_inputs = super().apply(prompt, mm_data, hf_processor_mm_kwargs,

172 - return_mm_hashes)

173 + tokenization_kwargs, return_mm_hashes)

174

175 image_token_id = self.info.get_hf_config().image_token_index

176 # Check that the number of image tokens in the decoder prompt matches

@@ -239,6 +240,7 @@ class MllamaMultiModalProcessor(EncDecMultiModalProcessor[MllamaProcessingInfo]

240 prompt: str,

241 mm_data: Mapping[str, object],

242 mm_kwargs: Mapping[str, object],

243 + tok_kwargs: Mapping[str, object],

244 ) -> BatchFeature:

245 tokenizer = self.info.get_tokenizer()

246 if mm_data:

@@ -247,7 +249,7 @@ class MllamaMultiModalProcessor(EncDecMultiModalProcessor[MllamaProcessingInfo]

249 for img in mm_data["images"]

250 ]

251 processed_outputs = super()._call_hf_processor(

250 - prompt, mm_data, mm_kwargs)

252 + prompt, mm_data, mm_kwargs, tok_kwargs)

253 processed_outputs["num_tiles"] = torch.tensor(num_tiles)

254 for k in ('pixel_values', 'aspect_ratio_ids', "aspect_ratio_mask"):

255 processed_outputs[k] = processed_outputs[k].squeeze(0)

vllm/model_executor/models/mllama4.py

@@ -574,6 +574,7 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo]

574 prompt: str,

575 mm_data: Mapping[str, object],

576 mm_kwargs: Mapping[str, object],

577 + tok_kwargs: Mapping[str, object],

578 ) -> BatchFeature:

579 tokenizer = self.info.get_tokenizer()

580

@@ -583,6 +584,7 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo]

584 prompt=prompt,

585 mm_data=mm_data,

586 mm_kwargs=mm_kwargs,

587 + tok_kwargs=tok_kwargs,

588 )

589

590 processor = self.info.get_hf_processor(**mm_kwargs)

vllm/model_executor/models/ovis.py

@@ -335,6 +335,7 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):

335 prompt: str,

336 mm_data: Mapping[str, object],

337 mm_kwargs: Mapping[str, object],

338 + tok_kwargs: Mapping[str, object],

339 ) -> BatchFeature:

340 if not mm_data:

341 # Avoid warning from HF logger for text-only input

@@ -346,6 +347,7 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):

347 prompt=prompt,

348 mm_data=mm_data,

349 mm_kwargs=mm_kwargs,

350 + tok_kwargs=tok_kwargs,

351 )

352

353 hf_processor = self.info.get_hf_processor()

vllm/model_executor/models/paligemma.py

+4 -1

@@ -121,6 +121,7 @@ class PaliGemmaMultiModalProcessor(

121 prompt: str,

122 mm_data: Mapping[str, object],

123 mm_kwargs: Mapping[str, object],

124 + tok_kwargs: Mapping[str, object],

125 ) -> BatchFeature:

126 tokenizer = self.info.get_tokenizer()

127 if not mm_data:

@@ -131,6 +132,7 @@ class PaliGemmaMultiModalProcessor(

132 prompt=prompt,

133 mm_data=mm_data,

134 mm_kwargs=mm_kwargs,

135 + tok_kwargs=tok_kwargs,

136 )

137

138 def _get_mm_fields_config(

@@ -191,10 +193,11 @@ class PaliGemmaMultiModalProcessor(

193 prompt: Union[str, list[int]],

194 mm_data: MultiModalDataDict,

195 hf_processor_mm_kwargs: Mapping[str, object],

196 + tokenization_kwargs: Optional[Mapping[str, object]] = None,

197 return_mm_hashes: bool = False,

198 ) -> MultiModalInputs:

199 mm_inputs = super().apply(prompt, mm_data, hf_processor_mm_kwargs,

197 - return_mm_hashes)

200 + tokenization_kwargs, return_mm_hashes)

201 prompt_token_ids = mm_inputs["prompt_token_ids"]

202

203 tokenizer = self.info.get_tokenizer()

vllm/model_executor/models/phi3v.py

@@ -376,11 +376,13 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):

376 prompt: str,

377 mm_data: Mapping[str, object],

378 mm_kwargs: Mapping[str, object],

379 + tok_kwargs: Mapping[str, object],

380 ) -> BatchFeature:

381 processed_outputs = super()._call_hf_processor(

382 prompt=prompt,

383 mm_data=mm_data,

384 mm_kwargs=mm_kwargs,

385 + tok_kwargs=tok_kwargs,

386 )

387

388 input_ids = processed_outputs["input_ids"]

vllm/model_executor/models/phi4mm.py

+2 -1

@@ -762,6 +762,7 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):

762 prompt: str,

763 mm_data: Mapping[str, object],

764 mm_kwargs: Mapping[str, object],

765 + tok_kwargs: Mapping[str, object],

766 ) -> BatchFeature:

767 if not mm_data:

768 prompt_ids = self.info.get_tokenizer().encode(prompt)

@@ -773,7 +774,7 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):

774 mm_data['audios'] = [(data, sr) for data in audio_data]

775

776 processed_outputs = super()._call_hf_processor(prompt, mm_data,

776 - mm_kwargs)

777 + mm_kwargs, tok_kwargs)

778

779 num_img_tokens = [

780 self.info.get_num_image_tokens(image_width=img_size[0],

vllm/model_executor/models/pixtral.py

+6 -1

@@ -237,6 +237,7 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):

237 dummy_text = self.get_dummy_text(mm_counts)

238 dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts)

239 dummy_images = dummy_mm_data.get("image", [])

240 + tokenization_kwargs = {"truncation": False}

241

242 request = ChatCompletionRequest(messages=[

243 UserMessage(content=[

@@ -247,7 +248,9 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):

248 res = tokenizer.mistral.encode_chat_completion(request)

249 dummy_tokens = res.tokens

250

250 - return ProcessorInputs(prompt=dummy_tokens, mm_data=dummy_mm_data)

251 + return ProcessorInputs(prompt=dummy_tokens,

252 + mm_data=dummy_mm_data,

253 + tokenization_kwargs=tokenization_kwargs)

254

255

256 class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]

@@ -297,6 +300,7 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]

300 prompt: Union[str, list[int]],

301 mm_data_items: MultiModalDataItems,

302 hf_processor_mm_kwargs: Mapping[str, object],

303 + tokenization_kwargs: Mapping[str, object],

304 *,

305 return_mm_hashes: bool,

306 ) -> tuple[list[int], MultiModalKwargs, Optional[MultiModalHashes], bool]:

@@ -309,6 +313,7 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]

313 prompt=prompt,

314 mm_data_items=mm_data_items,

315 hf_processor_mm_kwargs=hf_processor_mm_kwargs,

316 + tokenization_kwargs=tokenization_kwargs,

317 return_mm_hashes=return_mm_hashes,

318 )

319

vllm/model_executor/models/prithvi_geospatial_mae.py

@@ -92,6 +92,7 @@ class PrithviGeoSpatialMAEMultiModalProcessor(BaseMultiModalProcessor):

92 prompt: Union[str, list[int]],

93 mm_data: MultiModalDataDict,

94 hf_processor_mm_kwargs: Mapping[str, object],

95 + tokenization_kwargs: Optional[Mapping[str, object]] = None,

96 return_mm_hashes: bool = False,

97 ) -> MultiModalInputs:

98 mm_kwargs = {}

vllm/model_executor/models/qwen2_5_omni_thinker.py

@@ -244,6 +244,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(

244 prompt: str,

245 mm_data: Mapping[str, object],

246 mm_kwargs: Mapping[str, object],

247 + tok_kwargs: Mapping[str, object],

248 ) -> BatchFeature:

249 mm_data = dict(mm_data)

250 audios = mm_data.pop("audios", [])

@@ -258,6 +259,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(

259 prompt=prompt,

260 mm_data=mm_data,

261 mm_kwargs=mm_kwargs,

262 + tok_kwargs=tok_kwargs,

263 )

264

265 input_features = hf_inputs.pop('input_features', None)

@@ -453,6 +455,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(

455 prompt: Union[str, list[int]],

456 mm_items: MultiModalDataItems,

457 hf_processor_mm_kwargs: Mapping[str, object],

458 + tokenization_kwargs: Mapping[str, object],

459 *,

460 enable_hf_prompt_update: bool,

461 ) -> tuple[list[int], MultiModalKwargs, bool]:

@@ -465,6 +468,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(

468 prompt_text=prompt,

469 mm_items=mm_items,

470 hf_processor_mm_kwargs=hf_processor_mm_kwargs,

471 + tokenization_kwargs=tokenization_kwargs,

472 )

473 tokenizer = self.info.get_tokenizer()

474 prompt_ids = encode_tokens(tokenizer, prompt)

@@ -474,6 +478,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(

478 mm_kwargs = self._apply_hf_processor_mm_only(

479 mm_items=mm_items,

480 hf_processor_mm_kwargs=hf_processor_mm_kwargs,

481 + tokenization_kwargs=tokenization_kwargs,

482 )

483

484 return prompt_ids, mm_kwargs, False

@@ -482,6 +487,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(

487 self,

488 mm_items: MultiModalDataItems,

489 hf_processor_mm_kwargs: Mapping[str, object],

490 + tokenization_kwargs: Mapping[str, object],

491 ) -> MultiModalKwargs:

492 """

493 Qwen2.5-Omni reimplements this function to handle `use_audio_in_video`.

@@ -498,6 +504,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(

504 prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),

505 mm_items=mm_items,

506 hf_processor_mm_kwargs=hf_processor_mm_kwargs,

507 + tokenization_kwargs=tokenization_kwargs,

508 )

509

510 return mm_kwargs

vllm/model_executor/models/qwen2_audio.py

@@ -150,6 +150,7 @@ class Qwen2AudioMultiModalProcessor(

150 prompt: str,

151 mm_data: Mapping[str, object],

152 mm_kwargs: Mapping[str, Any],

153 + tok_kwargs: Mapping[str, object],

154 ) -> BatchFeature:

155 # NOTE - we rename audios -> audio in mm data because transformers has

156 # deprecated audios for the qwen2audio processor and will remove

@@ -174,6 +175,7 @@ class Qwen2AudioMultiModalProcessor(

175 prompt=prompt,

176 mm_data=mm_data,

177 mm_kwargs=mm_kwargs,

178 + tok_kwargs=tok_kwargs,

179 )

180

181 def _get_mm_fields_config(

vllm/model_executor/models/qwen2_vl.py

+3 -1

@@ -1027,11 +1027,13 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo]

1027 prompt: str,

1028 mm_data: Mapping[str, object],

1029 mm_kwargs: Mapping[str, object],

1030 + tok_kwargs: Mapping[str, object],

1031 ) -> BatchFeature:

1032 + mm_kwargs = self.info._get_image_processor_kwargs(**mm_kwargs)

1033 return self.info.ctx.call_hf_processor(

1034 self.info.get_hf_processor(**mm_kwargs),

1035 dict(text=prompt, **mm_data),

1034 - self.info._get_image_processor_kwargs(**mm_kwargs),

1036 + dict(**mm_kwargs, **tok_kwargs),

1037 )

1038

1039 def _get_prompt_updates(

vllm/model_executor/models/qwen_vl.py

@@ -580,6 +580,7 @@ class QwenVLMultiModalProcessor(BaseMultiModalProcessor[QwenVLProcessingInfo]):

580 prompt: str,

581 mm_data: Mapping[str, object],

582 mm_kwargs: Mapping[str, object],

583 + tok_kwargs: Mapping[str, object],

584 ) -> BatchFeature:

585 # Drops anything between <img>/</img> tags; encoding with the tokenizer

586 # will automatically add the image pads for the context.

@@ -600,6 +601,7 @@ class QwenVLMultiModalProcessor(BaseMultiModalProcessor[QwenVLProcessingInfo]):

601 prompt=prompt,

602 mm_data=mm_data,

603 mm_kwargs=mm_kwargs,

604 + tok_kwargs=tok_kwargs,

605 )

606

607 def _hf_processor_applies_updates(

@@ -607,6 +609,7 @@ class QwenVLMultiModalProcessor(BaseMultiModalProcessor[QwenVLProcessingInfo]):

609 prompt_text: str,

610 mm_items: MultiModalDataItems,

611 hf_processor_mm_kwargs: Mapping[str, object],

612 + tokenization_kwargs: Mapping[str, object],

613 ) -> bool:

614 return False

615

vllm/model_executor/models/skyworkr1v.py

@@ -534,11 +534,13 @@ class SkyworkR1VMultiModalProcessor(BaseMultiModalProcessor[_I]):

534 prompt: str,

535 mm_data: Mapping[str, object],

536 mm_kwargs: Mapping[str, object],

537 + tok_kwargs: Mapping[str, object],

538 ) -> Mapping[str, NestedTensors]:

539 processed_outputs = super()._call_hf_processor(

540 prompt=prompt,

541 mm_data=mm_data,

542 mm_kwargs=mm_kwargs,

543 + tok_kwargs=tok_kwargs,

544 )

545

546 hf_processor = self.info.get_hf_processor(**mm_kwargs)

vllm/model_executor/models/ultravox.py

@@ -144,6 +144,7 @@ class UltravoxMultiModalProcessor(

144 prompt: str,

145 mm_data: Mapping[str, object],

146 mm_kwargs: Mapping[str, object],

147 + tok_kwargs: Mapping[str, object],

148 ) -> BatchFeature:

149 # Text-only input not supported in composite processor

150 if not mm_data.get("audios", []):

@@ -165,10 +166,15 @@ class UltravoxMultiModalProcessor(

166

167 item_processor_data = dict(**mm_data, audios=audios)

168

169 + # some tokenizer kwargs are incompatible with UltravoxProcessor

170 + tok_kwargs.pop("padding", None)

171 + tok_kwargs.pop("truncation", None)

172 +

173 output = super()._call_hf_processor(

174 prompt=prompt,

175 mm_data=item_processor_data,

176 mm_kwargs=mm_kwargs,

177 + tok_kwargs=tok_kwargs,

178 )

179 output['audio_features'] = output.pop('audio_values')

180

vllm/model_executor/models/whisper.py

+3 -1

@@ -700,9 +700,10 @@ class WhisperMultiModalProcessor(

700 prompt: str,

701 mm_data: Mapping[str, object],

702 mm_kwargs: Mapping[str, object],

703 + tok_kwargs: Mapping[str, object],

704 ) -> BatchFeature:

705 if mm_data:

705 - feature_extractor = self.info.get_feature_extractor(**mm_kwargs)

706 + feature_extractor = self.info.get_feature_extractor()

707 mm_data = dict(audio=mm_data.pop("audios"))

708 mm_kwargs = dict(

709 **mm_kwargs,

@@ -712,6 +713,7 @@ class WhisperMultiModalProcessor(

713 prompt=prompt,

714 mm_data=mm_data,

715 mm_kwargs=mm_kwargs,

716 + tok_kwargs=tok_kwargs,

717 )

718 if "labels" in processed_outputs:

719 processed_outputs["input_ids"] = processed_outputs.pop("labels")

vllm/multimodal/processing.py

+43 -11

@@ -1267,6 +1267,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1267 # This refers to the data to be passed to HF processor.

1268 mm_data: Mapping[str, object],

1269 mm_kwargs: Mapping[str, object],

1270 + tok_kwargs: Mapping[str, object],

1271 ) -> "BatchFeature":

1272 """

1273 Call the HF processor on the prompt text and

@@ -1275,7 +1276,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1276 return self.info.ctx.call_hf_processor(

1277 self.info.get_hf_processor(**mm_kwargs),

1278 dict(text=prompt, **mm_data),

1278 - mm_kwargs,

1279 + dict(**mm_kwargs, **tok_kwargs),

1280 )

1281

1282 def _hf_processor_applies_updates(

@@ -1283,6 +1284,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1284 prompt_text: str,

1285 mm_items: MultiModalDataItems,

1286 hf_processor_mm_kwargs: Mapping[str, object],

1287 + tokenization_kwargs: Mapping[str, object],

1288 ) -> bool:

1289 """

1290 Return whether the HF processor applies prompt updates.

@@ -1300,6 +1302,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1302 prompt_text: str,

1303 mm_items: MultiModalDataItems,

1304 hf_processor_mm_kwargs: Mapping[str, object],

1305 + tokenization_kwargs: Mapping[str, object],

1306 ) -> tuple[list[int], MultiModalKwargs, bool]:

1307 """

1308 Apply the HF processor on the prompt text and multi-modal data

@@ -1313,6 +1316,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1316 prompt=prompt_text,

1317 mm_data=processor_data,

1318 mm_kwargs=hf_processor_mm_kwargs,

1319 + tok_kwargs=tokenization_kwargs,

1320 )

1321 processed_data.update(passthrough_data)

1322

@@ -1327,11 +1331,14 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1331 prompt_text=prompt_text,

1332 mm_items=mm_items,

1333 hf_processor_mm_kwargs=hf_processor_mm_kwargs,

1334 + tokenization_kwargs=tokenization_kwargs,

1335 )

1336

1337 return prompt_ids, mm_kwargs, is_update_applied

1338

1334 - def _apply_hf_processor_text_only(self, prompt_text: str) -> list[int]:

1339 + def _apply_hf_processor_text_only(

1340 + self, prompt_text: str,

1341 + tokenization_kwargs: Mapping[str, object]) -> list[int]:

1342 """

1343 Apply the HF processor on the prompt text only.

1344

@@ -1343,6 +1350,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1350 prompt_text=prompt_text,

1351 mm_items=MultiModalDataItems({}),

1352 hf_processor_mm_kwargs={},

1353 + tokenization_kwargs=tokenization_kwargs,

1354 )

1355

1356 return prompt_ids

@@ -1368,6 +1376,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1376 self,

1377 mm_items: MultiModalDataItems,

1378 hf_processor_mm_kwargs: Mapping[str, object],

1379 + tokenization_kwargs: Mapping[str, object],

1380 ) -> MultiModalKwargs:

1381 """

1382 Apply the HF processor on the multi-modal data only.

@@ -1383,6 +1392,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1392 prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),

1393 mm_items=mm_items,

1394 hf_processor_mm_kwargs=hf_processor_mm_kwargs,

1395 + tokenization_kwargs=tokenization_kwargs,

1396 )

1397

1398 return mm_kwargs

@@ -1392,6 +1402,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1402 prompt: Union[str, list[int]],

1403 mm_items: MultiModalDataItems,

1404 hf_processor_mm_kwargs: Mapping[str, object],

1405 + tokenization_kwargs: Mapping[str, object],

1406 *,

1407 enable_hf_prompt_update: bool,

1408 ) -> tuple[list[int], MultiModalKwargs, bool]:

@@ -1412,15 +1423,18 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1423 prompt_text=prompt,

1424 mm_items=mm_items,

1425 hf_processor_mm_kwargs=hf_processor_mm_kwargs,

1426 + tokenization_kwargs=tokenization_kwargs,

1427 )

1428

1417 - prompt_ids = self._apply_hf_processor_text_only(prompt)

1429 + prompt_ids = self._apply_hf_processor_text_only(

1430 + prompt, tokenization_kwargs)

1431 else:

1432 prompt_ids = self._apply_hf_processor_tokens_only(prompt)

1433

1434 mm_kwargs = self._apply_hf_processor_mm_only(

1435 mm_items=mm_items,

1436 hf_processor_mm_kwargs=hf_processor_mm_kwargs,

1437 + tokenization_kwargs=tokenization_kwargs,

1438 )

1439

1440 return prompt_ids, mm_kwargs, False

@@ -1430,14 +1444,17 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1444 cache: ProcessingCache,

1445 mm_data_items: MultiModalDataItems,

1446 hf_processor_mm_kwargs: Mapping[str, object],

1447 + tokenization_kwargs: Mapping[str, object],

1448 ) -> tuple[dict[str, list[ProcessingCacheOptionalItem]], dict[

1449 str, list[object]]]:

1450 model_id = self.info.model_id

1451

1452 mm_cache_items = {

1453 modality: [

1439 - cache.get_item(model_id, modality, item,

1440 - hf_processor_mm_kwargs) for item in items

1454 + cache.get_item(

1455 + model_id, modality, item,

1456 + dict(**hf_processor_mm_kwargs, **tokenization_kwargs))

1457 + for item in items

1458 ]

1459 for modality, items in mm_data_items.items()

1460 }

@@ -1457,10 +1474,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1474 return mm_cache_items, mm_missing_data

1475

1476 def _hash_mm_items(

1460 - self,

1461 - mm_items: MultiModalDataItems,

1462 - hf_processor_mm_kwargs: Mapping[str, object],

1463 - ) -> MultiModalHashes:

1477 + self, mm_items: MultiModalDataItems,

1478 + hf_processor_mm_kwargs: Mapping[str, object],

1479 + tokenization_kwargs: Mapping[str, object]) -> MultiModalHashes:

1480 """Create MM hashes to be returned (only used in V1)."""

1481 model_id = self.info.model_id

1482

@@ -1468,7 +1484,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1484 modality: [

1485 MultiModalHasher.hash_kwargs(model_id=model_id,

1486 **{modality: item},

1471 - **hf_processor_mm_kwargs)

1487 + **hf_processor_mm_kwargs,

1488 + **tokenization_kwargs)

1489 for item in items

1490 ]

1491 for modality, items in mm_items.items()

@@ -1513,6 +1530,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1530 prompt: Union[str, list[int]],

1531 mm_data_items: MultiModalDataItems,

1532 hf_processor_mm_kwargs: Mapping[str, object],

1533 + tokenization_kwargs: Mapping[str, object],

1534 *,

1535 return_mm_hashes: bool,

1536 ) -> tuple[list[int], MultiModalKwargs, Optional[MultiModalHashes], bool]:

@@ -1524,10 +1542,12 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1542 prompt=prompt,

1543 mm_items=mm_data_items,

1544 hf_processor_mm_kwargs=hf_processor_mm_kwargs,

1545 + tokenization_kwargs=tokenization_kwargs,

1546 enable_hf_prompt_update=True,

1547 )

1548

1530 - mm_hashes = (self._hash_mm_items(mm_data_items, hf_processor_mm_kwargs)

1549 + mm_hashes = (self._hash_mm_items(mm_data_items, hf_processor_mm_kwargs,

1550 + tokenization_kwargs)

1551 if return_mm_hashes else None)

1552

1553 return prompt_ids, mm_kwargs, mm_hashes, is_update_applied

@@ -1537,6 +1557,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1557 prompt: Union[str, list[int]],

1558 mm_data_items: MultiModalDataItems,

1559 hf_processor_mm_kwargs: Mapping[str, object],

1560 + tokenization_kwargs: Mapping[str, object],

1561 *,

1562 return_mm_hashes: bool,

1563 ) -> tuple[list[int], MultiModalKwargs, Optional[MultiModalHashes], bool]:

@@ -1552,6 +1573,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1573 prompt=prompt,

1574 mm_data_items=mm_data_items,

1575 hf_processor_mm_kwargs=hf_processor_mm_kwargs,

1576 + tokenization_kwargs=tokenization_kwargs,

1577 return_mm_hashes=return_mm_hashes,

1578 )

1579

@@ -1562,6 +1584,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1584 cache=cache,

1585 mm_data_items=mm_data_items,

1586 hf_processor_mm_kwargs=hf_processor_mm_kwargs,

1587 + tokenization_kwargs=tokenization_kwargs,

1588 )

1589

1590 # NOTE: `prompt` does not correspond to `mm_missing_data_items`,

@@ -1575,6 +1598,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1598 prompt=prompt,

1599 mm_items=self._to_mm_items(mm_missing_data),

1600 hf_processor_mm_kwargs=hf_processor_mm_kwargs,

1601 + tokenization_kwargs=tokenization_kwargs,

1602 enable_hf_prompt_update=False,

1603 )

1604

@@ -1783,6 +1807,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1807 prompt: Union[str, list[int]],

1808 mm_data: MultiModalDataDict,

1809 hf_processor_mm_kwargs: Mapping[str, object],

1810 + tokenization_kwargs: Optional[Mapping[str, object]] = None,

1811 return_mm_hashes: bool = False,

1812 ) -> MultiModalInputs:

1813 """

@@ -1800,6 +1825,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1825 """

1826 mm_items = self._to_mm_items(mm_data)

1827

1828 + if tokenization_kwargs is None:

1829 + tokenization_kwargs = {}

1830 +

1831 (

1832 prompt_ids,

1833 mm_kwargs,

@@ -1809,9 +1837,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1837 prompt,

1838 mm_items,

1839 hf_processor_mm_kwargs,

1840 + tokenization_kwargs=tokenization_kwargs,

1841 return_mm_hashes=return_mm_hashes,

1842 )

1843

1844 + # NOTE: tokenization_kwargs are not required to init processor

1845 prompt_ids, prompt, mm_placeholders = self._maybe_apply_prompt_updates(

1846 mm_items=mm_items,

1847 hf_processor_mm_kwargs=hf_processor_mm_kwargs,

@@ -1892,6 +1922,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):

1922 prompt: Union[str, list[int]],

1923 mm_data: MultiModalDataDict,

1924 hf_processor_mm_kwargs: Mapping[str, object],

1925 + tokenization_kwargs: Optional[Mapping[str, object]] = None,

1926 return_mm_hashes: bool = False,

1927 ) -> MultiModalEncDecInputs:

1928 """

@@ -1906,6 +1937,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):

1937 encoder_prompt,

1938 mm_data,

1939 hf_processor_mm_kwargs,

1940 + tokenization_kwargs,

1941 return_mm_hashes,

1942 )

1943

vllm/multimodal/profiling.py

+6 -1

@@ -30,6 +30,7 @@ class ProcessorInputs:

30 prompt: Union[str, list[int]]

31 mm_data: MultiModalDataDict

32 hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)

33 + tokenization_kwargs: Mapping[str, object] = field(default_factory=dict)

36 class DummyEncoderData(NamedTuple):

@@ -90,8 +91,11 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):

91 """

92 dummy_text = self.get_dummy_text(mm_counts)

93 dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts)

94 + tokenization_kwargs = {"truncation": False}

94 - return ProcessorInputs(prompt=dummy_text, mm_data=dummy_mm_data)

96 + return ProcessorInputs(prompt=dummy_text,

97 + mm_data=dummy_mm_data,

98 + tokenization_kwargs=tokenization_kwargs)

100 def _get_dummy_audios(

101 self,

@@ -170,6 +174,7 @@ class MultiModalProfiler(Generic[_I]):

174 prompt=processor_inputs.prompt,

175 mm_data=processor_inputs.mm_data,

176 hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,

177 + tokenization_kwargs=processor_inputs.tokenization_kwargs,

178 )

179

180 def _get_mm_num_tokens(

vllm/utils.py

@@ -1729,6 +1729,7 @@ def supports_kw(

1729 last_param = params[next(reversed(params))] # type: ignore

1730 return (last_param.kind == inspect.Parameter.VAR_KEYWORD

1731 and last_param.name != kw_name)

1732 +

1733 return False

1734

1735

@@ -1771,6 +1772,7 @@ def resolve_mm_processor_kwargs(

1772 # Merge the final processor kwargs, prioritizing inference

1773 # time values over the initialization time values.

1774 mm_processor_kwargs = {**init_mm_kwargs, **runtime_mm_kwargs}

1775 +

1776 return mm_processor_kwargs

1777

1778