A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 02:00 UTC
[Bugfix][Perf] Revert applying HF processor on text-only inputs for multimodal models (#28858)
Signed-off-by: Roger Wang <hey@rogerw.io>
Roger Wang committed
Nov 17, 2025 at 06:49 UTC
7f064491f80ba20e782f33f4da566ec7da5118d7
2 files changed
+11
-38
tests/test_inputs.py
+7
-28
@@ -86,34 +86,6 @@ def test_zip_enc_dec_prompts(mm_processor_kwargs, expected_mm_kwargs):
86
assert zipped["mm_processor_kwargs"] == exp_kwargs
87
88
89
-@pytest.mark.parametrize(
90
- "model_id",
91
- [
92
- "facebook/opt-125m",
93
- ],
94
-)
95
-@pytest.mark.parametrize(
96
- "prompt",
97
- [
98
- {
99
- "prompt": "",
100
- "multi_modal_data": {"dummy": []},
101
- },
102
- {
103
- "prompt_token_ids": [],
104
- "multi_modal_data": {"dummy": []},
105
- },
106
- ],
107
-)
108
-def test_preprocessor_text_no_mm_inputs(model_id, prompt):
109
- model_config = ModelConfig(model=model_id)
110
- tokenizer = init_tokenizer_from_configs(model_config)
111
- input_preprocessor = InputPreprocessor(model_config, tokenizer)
112
-
113
- with pytest.raises(ValueError, match="does not support multimodal inputs"):
114
- input_preprocessor.preprocess(prompt)
115
-
116
-
89
@pytest.mark.parametrize(
90
"model_id",
91
[
@@ -127,6 +99,13 @@ def test_preprocessor_text_no_mm_inputs(model_id, prompt):
99
{"prompt_token_ids": []},
100
],
101
)
102
+@pytest.mark.skip(
103
+ reason=(
104
+ "Applying huggingface processor on text inputs results in "
105
+ "significant performance regression for multimodal models. "
106
+ "See https://github.com/vllm-project/vllm/issues/26320"
107
+ )
108
+)
109
def test_preprocessor_always_mm_code_path(model_id, prompt):
110
model_config = ModelConfig(model=model_id)
111
tokenizer = init_tokenizer_from_configs(model_config)
vllm/inputs/preprocess.py
+4
-10
@@ -348,18 +348,15 @@ class InputPreprocessor:
348
)
349
350
inputs: TokenInputs | MultiModalInputs
351
- if self.model_config.is_multimodal_model:
351
+ if multi_modal_data := parsed_content.get("multi_modal_data"):
352
inputs = self._process_multimodal(
353
prompt_token_ids,
354
- parsed_content.get("multi_modal_data") or {},
354
+ multi_modal_data,
355
parsed_content.get("mm_processor_kwargs") or {},
356
tokenization_kwargs=tokenization_kwargs,
357
mm_uuids=mm_uuids,
358
)
359
else:
360
- if parsed_content.get("multi_modal_data"):
361
- raise ValueError("This model does not support multimodal inputs")
362
-
360
inputs = token_inputs(prompt_token_ids)
361
362
if cache_salt := parsed_content.get("cache_salt"):
@@ -377,18 +374,15 @@ class InputPreprocessor:
374
prompt_text = parsed_content["prompt"]
375
376
inputs: TokenInputs | MultiModalInputs
380
- if self.model_config.is_multimodal_model:
377
+ if multi_modal_data := parsed_content.get("multi_modal_data"):
378
inputs = self._process_multimodal(
379
prompt_text,
383
- parsed_content.get("multi_modal_data") or {},
380
+ multi_modal_data,
381
parsed_content.get("mm_processor_kwargs") or {},
382
tokenization_kwargs=tokenization_kwargs,
383
mm_uuids=mm_uuids,
384
)
385
else:
389
- if parsed_content.get("multi_modal_data"):
390
- raise ValueError("This model does not support multimodal inputs")
391
-
386
prompt_token_ids = self._tokenize_prompt(
387
prompt_text,
388
tokenization_kwargs=tokenization_kwargs,