A high-throughput and memory-efficient inference and serving engine for LLMs
Remove `raw_inputs` from transformers backend (#39425)
Signed-off-by: raushan <raushan@huggingface.co>
Raushan Turganbay committed
Jun 9, 2026 at 17:01 UTC
2ee51063722c9e6d28f71340966942394f330c0e
1 file changed
+3
-12
vllm/model_executor/models/transformers/multimodal.py
+3
-12
@@ -234,12 +234,9 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
234
)
235
236
# For gemma3 we check `token_type_ids` as the key
237
- token_type_key = (
238
- "mm_token_type_ids"
239
- if "mm_token_type_ids" in processed_data
240
- else "token_type_ids"
237
+ mm_token_type_ids = processed_data.get(
238
+ "mm_token_type_ids", processed_data.pop("token_type_ids", None)
239
)
242
- mm_token_type_ids = processed_data.get(token_type_key)
240
241
# We can infer vLLM style placeholder from token type ids, if we split
242
# it for each input `mm_data`.
@@ -292,8 +289,6 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
289
290
291
class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
295
- supports_multimodal_raw_input_only = True
296
-
292
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
293
# Skip SupportsMRoPE.__init__ and call the next class in MRO
294
super(SupportsMRoPE, self).__init__(vllm_config=vllm_config, prefix=prefix)
@@ -366,15 +361,12 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
361
inputs_embeds: torch.Tensor | None = None,
362
**kwargs: object,
363
) -> torch.Tensor | IntermediateTensors:
369
- # Gemma3 and PaliGemma needs `token_type_ids` to work correctly
370
- # Other models will not have `token_type_ids` in kwargs
371
- kwargs = {k: v for k, v in kwargs.items() if k == "token_type_ids"}
364
# Positions shape handling for MRoPE models
365
if self.model_config.uses_mrope:
366
# [3, seq_len] -> [3, 1, seq_len]
367
positions = positions[:, None]
368
model_output = super().forward(
377
- input_ids, positions, intermediate_tensors, inputs_embeds, **kwargs
369
+ input_ids, positions, intermediate_tensors, inputs_embeds
370
)
371
return model_output
372
@@ -413,7 +405,6 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
405
return None
406
407
num_image_patches = kwargs.pop("num_image_patches")
416
- kwargs.pop("token_type_ids", None) # used only in `forward`
408
kwargs.pop("mm_token_type_ids", None) # used only in `model.get_rope_index`
409
410
if pixel_values is not None: