A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Fix mrope in Transformers Backend (#26087)
Signed-off-by: raushan <raushan@huggingface.co> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Raushan Turganbay committed
Oct 6, 2025 at 13:40 UTC
ab5e7d93f41fefd11aec7977b1a7d294897c26be
2 files changed
+38
-68
tests/models/multimodal/generation/test_common.py
+1
-2
@@ -222,8 +222,7 @@ VLM_TEST_SETTINGS = {
222
vllm_runner_kwargs={
223
"model_impl": "transformers",
224
},
225
- # FIXME: Investigate mrope issue
226
- marks=[large_gpu_mark(min_gb=32), pytest.mark.skip(reason="Mrope issue")],
225
+ marks=[large_gpu_mark(min_gb=32)],
226
),
227
#### Extended model tests
228
"aria": VLMTestInfo(
vllm/model_executor/models/transformers.py
+37
-66
@@ -79,7 +79,6 @@ from .utils import (
79
AutoWeightsLoader,
80
PPMissingLayer,
81
WeightsMapper,
82
- flatten_bn,
82
make_empty_intermediate_tensors_factory,
83
maybe_prefix,
84
)
@@ -347,12 +346,12 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
346
347
def _get_mm_fields_config(
348
self,
350
- hf_inputs,
351
- hf_processor_mm_kwargs,
352
- num_image_patches: torch.Tensor = None,
353
- ):
349
+ hf_inputs: BatchFeature,
350
+ hf_processor_mm_kwargs: Mapping[str, object],
351
+ ) -> Mapping[str, MultiModalFieldConfig]:
352
# HF Processors always return a mask but vLLM doesn't need it
353
hf_inputs.pop("attention_mask", None)
354
+ num_image_patches = hf_inputs.get("num_image_patches")
355
mm_fields = {
356
key: MultiModalFieldConfig.flat_from_sizes("image", num_image_patches)
357
for key in hf_inputs
@@ -360,41 +359,24 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
359
mm_fields["image_embeds"] = MultiModalFieldConfig.flat_from_sizes(
360
"image", num_image_patches
361
)
362
+
363
+ # Keep these as batched, as they always have batch size as first dim
364
+ mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched("image")
365
+ mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched("image")
366
mm_fields["num_image_patches"] = MultiModalFieldConfig.batched("image")
367
return mm_fields
368
366
- def _apply_hf_processor_text_mm(
369
+ def _get_hf_mm_data(
370
self,
368
- prompt_text: str,
371
mm_items: MultiModalDataItems,
370
- hf_processor_mm_kwargs: Mapping[str, object],
371
- tokenization_kwargs: Mapping[str, object],
372
- ) -> tuple[list[int], BatchFeature, bool]:
372
+ ) -> tuple[Mapping[str, object], Mapping[str, object]]:
373
"""
374
- Apply the HF processor on the prompt text and multi-modal data
375
- together.
376
-
377
- In addition, return whether prompt replacements have been applied.
374
+ In contrast to the base class, this method always adds
375
+ `return_mm_token_type_ids` to the processor data
376
"""
379
- processor_data, passthrough_data = self._get_hf_mm_data(mm_items)
377
+ processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
378
processor_data["return_mm_token_type_ids"] = True
381
-
382
- processed_data = self._call_hf_processor(
383
- prompt=prompt_text,
384
- mm_data=processor_data,
385
- mm_kwargs=hf_processor_mm_kwargs,
386
- tok_kwargs=tokenization_kwargs,
387
- )
388
- processed_data.update(passthrough_data)
389
-
390
- (prompt_ids,) = processed_data.pop("input_ids").tolist()
391
- mm_token_type_ids = (
392
- processed_data.pop("mm_token_type_ids")
393
- if "mm_token_type_ids" in processed_data
394
- else processed_data.pop("token_type_ids")
395
- ) # for gemma3 only
396
-
397
- return prompt_ids, processed_data, mm_token_type_ids
379
+ return processor_data, passthrough_data
380
381
def apply(
382
self,
@@ -421,18 +403,28 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
403
# into string
404
prompt = hf_processor.decode(prompt)
405
424
- (prompt_ids, processed_data, mm_token_type_ids) = (
425
- self._apply_hf_processor_text_mm(
426
- prompt_text=prompt,
427
- mm_items=mm_items,
428
- hf_processor_mm_kwargs=hf_processor_mm_kwargs,
429
- tokenization_kwargs=tokenization_kwargs,
430
- )
406
+ # Bypass cached processor and always apply to the full set of mm inputs
407
+ # NOTE: we can't just set caching=False because base class method
408
+ # transforms outputs to `MultiModalKwargs` which is not going to
409
+ # work for Transformers. We have a lot of logic tied to
410
+ # `mm_tokens_per_modality` below
411
+ prompt_ids, processed_data, _ = self._apply_hf_processor_text_mm(
412
+ prompt_text=prompt,
413
+ mm_items=mm_items,
414
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
415
+ tokenization_kwargs=tokenization_kwargs,
416
)
417
433
- # HF processor will return `mm_token_type_ids` from which
434
- # we can infer mm_placeholders. Until then hardcode to make code run
435
- # Below tested on Llava. Prompts and `mm_token_type_ids` are always bs=1
418
+ # For gemma3 we check `token_type_ids` as the key
419
+ token_type_key = (
420
+ "mm_token_type_ids"
421
+ if "mm_token_type_ids" in processed_data
422
+ else "token_type_ids"
423
+ )
424
+ mm_token_type_ids = processed_data.pop(token_type_key)
425
+
426
+ # We can infer vLLM style placeholder from token type ids, if we split
427
+ # it for each input `mm_data`.
428
mm_positions = torch.where(mm_token_type_ids == 1)[1]
429
images = mm_items.get_items("image", ImageProcessorItems)
430
multimodal_config = self.info.ctx.model_config.multimodal_config
@@ -462,17 +454,12 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
454
]
455
mm_placeholders = {"image": ranges}
456
465
- num_image_patches = (
466
- torch.tensor(mm_tokens_per_modality["num_image_patches"])
467
- if "num_image_patches" in mm_tokens_per_modality
468
- else None
457
+ processed_data["num_image_patches"] = torch.tensor(
458
+ mm_tokens_per_modality["num_image_patches"]
459
)
470
- processed_data["num_image_patches"] = num_image_patches
460
mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
461
processed_data,
473
- self._get_mm_fields_config(
474
- processed_data, hf_processor_mm_kwargs, num_image_patches
475
- ),
462
+ self._get_mm_fields_config(processed_data, hf_processor_mm_kwargs),
463
)
464
465
# Use overrides if provided; fallback to data-dependent hashing.
@@ -531,8 +518,6 @@ class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):
518
self.ignore_unexpected_suffixes.append(".bias")
519
520
# Set correct attn and init on "meta" to delay allocating GPU tensors
534
- # TODO: @raushan, use the public `model.set_attn_implementation()`
535
- # method once its checks are fixed in Transformers.
521
self.text_config._attn_implementation = "vllm"
522
with init_on_device_without_buffers("meta"):
523
self.model: PreTrainedModel = AutoModel.from_config(
@@ -844,17 +829,6 @@ class TransformersForCausalLM(TransformersBase):
829
return logits
830
831
847
-def flatten_and_concat(x: list[torch.Tensor]) -> torch.Tensor:
848
- """Flatten until a list of tensors can be concatenated then do concat"""
849
-
850
- def _can_concat(x: list[torch.Tensor]):
851
- return len(set(map(lambda _x: _x.shape[1:], x))) == 1
852
-
853
- if _can_concat(x):
854
- return torch.concat(x)
855
- return flatten_and_concat(flatten_bn(x))
856
-
857
-
832
@MULTIMODAL_REGISTRY.register_processor(
833
MultiModalProcessor,
834
info=MultiModalProcessingInfo,
@@ -935,9 +909,6 @@ class TransformersForMultimodalLM(TransformersForCausalLM, SupportsMultiModal):
909
vision_embeddings = self.model.get_image_features(pixel_values, **kwargs)
910
911
if isinstance(vision_embeddings, torch.Tensor):
938
- if isinstance(num_image_patches, list):
939
- num_image_patches = torch.cat(num_image_patches)
940
-
912
if vision_embeddings.ndim == 2:
913
vision_embeddings = vision_embeddings.unsqueeze(0)
914