@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / afffd3cc8a

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 13, 12:00 UTC

[Model] Pass `mm_features` directly into `get_mrope_input_positions` (#28399)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Nov 11, 2025 at 21:14 UTC afffd3cc8a99ce1cf0f6f1687852e5519d725a3b

15 files changed +225 -272

vllm/model_executor/models/ernie45_vl.py

+13 -22

@@ -34,7 +34,7 @@ import torch

34 import torch.nn as nn

35 import torch.nn.functional as F

36 from einops import rearrange, repeat

37 -from transformers import BatchFeature, PretrainedConfig

37 +from transformers import BatchFeature

39 from vllm.attention.backends.registry import AttentionBackendEnum

40 from vllm.attention.layer import (

@@ -58,6 +58,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader

58 from vllm.multimodal import MULTIMODAL_REGISTRY

59 from vllm.multimodal.inputs import (

60 MultiModalDataDict,

61 + MultiModalFeatureSpec,

62 MultiModalFieldConfig,

63 MultiModalKwargsItems,

64 )

@@ -1433,15 +1434,16 @@ class Ernie4_5_VLMoeForConditionalGeneration(

1434 def get_mrope_input_positions(

1435 self,

1436 input_tokens: list[int],

1436 - hf_config: PretrainedConfig,

1437 - image_grid_thw: list[list[int]] | torch.Tensor,

1438 - video_grid_thw: list[list[int]] | torch.Tensor,

1439 - second_per_grid_ts: list[float] | None = None,

1440 - audio_feature_lengths: torch.Tensor | None = None,

1441 - use_audio_in_video: bool = False,

1437 + mm_features: list[MultiModalFeatureSpec],

1438 ) -> tuple[torch.Tensor, int]:

1443 - """Get mrope input positions and delta value for Ernie VL."""

1439 + kwargs = MultiModalFeatureSpec.gather_kwargs(

1440 + mm_features,

1441 + {"image_grid_thw", "video_grid_thw"},

1442 + )

1443 + image_grid_thw = [item.tolist() for item in kwargs.get("image_grid_thw", [])]

1444 + video_grid_thw = [item.tolist() for item in kwargs.get("video_grid_thw", [])]

1445

1446 + hf_config = self.config

1447 image_token_id = hf_config.im_patch_id

1448 video_start_token_id = hf_config.video_start_token_id

1449 video_end_token_id = hf_config.video_end_token_id

@@ -1449,10 +1451,7 @@ class Ernie4_5_VLMoeForConditionalGeneration(

1451 temporal_conv_size = hf_config.temporal_conv_size

1452 llm_pos_ids_list: list = []

1453

1452 - if not (image_grid_thw is None and video_grid_thw is None):

1453 - if isinstance(image_grid_thw, torch.Tensor):

1454 - image_grid_thw = image_grid_thw.tolist()

1455 -

1454 + if image_grid_thw or video_grid_thw:

1455 input_token_type: list[str] = []

1456 video_check_flg = False

1457 for token in input_tokens:

@@ -1484,11 +1483,7 @@ class Ernie4_5_VLMoeForConditionalGeneration(

1483 llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0

1484 )

1485 if modality_type == "image":

1487 - t, h, w = (

1488 - image_grid_thw[mm_data_idx][0],

1489 - image_grid_thw[mm_data_idx][1],

1490 - image_grid_thw[mm_data_idx][2],

1491 - )

1486 + t, h, w = image_grid_thw[mm_data_idx]

1487 llm_grid_t, llm_grid_h, llm_grid_w = (

1488 t,

1489 h // spatial_conv_size,

@@ -1519,11 +1514,7 @@ class Ernie4_5_VLMoeForConditionalGeneration(

1514 mm_data_idx += 1

1515

1516 elif modality_type == "video":

1522 - t, h, w = (

1523 - video_grid_thw[mm_data_idx][0],

1524 - video_grid_thw[mm_data_idx][1],

1525 - video_grid_thw[mm_data_idx][2],

1526 - )

1517 + t, h, w = video_grid_thw[mm_data_idx]

1518 llm_grid_t, llm_grid_h, llm_grid_w = (

1519 t // temporal_conv_size,

1520 h // spatial_conv_size,

vllm/model_executor/models/glm4_1v.py

+13 -19

@@ -37,7 +37,7 @@ import torch

37 import torch.nn as nn

38 import torch.nn.functional as F

39 from einops import rearrange

40 -from transformers import BatchFeature, PretrainedConfig

40 +from transformers import BatchFeature

41 from transformers.models.glm4v.configuration_glm4v import Glm4vVisionConfig

42 from transformers.models.glm4v.image_processing_glm4v import (

43 Glm4vImageProcessor,

@@ -70,6 +70,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys

70 from vllm.multimodal import MULTIMODAL_REGISTRY

71 from vllm.multimodal.inputs import (

72 MultiModalDataDict,

73 + MultiModalFeatureSpec,

74 MultiModalFieldConfig,

75 MultiModalKwargsItems,

76 VideoItem,

@@ -1619,25 +1620,23 @@ class Glm4vForConditionalGeneration(

1620 def get_mrope_input_positions(

1621 self,

1622 input_tokens: list[int],

1622 - hf_config: "PretrainedConfig",

1623 - image_grid_thw: list[list[int]] | torch.Tensor | None,

1624 - video_grid_thw: list[list[int]] | torch.Tensor | None,

1625 - second_per_grid_ts: list[float] | None = None,

1626 - audio_feature_lengths: torch.Tensor | None = None,

1627 - use_audio_in_video: bool = False,

1623 + mm_features: list[MultiModalFeatureSpec],

1624 ) -> tuple[torch.Tensor, int]:

1629 - """Get mrope input positions and delta value for GLM4V."""

1625 + kwargs = MultiModalFeatureSpec.gather_kwargs(

1626 + mm_features,

1627 + {"image_grid_thw", "video_grid_thw"},

1628 + )

1629 + image_grid_thw = [item.tolist() for item in kwargs.get("image_grid_thw", [])]

1630 + video_grid_thw = [item.tolist() for item in kwargs.get("video_grid_thw", [])]

1631

1632 + hf_config = self.config

1633 image_token_id = hf_config.image_token_id

1634 video_start_token_id = hf_config.video_start_token_id

1635 video_end_token_id = hf_config.video_end_token_id

1636 spatial_merge_size = hf_config.vision_config.spatial_merge_size

1637 llm_pos_ids_list: list = []

1638

1637 - if not (image_grid_thw is None and video_grid_thw is None):

1638 - if isinstance(image_grid_thw, torch.Tensor):

1639 - image_grid_thw = image_grid_thw.tolist()

1640 -

1639 + if image_grid_thw or video_grid_thw:

1640 input_token_type: list[str] = []

1641 video_check_flg = False

1642 for token in input_tokens:

@@ -1669,11 +1668,7 @@ class Glm4vForConditionalGeneration(

1668 llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0

1669 )

1670 if modality_type == "image":

1672 - t, h, w = (

1673 - image_grid_thw[mm_data_idx][0],

1674 - image_grid_thw[mm_data_idx][1],

1675 - image_grid_thw[mm_data_idx][2],

1676 - )

1671 + t, h, w = image_grid_thw[mm_data_idx]

1672 llm_grid_t, llm_grid_h, llm_grid_w = (

1673 t,

1674 h // spatial_merge_size,

@@ -1706,8 +1701,7 @@ class Glm4vForConditionalGeneration(

1701 elif modality_type == "video":

1702 t, h, w = (

1703 video_frame_num,

1709 - image_grid_thw[mm_data_idx][1],

1710 - image_grid_thw[mm_data_idx][2],

1704 + *image_grid_thw[mm_data_idx][1:],

1705 )

1706 llm_grid_t, llm_grid_h, llm_grid_w = (

1707 t,

vllm/model_executor/models/glm4v.py

+13 -19

@@ -15,7 +15,7 @@ from torch import nn

15 from torch.nn import LayerNorm

16 from torchvision import transforms

17 from torchvision.transforms import InterpolationMode

18 -from transformers import BatchFeature, PretrainedConfig, PreTrainedTokenizer, TensorType

18 +from transformers import BatchFeature, PreTrainedTokenizer, TensorType

19 from transformers.image_utils import ImageInput

20 from transformers.tokenization_utils_base import TextInput

@@ -36,6 +36,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys

36 from vllm.multimodal import MULTIMODAL_REGISTRY

37 from vllm.multimodal.inputs import (

38 MultiModalDataDict,

39 + MultiModalFeatureSpec,

40 MultiModalFieldConfig,

41 MultiModalKwargsItems,

42 )

@@ -622,25 +623,23 @@ class GLM4VForCausalLM(

623 def get_mrope_input_positions(

624 self,

625 input_tokens: list[int],

625 - hf_config: PretrainedConfig,

626 - image_grid_thw: list[list[int]] | torch.Tensor,

627 - video_grid_thw: list[list[int]] | torch.Tensor,

628 - second_per_grid_ts: list[float] | None = None,

629 - audio_feature_lengths: torch.Tensor | None = None,

630 - use_audio_in_video: bool = False,

626 + mm_features: list[MultiModalFeatureSpec],

627 ) -> tuple[torch.Tensor, int]:

632 - """Get mrope input positions and delta value for GLM4V."""

628 + kwargs = MultiModalFeatureSpec.gather_kwargs(

629 + mm_features,

630 + {"image_grid_thw", "video_grid_thw"},

631 + )

632 + image_grid_thw = [item.tolist() for item in kwargs.get("image_grid_thw", [])]

633 + video_grid_thw = [item.tolist() for item in kwargs.get("video_grid_thw", [])]

634

635 + hf_config = self.config

636 image_token_id = hf_config.image_token_id

637 video_start_token_id = hf_config.video_start_token_id

638 video_end_token_id = hf_config.video_end_token_id

639 spatial_merge_size = hf_config.vision_config.spatial_merge_size

640 llm_pos_ids_list: list = []

641

640 - if not (image_grid_thw is None and video_grid_thw is None):

641 - if isinstance(image_grid_thw, torch.Tensor):

642 - image_grid_thw = image_grid_thw.tolist()

643 -

642 + if image_grid_thw or video_grid_thw:

643 input_token_type: list[str] = []

644 video_check_flg = False

645 for token in input_tokens:

@@ -672,11 +671,7 @@ class GLM4VForCausalLM(

671 llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0

672 )

673 if modality_type == "image":

675 - t, h, w = (

676 - image_grid_thw[mm_data_idx][0],

677 - image_grid_thw[mm_data_idx][1],

678 - image_grid_thw[mm_data_idx][2],

679 - )

674 + t, h, w = image_grid_thw[mm_data_idx]

675 llm_grid_t, llm_grid_h, llm_grid_w = (

676 t,

677 h // spatial_merge_size,

@@ -709,8 +704,7 @@ class GLM4VForCausalLM(

704 elif modality_type == "video":

705 t, h, w = (

706 video_frame_num,

712 - image_grid_thw[mm_data_idx][1],

713 - image_grid_thw[mm_data_idx][2],

707 + *image_grid_thw[mm_data_idx][1:],

708 )

709 llm_grid_t, llm_grid_h, llm_grid_w = (

710 t,

vllm/model_executor/models/interfaces.py

+6 -16

@@ -16,7 +16,6 @@ import numpy as np

16 import torch

17 import torch.nn as nn

18 from torch import Tensor

19 -from transformers import PretrainedConfig

19 from transformers.models.whisper.tokenization_whisper import LANGUAGES

20 from typing_extensions import Self, TypeIs

@@ -32,10 +31,12 @@ from .interfaces_base import VllmModel, is_pooling_model

31 if TYPE_CHECKING:

32 from vllm.config import VllmConfig

33 from vllm.model_executor.models.utils import WeightsMapper

34 + from vllm.multimodal.inputs import MultiModalFeatureSpec

35 from vllm.sequence import IntermediateTensors

36 else:

37 VllmConfig = object

38 WeightsMapper = object

39 + MultiModalFeatureSpec = object

40 IntermediateTensors = object

42 logger = init_logger(__name__)

@@ -991,12 +992,7 @@ class SupportsMRoPE(Protocol):

992 def get_mrope_input_positions(

993 self,

994 input_tokens: list[int],

994 - hf_config: PretrainedConfig,

995 - image_grid_thw: list[list[int]] | torch.Tensor | None,

996 - video_grid_thw: list[list[int]] | torch.Tensor | None,

997 - second_per_grid_ts: list[float] | None = None,

998 - audio_feature_lengths: torch.Tensor | None = None,

999 - use_audio_in_video: bool = False,

995 + mm_features: list["MultiModalFeatureSpec"],

996 ) -> tuple[torch.Tensor, int]:

997 """

998 Get M-RoPE input positions and delta value for this specific model.

@@ -1006,17 +1002,11 @@ class SupportsMRoPE(Protocol):

1002

1003 Args:

1004 input_tokens: List of input token IDs

1009 - hf_config: HuggingFace model configuration

1010 - image_grid_thw: Image grid dimensions (t, h, w)

1011 - video_grid_thw: Video grid dimensions (t, h, w)

1012 - second_per_grid_ts: Seconds per grid timestep for videos

1013 - audio_feature_lengths: Audio feature lengths for multimodal models

1014 - use_audio_in_video: Whether to use audio in video for interleaving

1005 + mm_features: Information about each multi-modal data item

1006

1007 Returns:

1017 - Tuple of (llm_positions, mrope_position_delta)

1018 - - llm_positions: Tensor of shape [3, num_tokens]

1019 - with T/H/W positions

1008 + Tuple of `(llm_positions, mrope_position_delta)`

1009 + - llm_positions: Tensor of shape `[3, num_tokens]` with T/H/W positions

1010 - mrope_position_delta: Delta for position calculations

1011 """

1012 ...

vllm/model_executor/models/keye.py

+12 -17

@@ -40,6 +40,7 @@ from vllm.multimodal.inputs import (

40 ImageItem,

41 ModalityData,

42 MultiModalDataDict,

43 + MultiModalFeatureSpec,

44 MultiModalFieldConfig,

45 MultiModalKwargsItems,

46 VideoItem,

@@ -1627,16 +1628,17 @@ class KeyeForConditionalGeneration(

1628 def get_mrope_input_positions(

1629 self,

1630 input_tokens: list[int],

1630 - hf_config: PretrainedConfig,

1631 - image_grid_thw: list[list[int]] | torch.Tensor,

1632 - video_grid_thw: list[list[int]] | torch.Tensor,

1633 - second_per_grid_ts: list[float] | None = None,

1634 - audio_feature_lengths: torch.Tensor | None = None,

1635 - use_audio_in_video: bool = False,

1631 + mm_features: list[MultiModalFeatureSpec],

1632 ) -> tuple[torch.Tensor, int]:

1633 + kwargs = MultiModalFeatureSpec.gather_kwargs(

1634 + mm_features,

1635 + {"image_grid_thw", "video_grid_thw"},

1636 + )

1637 + image_grid_thw = [item.tolist() for item in kwargs.get("image_grid_thw", [])]

1638 + video_grid_thw = [item.tolist() for item in kwargs.get("video_grid_thw", [])]

1639 +

1640 if isinstance(video_grid_thw, list) and len(video_grid_thw) > 0:

1641 video_grid_thw = video_grid_thw[0]

1639 - """Get mrope input positions and delta value (Keye series)."""

1642

1643 def split_thw(grid_thw: torch.Tensor | list[int]) -> list[list[int]]:

1644 """

@@ -1662,6 +1664,7 @@ class KeyeForConditionalGeneration(

1664

1665 video_grid_thw = split_thw(video_grid_thw)

1666

1667 + hf_config = self.config

1668 image_token_id = hf_config.image_token_id

1669 video_token_id = hf_config.video_token_id

1670 spatial_merge_size = hf_config.vision_config.spatial_merge_size

@@ -1691,20 +1694,12 @@ class KeyeForConditionalGeneration(

1694 ed_video = len(input_tokens) + 1

1695

1696 if ed_image < ed_video:

1694 - t, h, w = (

1695 - image_grid_thw[image_index][0],

1696 - image_grid_thw[image_index][1],

1697 - image_grid_thw[image_index][2],

1698 - )

1697 + t, h, w = image_grid_thw[image_index]

1698 image_index += 1

1699 remain_images -= 1

1700 ed = ed_image

1701 else:

1703 - t, h, w = (

1704 - video_grid_thw[video_index][0],

1705 - video_grid_thw[video_index][1],

1706 - video_grid_thw[video_index][2],

1707 - )

1702 + t, h, w = video_grid_thw[video_index]

1703 video_index += 1

1704 remain_frames -= 1

1705 ed = ed_video

vllm/model_executor/models/keye_vl1_5.py

+12 -17

@@ -21,6 +21,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY

21 from vllm.multimodal.inputs import (

22 ImageItem,

23 ModalityData,

24 + MultiModalFeatureSpec,

25 MultiModalFieldConfig,

26 MultiModalKwargsItems,

27 VideoItem,

@@ -597,16 +598,17 @@ class KeyeVL1_5ForConditionalGeneration(

598 def get_mrope_input_positions(

599 self,

600 input_tokens: list[int],

600 - hf_config: PretrainedConfig,

601 - image_grid_thw: list[list[int]] | torch.Tensor,

602 - video_grid_thw: list[list[int]] | torch.Tensor,

603 - second_per_grid_ts: list[float] | None = None,

604 - audio_feature_lengths: torch.Tensor | None = None,

605 - use_audio_in_video: bool = False,

601 + mm_features: list[MultiModalFeatureSpec],

602 ) -> tuple[torch.Tensor, int]:

603 + kwargs = MultiModalFeatureSpec.gather_kwargs(

604 + mm_features,

605 + {"image_grid_thw", "video_grid_thw"},

606 + )

607 + image_grid_thw = [item.tolist() for item in kwargs.get("image_grid_thw", [])]

608 + video_grid_thw = [item.tolist() for item in kwargs.get("video_grid_thw", [])]

609 +

610 if isinstance(video_grid_thw, list) and len(video_grid_thw) > 0:

611 video_grid_thw = video_grid_thw[0]

609 - """Get mrope input positions and delta value (Keye series)."""

612

613 def split_thw(grid_thw: torch.Tensor | list[int]) -> list[list[int]]:

614 """

@@ -632,6 +634,7 @@ class KeyeVL1_5ForConditionalGeneration(

634

635 video_grid_thw = split_thw(video_grid_thw)

636

637 + hf_config = self.config

638 image_token_id = hf_config.image_token_id

639 video_token_id = hf_config.video_token_id

640 spatial_merge_size = hf_config.vision_config.spatial_merge_size

@@ -661,20 +664,12 @@ class KeyeVL1_5ForConditionalGeneration(

664 ed_video = len(input_tokens) + 1

665

666 if ed_image < ed_video:

664 - t, h, w = (

665 - image_grid_thw[image_index][0],

666 - image_grid_thw[image_index][1],

667 - image_grid_thw[image_index][2],

668 - )

667 + t, h, w = image_grid_thw[image_index]

668 image_index += 1

669 remain_images -= 1

670 ed = ed_image

671 else:

673 - t, h, w = (

674 - video_grid_thw[video_index][0],

675 - video_grid_thw[video_index][1],

676 - video_grid_thw[video_index][2],

677 - )

672 + t, h, w = video_grid_thw[video_index]

673 video_index += 1

674 remain_frames -= 1

675 ed = ed_video

vllm/model_executor/models/paddleocr_vl.py

+12 -17

@@ -61,6 +61,7 @@ from vllm.model_executor.model_loader.weight_utils import (

61 from vllm.multimodal import MULTIMODAL_REGISTRY

62 from vllm.multimodal.inputs import (

63 MultiModalDataDict,

64 + MultiModalFeatureSpec,

65 MultiModalFieldConfig,

66 MultiModalKwargs,

67 )

@@ -1184,15 +1185,17 @@ class PaddleOCRVLForConditionalGeneration(nn.Module, SupportsMultiModal, Support

1185 def get_mrope_input_positions(

1186 self,

1187 input_tokens: list[int],

1187 - hf_config: PretrainedConfig,

1188 - image_grid_thw: list[list[int]] | torch.Tensor,

1189 - video_grid_thw: list[list[int]] | torch.Tensor,

1190 - second_per_grid_ts: list[float],

1191 - audio_feature_lengths: torch.Tensor | None = None,

1192 - use_audio_in_video: bool = False,

1188 + mm_features: list[MultiModalFeatureSpec],

1189 ) -> tuple[torch.Tensor, int]:

1194 - """Get mrope input positions and delta value."""

1190 + kwargs = MultiModalFeatureSpec.gather_kwargs(

1191 + mm_features,

1192 + {"image_grid_thw", "video_grid_thw", "second_per_grid_ts"},

1193 + )

1194 + image_grid_thw = [item.tolist() for item in kwargs.get("image_grid_thw", [])]

1195 + video_grid_thw = [item.tolist() for item in kwargs.get("video_grid_thw", [])]

1196 + second_per_grid_ts = kwargs.get("second_per_grid_ts", [])

1197

1198 + hf_config = self.config

1199 image_token_id = hf_config.image_token_id

1200 video_token_id = hf_config.video_token_id

1201 vision_start_token_id = hf_config.vision_start_token_id

@@ -1229,20 +1232,12 @@ class PaddleOCRVLForConditionalGeneration(nn.Module, SupportsMultiModal, Support

1232 else:

1233 ed_video = len(input_tokens) + 1

1234 if ed_image < ed_video:

1232 - t, h, w = (

1233 - image_grid_thw[image_index][0],

1234 - image_grid_thw[image_index][1],

1235 - image_grid_thw[image_index][2],

1236 - )

1235 + t, h, w = image_grid_thw[image_index]

1236 image_index += 1

1237 remain_images -= 1

1238 ed = ed_image

1239 else:

1241 - t, h, w = (

1242 - video_grid_thw[video_index][0],

1243 - video_grid_thw[video_index][1],

1244 - video_grid_thw[video_index][2],

1245 - )

1240 + t, h, w = video_grid_thw[video_index]

1241 video_second_per_grid_t = 1.0

1242 if second_per_grid_ts:

1243 video_second_per_grid_t = second_per_grid_ts[video_index]

vllm/model_executor/models/qwen2_5_omni_thinker.py

+26 -20

@@ -68,6 +68,7 @@ from vllm.multimodal.inputs import (

68 ImageItem,

69 ModalityData,

70 MultiModalDataDict,

71 + MultiModalFeatureSpec,

72 MultiModalFieldConfig,

73 MultiModalKwargsItems,

74 NestedTensors,

@@ -923,21 +924,9 @@ class Qwen2_5OmniThinkerForConditionalGeneration(

924 def get_mrope_input_positions(

925 self,

926 input_tokens: list[int],

926 - hf_config: PretrainedConfig,

927 - image_grid_thw: list[list[int]] | torch.Tensor,

928 - video_grid_thw: list[list[int]] | torch.Tensor,

929 - second_per_grid_ts: list[float] | None = None,

930 - audio_feature_lengths: torch.Tensor | None = None,

931 - use_audio_in_video: bool = False,

927 + mm_features: list[MultiModalFeatureSpec],

928 ) -> tuple[torch.Tensor, int]:

933 - """Get mrope input positions and delta value (Qwen2.5-Omni version).

934 -

935 - Differences from MRotaryEmbedding:

936 - 1. Add audio support (and related `audio_feature_lengths`).

937 - 2. Add `use_audio_in_video` option to read audio from video inputs.

938 - In this case, audio and vision position ids will be split into

939 - chunks and interleaved.

940 -

929 + """

930 Example:

931

932 (V_i are vision position ids, A_i are audio position ids)

@@ -945,11 +934,33 @@ class Qwen2_5OmniThinkerForConditionalGeneration(

934 |V_1 ... V_n|A_1 ... A_n|V_n+1 ... V_2n|A_n+1 ... A_2n|...

936 """

937 + kwargs = MultiModalFeatureSpec.gather_kwargs(

938 + mm_features,

939 + {

940 + "image_grid_thw",

941 + "video_grid_thw",

942 + "second_per_grid_ts",

943 + "audio_feature_lengths",

944 + "use_audio_in_video",

945 + },

946 + )

947 + image_grid_thw = kwargs.get("image_grid_thw", [])

948 + video_grid_thw = kwargs.get("video_grid_thw", [])

949 + second_per_grid_ts = kwargs.get("second_per_grid_ts", [])

950 + audio_feature_lengths = kwargs.get("audio_feature_lengths", [])

951 + use_audio_in_video = any(kwargs.get("use_audio_in_video", []))

952 +

953 + image_grid_thw = (torch.stack if image_grid_thw else torch.tensor)(

954 + image_grid_thw

955 + )

956 + video_grid_thw = (torch.stack if video_grid_thw else torch.tensor)(

957 + video_grid_thw

958 + )

959

960 # TODO(fyabc): refactor and share more code with

961 # _vl_get_input_positions_tensor.

962

952 - thinker_config = hf_config.thinker_config

963 + thinker_config = self.config

964 audio_token_id = thinker_config.audio_token_index

965 image_token_id = thinker_config.image_token_index

966 video_token_id = thinker_config.video_token_index

@@ -963,11 +974,6 @@ class Qwen2_5OmniThinkerForConditionalGeneration(

974 thinker_config.vision_config, "tokens_per_second", 25

975 )

976

966 - if isinstance(image_grid_thw, list):

967 - image_grid_thw = torch.tensor(image_grid_thw)

968 - if isinstance(video_grid_thw, list):

969 - video_grid_thw = torch.tensor(video_grid_thw)

970 -

977 src_item = input_tokens

978 audio_seqlens = audio_feature_lengths

979 if not second_per_grid_ts:

vllm/model_executor/models/qwen2_5_vl.py

+17 -19

@@ -35,7 +35,7 @@ import einops

35 import torch

36 import torch.nn as nn

37 import torch.nn.functional as F

38 -from transformers import BatchFeature, PretrainedConfig

38 +from transformers import BatchFeature

39 from transformers.models.qwen2_5_vl import Qwen2_5_VLProcessor

40 from transformers.models.qwen2_5_vl.configuration_qwen2_5_vl import (

41 Qwen2_5_VLConfig,

@@ -75,7 +75,11 @@ from vllm.multimodal.evs import (

75 compute_retention_mask,

76 recompute_mrope_positions,

77 )

78 -from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs

78 +from vllm.multimodal.inputs import (

79 + MultiModalFeatureSpec,

80 + MultiModalFieldConfig,

81 + MultiModalKwargs,

82 +)

83 from vllm.multimodal.parse import MultiModalDataItems

84 from vllm.multimodal.processing import PromptReplacement, PromptUpdate

85 from vllm.sequence import IntermediateTensors

@@ -1120,15 +1124,17 @@ class Qwen2_5_VLForConditionalGeneration(

1124 def get_mrope_input_positions(

1125 self,

1126 input_tokens: list[int],

1123 - hf_config: PretrainedConfig,

1124 - image_grid_thw: list[list[int]] | torch.Tensor,

1125 - video_grid_thw: list[list[int]] | torch.Tensor,

1126 - second_per_grid_ts: list[float],

1127 - audio_feature_lengths: torch.Tensor | None = None,

1128 - use_audio_in_video: bool = False,

1127 + mm_features: list[MultiModalFeatureSpec],

1128 ) -> tuple[torch.Tensor, int]:

1130 - """Get mrope input positions and delta value."""

1129 + kwargs = MultiModalFeatureSpec.gather_kwargs(

1130 + mm_features,

1131 + {"image_grid_thw", "video_grid_thw", "second_per_grid_ts"},

1132 + )

1133 + image_grid_thw = [item.tolist() for item in kwargs.get("image_grid_thw", [])]

1134 + video_grid_thw = [item.tolist() for item in kwargs.get("video_grid_thw", [])]

1135 + second_per_grid_ts = kwargs.get("second_per_grid_ts", [])

1136

1137 + hf_config = self.config

1138 image_token_id = hf_config.image_token_id

1139 video_token_id = hf_config.video_token_id

1140 vision_start_token_id = hf_config.vision_start_token_id

@@ -1165,20 +1171,12 @@ class Qwen2_5_VLForConditionalGeneration(

1171 else:

1172 ed_video = len(input_tokens) + 1

1173 if ed_image < ed_video:

1168 - t, h, w = (

1169 - image_grid_thw[image_index][0],

1170 - image_grid_thw[image_index][1],

1171 - image_grid_thw[image_index][2],

1172 - )

1174 + t, h, w = image_grid_thw[image_index]

1175 image_index += 1

1176 remain_images -= 1

1177 ed = ed_image

1178 else:

1177 - t, h, w = (

1178 - video_grid_thw[video_index][0],

1179 - video_grid_thw[video_index][1],

1180 - video_grid_thw[video_index][2],

1181 - )

1179 + t, h, w = video_grid_thw[video_index]

1180 video_second_per_grid_t = 1.0

1181 if second_per_grid_ts:

1182 video_second_per_grid_t = second_per_grid_ts[video_index]

vllm/model_executor/models/qwen2_vl.py

+13 -24

@@ -34,7 +34,7 @@ import torch

34 import torch.nn as nn

35 import torch.nn.functional as F

36 from einops import rearrange, repeat

37 -from transformers import BatchFeature, PretrainedConfig

37 +from transformers import BatchFeature

38 from transformers.models.qwen2_vl import Qwen2VLImageProcessor, Qwen2VLProcessor

39 from transformers.models.qwen2_vl.configuration_qwen2_vl import (

40 Qwen2VLConfig,

@@ -70,6 +70,7 @@ from vllm.multimodal.inputs import (

70 ImageItem,

71 ModalityData,

72 MultiModalDataDict,

73 + MultiModalFeatureSpec,

74 MultiModalFieldConfig,

75 MultiModalKwargsItems,

76 VideoItem,

@@ -1240,21 +1241,17 @@ class Qwen2VLForConditionalGeneration(

1241 def get_mrope_input_positions(

1242 self,

1243 input_tokens: list[int],

1243 - hf_config: PretrainedConfig,

1244 - image_grid_thw: list[list[int]] | torch.Tensor | None,

1245 - video_grid_thw: list[list[int]] | torch.Tensor | None,

1246 - second_per_grid_ts: list[float] | None = None,

1247 - audio_feature_lengths: torch.Tensor | None = None,

1248 - use_audio_in_video: bool = False,

1244 + mm_features: list[MultiModalFeatureSpec],

1245 ) -> tuple[torch.Tensor, int]:

1250 - """Get M-RoPE input positions for Qwen2-VL model."""

1251 - if image_grid_thw is None:

1252 - image_grid_thw = []

1253 - if video_grid_thw is None:

1254 - video_grid_thw = []

1255 - if second_per_grid_ts is None:

1256 - second_per_grid_ts = []

1246 + kwargs = MultiModalFeatureSpec.gather_kwargs(

1247 + mm_features,

1248 + {"image_grid_thw", "video_grid_thw", "second_per_grid_ts"},

1249 + )

1250 + image_grid_thw = [item.tolist() for item in kwargs.get("image_grid_thw", [])]

1251 + video_grid_thw = [item.tolist() for item in kwargs.get("video_grid_thw", [])]

1252 + second_per_grid_ts = kwargs.get("second_per_grid_ts", [])

1253

1254 + hf_config = self.config

1255 image_token_id = hf_config.image_token_id

1256 video_token_id = hf_config.video_token_id

1257 vision_start_token_id = hf_config.vision_start_token_id

@@ -1291,20 +1288,12 @@ class Qwen2VLForConditionalGeneration(

1288 else:

1289 ed_video = len(input_tokens) + 1

1290 if ed_image < ed_video:

1294 - t, h, w = (

1295 - image_grid_thw[image_index][0],

1296 - image_grid_thw[image_index][1],

1297 - image_grid_thw[image_index][2],

1298 - )

1291 + t, h, w = image_grid_thw[image_index]

1292 image_index += 1

1293 remain_images -= 1

1294 ed = ed_image

1295 else:

1303 - t, h, w = (

1304 - video_grid_thw[video_index][0],

1305 - video_grid_thw[video_index][1],

1306 - video_grid_thw[video_index][2],

1307 - )

1296 + t, h, w = video_grid_thw[video_index]

1297 video_second_per_grid_t = 1.0

1298 if second_per_grid_ts:

1299 video_second_per_grid_t = second_per_grid_ts[video_index]

vllm/model_executor/models/qwen3_omni_moe_thinker.py

+32 -23

@@ -65,7 +65,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead

65 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

66 from vllm.model_executor.models.qwen2_audio import Qwen2AudioProcessingInfo

67 from vllm.multimodal import MULTIMODAL_REGISTRY

68 -from vllm.multimodal.inputs import MultiModalKwargsItems

68 +from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalKwargsItems

69 from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataItems

70 from vllm.multimodal.processing import (

71 BaseMultiModalProcessor,

@@ -1414,39 +1414,48 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(

1414 def get_mrope_input_positions(

1415 self,

1416 input_tokens: list[int],

1417 - hf_config: PretrainedConfig,

1418 - image_grid_thw: list[list[int]] | torch.Tensor | None,

1419 - video_grid_thw: list[list[int]] | torch.Tensor | None,

1420 - second_per_grid_ts: list[float] | None = None,

1421 - audio_feature_lengths: torch.Tensor | None = None,

1422 - use_audio_in_video: bool = False,

1417 + mm_features: list[MultiModalFeatureSpec],

1418 ) -> tuple[torch.Tensor, int]:

1424 - config = hf_config.thinker_config

1425 - if isinstance(image_grid_thw, list):

1426 - image_grid_thw = torch.tensor(image_grid_thw)

1427 - if isinstance(video_grid_thw, list):

1428 - video_grid_thw = torch.tensor(video_grid_thw)

1419 + kwargs = MultiModalFeatureSpec.gather_kwargs(

1420 + mm_features,

1421 + {

1422 + "image_grid_thw",

1423 + "video_grid_thw",

1424 + "second_per_grid_ts",

1425 + "audio_feature_lengths",

1426 + "use_audio_in_video",

1427 + },

1428 + )

1429 + image_grid_thw = kwargs.get("image_grid_thw", [])

1430 + video_grid_thw = kwargs.get("video_grid_thw", [])

1431 + second_per_grid_ts = kwargs.get("second_per_grid_ts", [])

1432 + audio_feature_lengths = kwargs.get("audio_feature_lengths", [])

1433 + use_audio_in_video = any(kwargs.get("use_audio_in_video", []))

1434 +

1435 + image_grid_thw = (torch.stack if image_grid_thw else torch.tensor)(

1436 + image_grid_thw

1437 + )

1438 + video_grid_thw = (torch.stack if video_grid_thw else torch.tensor)(

1439 + video_grid_thw

1440 + )

1441 +

1442 input_ids = torch.tensor(input_tokens)

1443 if input_ids is None or input_ids.ndim != 1:

1444 raise ValueError("_omni3_get_input_positions_tensor expects 1D input_ids")

1445

1446 seq_len = input_ids.shape[0]

1434 - if audio_feature_lengths is not None and not isinstance(

1435 - audio_feature_lengths, torch.Tensor

1436 - ):

1437 - audio_feature_lengths = torch.as_tensor(

1447 +

1448 + if isinstance(audio_feature_lengths, list):

1449 + audio_feature_lengths = torch.tensor(

1450 audio_feature_lengths, dtype=torch.long

1451 )

1440 - if second_per_grid_ts is None:

1441 - if video_grid_thw is not None and video_grid_thw.numel() > 0:

1442 - second_per_grids = torch.ones(

1443 - video_grid_thw.shape[0], dtype=torch.float32

1444 - )

1445 - else:

1446 - second_per_grids = torch.tensor([], dtype=torch.float32)

1452 +

1453 + if not len(second_per_grid_ts) and len(video_grid_thw):

1454 + second_per_grids = torch.ones(len(video_grid_thw), dtype=torch.float32)

1455 else:

1456 second_per_grids = torch.tensor(second_per_grid_ts, dtype=torch.float32)

1457

1458 + config = self.config

1459 spatial_merge_size = config.vision_config.spatial_merge_size

1460 image_token_id = config.image_token_id

1461 video_token_id = config.video_token_id

vllm/model_executor/models/qwen3_vl.py

+12 -18

@@ -34,7 +34,7 @@ import numpy as np

34 import torch

35 import torch.nn as nn

36 import torch.nn.functional as F

37 -from transformers import BatchFeature, PretrainedConfig

37 +from transformers import BatchFeature

38 from transformers.models.qwen2_vl import Qwen2VLImageProcessorFast

39 from transformers.models.qwen2_vl.image_processing_qwen2_vl import (

40 smart_resize as image_smart_resize,

@@ -70,6 +70,7 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys

70 from vllm.multimodal import MULTIMODAL_REGISTRY

71 from vllm.multimodal.inputs import (

72 MultiModalDataDict,

73 + MultiModalFeatureSpec,

74 MultiModalFieldConfig,

75 MultiModalKwargsItem,

76 MultiModalKwargsItems,

@@ -1416,17 +1417,18 @@ class Qwen3VLForConditionalGeneration(

1417 def get_mrope_input_positions(

1418 self,

1419 input_tokens: list[int],

1419 - hf_config: PretrainedConfig,

1420 - image_grid_thw: list[list[int]] | torch.Tensor,

1421 - video_grid_thw: list[list[int]] | torch.Tensor,

1422 - second_per_grid_ts: list[float] | None = None,

1423 - audio_feature_lengths: torch.Tensor | None = None,

1424 - use_audio_in_video: bool = False,

1420 + mm_features: list[MultiModalFeatureSpec],

1421 ) -> tuple[torch.Tensor, int]:

1426 - """Get mrope input positions and delta value."""

1422 + kwargs = MultiModalFeatureSpec.gather_kwargs(

1423 + mm_features,

1424 + {"image_grid_thw", "video_grid_thw"},

1425 + )

1426 + image_grid_thw = [item.tolist() for item in kwargs.get("image_grid_thw", [])]

1427 + video_grid_thw = [item.tolist() for item in kwargs.get("video_grid_thw", [])]

1428

1429 video_grid_thw = [[1, h, w] for t, h, w in video_grid_thw for _ in range(t)]

1430

1431 + hf_config = self.config

1432 image_token_id = hf_config.image_token_id

1433 video_token_id = hf_config.video_token_id

1434 vision_start_token_id = hf_config.vision_start_token_id

@@ -1455,20 +1457,12 @@ class Qwen3VLForConditionalGeneration(

1457 else:

1458 ed_video = len(input_tokens) + 1

1459 if ed_image < ed_video:

1458 - t, h, w = (

1459 - image_grid_thw[image_index][0],

1460 - image_grid_thw[image_index][1],

1461 - image_grid_thw[image_index][2],

1462 - )

1460 + t, h, w = image_grid_thw[image_index]

1461 image_index += 1

1462 remain_images -= 1

1463 ed = ed_image

1464 else:

1467 - t, h, w = (

1468 - video_grid_thw[video_index][0],

1469 - video_grid_thw[video_index][1],

1470 - video_grid_thw[video_index][2],

1471 - )

1465 + t, h, w = video_grid_thw[video_index]

1466 video_index += 1

1467 remain_videos -= 1

1468 ed = ed_video

vllm/model_executor/models/transformers/multimodal.py

+27 -12

@@ -27,6 +27,7 @@ from vllm.model_executor.models.utils import WeightsMapper

27 from vllm.multimodal import MultiModalKwargsItems

28 from vllm.multimodal.inputs import (

29 MultiModalDataDict,

30 + MultiModalFeatureSpec,

31 MultiModalFieldConfig,

32 MultiModalInputs,

33 MultiModalUUIDDict,

@@ -38,7 +39,7 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder

39 from vllm.sequence import IntermediateTensors

41 if TYPE_CHECKING:

41 - from transformers import BatchFeature, PretrainedConfig

42 + from transformers import BatchFeature

44 from vllm.config import VllmConfig

45 from vllm.config.multimodal import BaseDummyOptions

@@ -367,20 +368,34 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):

368 def get_mrope_input_positions(

369 self,

370 input_tokens: list[int],

370 - hf_config: "PretrainedConfig",

371 - image_grid_thw: list[list[int]] | torch.Tensor | None,

372 - video_grid_thw: list[list[int]] | torch.Tensor | None,

373 - second_per_grid_ts: list[float] | None = None,

374 - audio_feature_lengths: torch.Tensor | None = None,

375 - use_audio_in_video: bool = False,

371 + mm_features: list[MultiModalFeatureSpec],

372 ) -> tuple[torch.Tensor, int]:

377 - if any((second_per_grid_ts, audio_feature_lengths, use_audio_in_video)):

373 + kwargs = MultiModalFeatureSpec.gather_kwargs(

374 + mm_features,

375 + {

376 + "image_grid_thw",

377 + "video_grid_thw",

378 + "second_per_grid_ts",

379 + "audio_feature_lengths",

380 + "use_audio_in_video",

381 + },

382 + )

383 + if any(

384 + v

385 + for k, v in kwargs.items()

386 + if k not in {"image_grid_thw", "video_grid_thw"}

387 + ):

388 raise NotImplementedError("Transformers backend only supports images.")

389

380 - if isinstance(image_grid_thw, list):

381 - image_grid_thw = torch.tensor(image_grid_thw)

382 - if isinstance(video_grid_thw, list):

383 - video_grid_thw = torch.tensor(video_grid_thw)

390 + image_grid_thw = kwargs.get("image_grid_thw", [])

391 + video_grid_thw = kwargs.get("video_grid_thw", [])

392 +

393 + image_grid_thw = (torch.stack if image_grid_thw else torch.tensor)(

394 + image_grid_thw

395 + )

396 + video_grid_thw = (torch.stack if video_grid_thw else torch.tensor)(

397 + video_grid_thw

398 + )

399

400 mrope_positions, mrope_position_delta = self.model.get_rope_index(

401 input_ids=torch.tensor(input_tokens).unsqueeze(0),

vllm/multimodal/inputs.py

+13

@@ -249,6 +249,19 @@ class MultiModalFeatureSpec:

249 mm_position: PlaceholderRange

250 """e.g., PlaceholderRange(offset=2, length=336)"""

251

252 + @staticmethod

253 + def gather_kwargs(features: list["MultiModalFeatureSpec"], keys: set[str]):

254 + kwargs = defaultdict[str, list[NestedTensors]](list)

255 +

256 + for f in features:

257 + item = f.data

258 + if item is not None:

259 + for k in keys:

260 + if k in item:

261 + kwargs[k].append(item[k].data)

262 +

263 + return dict(kwargs)

264 +

265

266 @dataclass

267 class MultiModalFieldElem:

vllm/v1/worker/gpu_model_runner.py

+4 -29

@@ -892,38 +892,13 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):

892 self.input_batch.num_accepted_tokens_cpu[i] = num_tokens

893

894 def _init_mrope_positions(self, req_state: CachedRequestState):

895 - image_grid_thw = []

896 - video_grid_thw = []

897 - second_per_grid_ts = []

898 - audio_feature_lengths = []

899 - use_audio_in_video = False

900 - for mm_feature in req_state.mm_features:

901 - mm_item = mm_feature.data

902 - if mm_item is None:

903 - continue

904 - mm_input = mm_item.get_data()

905 - if (t := mm_input.get("image_grid_thw")) is not None:

906 - image_grid_thw.append(t.tolist())

907 - if (t := mm_input.get("video_grid_thw")) is not None:

908 - video_grid_thw.append(t.tolist())

909 - if (t := mm_input.get("second_per_grid_ts")) is not None:

910 - second_per_grid_ts.append(t)

911 - if (t := mm_input.get("audio_feature_lengths")) is not None:

912 - audio_feature_lengths.append(t)

913 - if mm_input.get("use_audio_in_video") is True:

914 - use_audio_in_video = True

915 -

916 - assert supports_mrope(self.get_model()), "M-RoPE support is not implemented."

895 + model = self.get_model()

896 + assert supports_mrope(model), "M-RoPE support is not implemented."

897

898 req_state.mrope_positions, req_state.mrope_position_delta = (

919 - self.model.get_mrope_input_positions(

899 + model.get_mrope_input_positions(

900 req_state.prompt_token_ids,

921 - hf_config=self.model_config.hf_config,

922 - image_grid_thw=image_grid_thw,

923 - video_grid_thw=video_grid_thw,

924 - second_per_grid_ts=second_per_grid_ts,

925 - audio_feature_lengths=audio_feature_lengths,

926 - use_audio_in_video=use_audio_in_video,

901 + req_state.mm_features,

902 )

903 )

904