A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 14:45 UTC
[Model] Use mm_position to compute mrope positions for Qwen2.5-Omni (#32772)
Signed-off-by: Itay Etelis <itay.etelis@ibm.com> Co-authored-by: Itay Etelis <itay.etelis@ibm.com>
Itay Etelis committed
Jan 25, 2026 at 14:15 UTC
a698e8e7ad4bd06c0197bd79c9c200bef71be189
3 files changed
+386
-201
examples/offline_inference/qwen2_5_omni/only_thinker.py
+26
@@ -112,10 +112,36 @@ def get_multi_audios_query() -> QueryResult:
112
)
113
114
115
+def get_multi_images_query() -> QueryResult:
116
+ question = "What are the differences between these two images?"
117
+ prompt = (
118
+ f"<|im_start|>system\n{default_system}<|im_end|>\n"
119
+ "<|im_start|>user\n<|vision_bos|><|IMAGE|><|vision_eos|>"
120
+ "<|vision_bos|><|IMAGE|><|vision_eos|>"
121
+ f"{question}<|im_end|>\n"
122
+ f"<|im_start|>assistant\n"
123
+ )
124
+ return QueryResult(
125
+ inputs={
126
+ "prompt": prompt,
127
+ "multi_modal_data": {
128
+ "image": [
129
+ convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB"),
130
+ convert_image_mode(ImageAsset("stop_sign").pil_image, "RGB"),
131
+ ],
132
+ },
133
+ },
134
+ limit_mm_per_prompt={
135
+ "image": 2,
136
+ },
137
+ )
138
+
139
+
140
query_map = {
141
"mixed_modalities": get_mixed_modalities_query,
142
"use_audio_in_video": get_use_audio_in_video_query,
143
"multi_audios": get_multi_audios_query,
144
+ "multi_images": get_multi_images_query,
145
}
146
147
vllm/model_executor/models/qwen2_5_omni_thinker.py
+351
-198
@@ -22,10 +22,11 @@
22
# limitations under the License.
23
"""Inference-only Qwen2.5-Omni model (thinker part)."""
24
25
-from collections.abc import Callable, Iterable, Mapping, Sequence
25
+from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
26
from functools import partial
27
from typing import Annotated, Any, Literal
28
29
+import numpy as np
30
import torch
31
import torch.nn as nn
32
from transformers import PretrainedConfig
@@ -85,6 +86,7 @@ from vllm.multimodal.processing.processor import (
86
PlaceholderFeaturesInfo,
87
PromptReplacement,
88
PromptUpdate,
89
+ PromptUpdateDetails,
90
)
91
from vllm.sequence import IntermediateTensors
92
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -103,7 +105,6 @@ from .utils import (
105
maybe_prefix,
106
split_list_into_ranges,
107
)
106
-from .vision import get_llm_pos_ids_for_vision
108
109
try:
110
import flash_attn
@@ -374,6 +375,67 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
375
self.info.get_hf_config().vision_config.spatial_merge_size
376
)(hf_inputs)
377
378
+ def _derive_audio_from_video_placeholders(
379
+ self,
380
+ placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
381
+ mm_prompt_updates: MultiModalPromptUpdates,
382
+ ) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
383
+ """
384
+ Helper to derive audio placeholders from video placeholders when
385
+ use_audio_in_video=True.
386
+ """
387
+ if "video" not in placeholders:
388
+ return placeholders
389
+
390
+ # Validate audio and video counts match
391
+ num_videos = len(placeholders["video"])
392
+ num_audios = len(mm_prompt_updates.get("audio", []))
393
+ if num_audios != num_videos:
394
+ raise ValueError(
395
+ f"use_audio_in_video requires equal number of audio and video "
396
+ f"items, got {num_audios=}, {num_videos=}"
397
+ )
398
+
399
+ tokenizer = self.info.get_tokenizer()
400
+ processor = self.info.get_hf_processor()
401
+ audio_token_id = tokenizer.get_vocab()[processor.audio_token]
402
+ video_token_id = tokenizer.get_vocab()[processor.video_token]
403
+
404
+ result_placeholders = dict(placeholders)
405
+ audio_placeholders = []
406
+ video_placeholders = []
407
+
408
+ # Each video is paired with one audio
409
+ for video_idx, video_placeholder in enumerate(placeholders["video"]):
410
+ # Create is_embed mask selecting only audio tokens
411
+ audio_is_embed = torch.tensor(video_placeholder.tokens) == audio_token_id
412
+
413
+ # Create is_embed mask selecting only video tokens
414
+ video_is_embed = torch.tensor(video_placeholder.tokens) == video_token_id
415
+
416
+ audio_placeholder = PlaceholderFeaturesInfo(
417
+ modality="audio",
418
+ item_idx=video_idx,
419
+ start_idx=video_placeholder.start_idx,
420
+ tokens=video_placeholder.tokens,
421
+ is_embed=audio_is_embed,
422
+ )
423
+ audio_placeholders.append(audio_placeholder)
424
+
425
+ # Update video placeholder with is_embed mask
426
+ video_placeholder_with_mask = PlaceholderFeaturesInfo(
427
+ modality="video",
428
+ item_idx=video_idx,
429
+ start_idx=video_placeholder.start_idx,
430
+ tokens=video_placeholder.tokens,
431
+ is_embed=video_is_embed,
432
+ )
433
+ video_placeholders.append(video_placeholder_with_mask)
434
+
435
+ result_placeholders["audio"] = audio_placeholders
436
+ result_placeholders["video"] = video_placeholders
437
+ return result_placeholders
438
+
439
def _maybe_apply_prompt_updates(
440
self,
441
mm_items: MultiModalDataItems,
@@ -389,6 +451,16 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
451
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
452
self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
453
454
+ # Detect use_audio_in_video from mm_kwargs
455
+ use_audio_in_video = False
456
+ if "video" in mm_kwargs:
457
+ for item in mm_kwargs["video"]:
458
+ if item and item.get("use_audio_in_video"):
459
+ use_audio_in_video_tensor = item["use_audio_in_video"].data
460
+ if use_audio_in_video_tensor.numel() > 0:
461
+ use_audio_in_video = bool(use_audio_in_video_tensor.item())
462
+ break
463
+
464
if is_update_applied:
465
mm_placeholders = self._find_mm_placeholders(
466
prompt_ids,
@@ -399,10 +471,25 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
471
mm_item_counts,
472
)
473
else:
402
- prompt_ids, mm_placeholders = self._apply_prompt_updates(
403
- prompt_ids,
404
- mm_prompt_updates,
405
- )
474
+ if use_audio_in_video and "audio" in mm_prompt_updates:
475
+ # Filter out audio updates - they are embedded in video
476
+ filtered_updates = {
477
+ k: v for k, v in mm_prompt_updates.items() if k != "audio"
478
+ }
479
+ prompt_ids, mm_placeholders = self._apply_prompt_updates(
480
+ prompt_ids,
481
+ filtered_updates,
482
+ )
483
+ # Derive audio placeholders from video placeholders
484
+ mm_placeholders = self._derive_audio_from_video_placeholders(
485
+ mm_placeholders, mm_prompt_updates
486
+ )
487
+ else:
488
+ prompt_ids, mm_placeholders = self._apply_prompt_updates(
489
+ prompt_ids,
490
+ mm_prompt_updates,
491
+ )
492
+
493
self._validate_mm_placeholders(
494
mm_placeholders,
495
mm_item_counts,
@@ -542,13 +629,19 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
629
else:
630
video_second_per_grid_t = 1.0
631
545
- return self.omni_get_updates_use_audio_in_video(
632
+ updates = self.omni_get_updates_use_audio_in_video(
633
thinker_config=thinker_config,
634
audio_len=audio_num_features,
635
video_grid_thw=video_grid_thw,
636
video_second_per_grid_t=video_second_per_grid_t,
637
)
638
639
+ # Only video tokens should receive video embeddings
640
+ return PromptUpdateDetails.select_token_id(
641
+ seq=updates,
642
+ embed_token_id=video_token_id,
643
+ )
644
+
645
video_replacement_fn = (
646
get_replacement_qwen2_use_audio_in_video
647
if use_audio_in_video
@@ -889,216 +982,276 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
982
)
983
return mm_input_by_modality
984
892
- def get_mrope_input_positions(
893
- self,
894
- input_tokens: list[int],
895
- mm_features: list[MultiModalFeatureSpec],
896
- ) -> tuple[torch.Tensor, int]:
985
+ def _get_audio_for_video_mapping(
986
+ self, mm_features: list[MultiModalFeatureSpec]
987
+ ) -> tuple[dict[int, int], set[int]]:
988
"""
898
- Example:
989
+ Map video offset -> paired audio_feature_length for use_audio_in_video.
990
900
- (V_i are vision position ids, A_i are audio position ids)
991
+ When use_audio_in_video=True, audio is interleaved within video chunks.
992
+ The pairing is based on feature order in mm_features.
993
902
- |V_1 ... V_n|A_1 ... A_n|V_n+1 ... V_2n|A_n+1 ... A_2n|...
903
- |vision chunk 1|audio chunk 1|vision chunk 2|audio chunk 2 |...
994
+ Returns:
995
+ Tuple of (video_offset -> audio_feature_length mapping,
996
+ set of paired audio offsets to skip)
997
"""
905
- kwargs = MultiModalFeatureSpec.gather_kwargs(
906
- mm_features,
907
- {
908
- "image_grid_thw",
909
- "video_grid_thw",
910
- "second_per_grid_ts",
911
- "audio_feature_lengths",
912
- "use_audio_in_video",
913
- },
914
- )
915
- image_grid_thw = kwargs.get("image_grid_thw", [])
916
- video_grid_thw = kwargs.get("video_grid_thw", [])
917
- second_per_grid_ts = kwargs.get("second_per_grid_ts", [])
918
- audio_feature_lengths = kwargs.get("audio_feature_lengths", [])
919
- use_audio_in_video = any(kwargs.get("use_audio_in_video", []))
920
-
921
- image_grid_thw = (torch.stack if image_grid_thw else torch.tensor)(
922
- image_grid_thw
923
- )
924
- video_grid_thw = (torch.stack if video_grid_thw else torch.tensor)(
925
- video_grid_thw
926
- )
927
-
928
- # TODO(fyabc): refactor and share more code with
929
- # _vl_get_input_positions_tensor.
998
+ videos_with_audio = [
999
+ f
1000
+ for f in mm_features
1001
+ if f.modality == "video"
1002
+ and f.data.get("use_audio_in_video")
1003
+ and f.data["use_audio_in_video"].data.item()
1004
+ ]
1005
+ audios = [f for f in mm_features if f.modality == "audio"]
1006
+
1007
+ # Pair videos with audio features (assumes matching order)
1008
+ mapping: dict[int, int] = {}
1009
+ paired_audio_offsets: set[int] = set()
1010
+ for i, video_f in enumerate(videos_with_audio):
1011
+ if i < len(audios):
1012
+ audio_len = audios[i].data["audio_feature_lengths"].data.item()
1013
+ mapping[video_f.mm_position.offset] = audio_len
1014
+ paired_audio_offsets.add(audios[i].mm_position.offset)
1015
+ return mapping, paired_audio_offsets
1016
+
1017
+ def _compute_audio_token_count(self, audio_feature_length: int) -> int:
1018
+ """Compute audio tokens from feature length."""
1019
+ return ((audio_feature_length - 1) // 2 + 1 - 2) // 2 + 1
1020
+
1021
+ def iter_mm_features(
1022
+ self, mm_features: list[MultiModalFeatureSpec]
1023
+ ) -> Iterator[tuple[int, str, dict[str, Any]]]:
1024
+ """
1025
+ Iterate over multimodal features sorted by position offset.
1026
1027
+ Yields: (offset, modality, feature_data) where feature_data contains:
1028
+ - image: {"grid_t", "grid_h", "grid_w", "t_factor"}
1029
+ - video: {"grid_t", "grid_h", "grid_w", "t_factor",
1030
+ "use_audio_in_video", "audio_feature_length"}
1031
+ - audio: {"audio_feature_length"}
1032
+ """
1033
thinker_config = self.config
932
- audio_token_id = thinker_config.audio_token_index
933
- image_token_id = thinker_config.image_token_index
934
- video_token_id = thinker_config.video_token_index
935
- audio_start_token_id = thinker_config.audio_start_token_id
936
- audio_end_token_id = thinker_config.audio_end_token_id
937
- vision_start_token_id = thinker_config.vision_start_token_id
938
- vision_end_token_id = thinker_config.vision_end_token_id
939
- seconds_per_chunk = thinker_config.seconds_per_chunk
1034
spatial_merge_size = thinker_config.vision_config.spatial_merge_size
1035
tokens_per_second = getattr(
1036
thinker_config.vision_config, "tokens_per_second", 25
1037
)
1038
945
- src_item = input_tokens
946
- audio_seqlens = audio_feature_lengths
947
- if not second_per_grid_ts:
948
- second_per_grid_ts = [1] * video_grid_thw.shape[0]
949
- audio_idx = 0
950
- video_idx = 0
951
- image_idx = 0
952
- new_src_item: list[int] = []
953
- llm_pos_ids_list: list[torch.Tensor] = []
954
-
955
- idx = 0
956
- while idx < len(src_item):
957
- new_src_item_len = len(new_src_item)
958
- start_idx = (
959
- llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
960
- )
961
- if src_item[idx] not in [audio_token_id, video_token_id, image_token_id]:
962
- if use_audio_in_video and idx > 0:
963
- if (
964
- src_item[idx] == vision_end_token_id
965
- and src_item[idx - 1] == audio_end_token_id
966
- ):
967
- # processing the <|audio_eos|> before <|vision_eos|>
968
- start_idx -= 1
969
- elif (
970
- src_item[idx] == audio_start_token_id
971
- and src_item[idx - 1] == vision_start_token_id
972
- ):
973
- # processing the <|audio_bos|> after <|vision_eos|>
974
- start_idx -= 1
975
- new_src_item.append(src_item[idx])
976
- llm_pos_ids = torch.tensor([start_idx], dtype=torch.long).expand(3, -1)
977
- llm_pos_ids_list.append(llm_pos_ids)
978
- elif src_item[idx] == audio_token_id:
979
- assert audio_seqlens is not None
980
- audio_seqlen = audio_seqlens[audio_idx]
981
- place_num = ((audio_seqlen - 1) // 2 + 1 - 2) // 2 + 1
982
- new_src_item.extend([audio_token_id] * place_num)
983
- llm_pos_ids = torch.arange(place_num).expand(3, -1) + start_idx
984
- llm_pos_ids_list.append(llm_pos_ids)
985
- audio_idx += 1
986
- elif src_item[idx] == image_token_id:
987
- grid_t = image_grid_thw[image_idx][0]
988
- grid_hs = image_grid_thw[:, 1]
989
- grid_ws = image_grid_thw[:, 2]
990
- t_index = (torch.arange(grid_t) * 1 * tokens_per_second).long()
991
- llm_pos_ids = get_llm_pos_ids_for_vision(
992
- start_idx, image_idx, spatial_merge_size, t_index, grid_hs, grid_ws
1039
+ # Sort features by offset first, then pair audio with video
1040
+ sorted_features = sorted(mm_features, key=lambda f: f.mm_position.offset)
1041
+ audio_for_video, paired_audio_offsets = self._get_audio_for_video_mapping(
1042
+ sorted_features
1043
+ )
1044
+
1045
+ for mm_feature in sorted_features:
1046
+ offset = mm_feature.mm_position.offset
1047
+ modality = mm_feature.modality
1048
+
1049
+ if modality == "image":
1050
+ t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
1051
+ yield (
1052
+ offset,
1053
+ "image",
1054
+ {
1055
+ "grid_t": t,
1056
+ "grid_h": h // spatial_merge_size,
1057
+ "grid_w": w // spatial_merge_size,
1058
+ "t_factor": 1.0 * tokens_per_second,
1059
+ },
1060
)
994
- llm_pos_ids_list.append(llm_pos_ids)
995
- vision_seqlen = image_grid_thw[image_idx].prod() // (
996
- spatial_merge_size**2
1061
+ elif modality == "video":
1062
+ t, h, w = mm_feature.data["video_grid_thw"].data.tolist()
1063
+ second_per_grid_ts = 1.0
1064
+ if mm_feature.data.get("second_per_grid_ts"):
1065
+ second_per_grid_ts = mm_feature.data[
1066
+ "second_per_grid_ts"
1067
+ ].data.item()
1068
+ use_audio_in_video = False
1069
+ if mm_feature.data.get("use_audio_in_video"):
1070
+ use_audio_in_video = bool(
1071
+ mm_feature.data["use_audio_in_video"].data.item()
1072
+ )
1073
+
1074
+ yield (
1075
+ offset,
1076
+ "video",
1077
+ {
1078
+ "grid_t": t,
1079
+ "grid_h": h // spatial_merge_size,
1080
+ "grid_w": w // spatial_merge_size,
1081
+ "t_factor": second_per_grid_ts * tokens_per_second,
1082
+ "use_audio_in_video": use_audio_in_video,
1083
+ "audio_feature_length": audio_for_video.get(offset),
1084
+ },
1085
)
998
- new_src_item.extend([image_token_id] * vision_seqlen)
999
- image_idx += 1
1000
- elif src_item[idx] == video_token_id and not use_audio_in_video:
1001
- grid_t = video_grid_thw[video_idx][0]
1002
- grid_hs = video_grid_thw[:, 1]
1003
- grid_ws = video_grid_thw[:, 2]
1004
- t_index = (
1005
- torch.arange(grid_t)
1006
- * second_per_grid_ts[video_idx]
1007
- * tokens_per_second
1008
- ).long()
1009
- llm_pos_ids = get_llm_pos_ids_for_vision(
1010
- start_idx, video_idx, spatial_merge_size, t_index, grid_hs, grid_ws
1086
+ elif modality == "audio":
1087
+ # Skip audio that's paired with video (handled in video case)
1088
+ if offset not in paired_audio_offsets:
1089
+ audio_len = mm_feature.data["audio_feature_lengths"].data.item()
1090
+ yield offset, "audio", {"audio_feature_length": audio_len}
1091
+
1092
+ def _compute_interleaved_positions(
1093
+ self, start_idx: int, data: dict[str, Any]
1094
+ ) -> tuple[np.ndarray, int]:
1095
+ """
1096
+ Compute positions for interleaved video+audio chunks.
1097
+
1098
+ Returns: (position_ids, total_token_count)
1099
+ """
1100
+ grid_t = data["grid_t"]
1101
+ grid_h = data["grid_h"]
1102
+ grid_w = data["grid_w"]
1103
+ t_factor = data["t_factor"]
1104
+ audio_len = data["audio_feature_length"]
1105
+
1106
+ thinker_config = self.config
1107
+ tokens_per_second = getattr(
1108
+ thinker_config.vision_config, "tokens_per_second", 25
1109
+ )
1110
+ seconds_per_chunk = thinker_config.seconds_per_chunk
1111
+ t_ntoken_per_chunk = int(tokens_per_second * seconds_per_chunk)
1112
+
1113
+ # Temporal indices with scaling
1114
+ t_index = (np.arange(grid_t) * t_factor).astype(np.int64)
1115
+
1116
+ # Split temporal indices into chunks
1117
+ t_index_split_chunk: list[list[int]] = [
1118
+ [] for _ in range((int(t_index.max()) // t_ntoken_per_chunk) + 1)
1119
+ ]
1120
+ for t_val in t_index:
1121
+ idx = int(t_val) // t_ntoken_per_chunk
1122
+ t_index_split_chunk[idx].append(int(t_val))
1123
+
1124
+ pure_audio_len = self._compute_audio_token_count(audio_len)
1125
+ added_audio_len = 0
1126
+ pos_ids_list: list[np.ndarray] = []
1127
+ audio_start_idx = start_idx
1128
+
1129
+ for t_chunk in t_index_split_chunk:
1130
+ if not t_chunk:
1131
+ continue
1132
+
1133
+ chunk_t = len(t_chunk)
1134
+
1135
+ # Build vision positions for this chunk
1136
+ h_indices = np.tile(
1137
+ np.arange(grid_h).reshape(1, -1, 1), (chunk_t, 1, grid_w)
1138
+ ).flatten()
1139
+ w_indices = np.tile(
1140
+ np.arange(grid_w).reshape(1, 1, -1), (chunk_t, grid_h, 1)
1141
+ ).flatten()
1142
+ t_indices = np.repeat(np.array(t_chunk), grid_h * grid_w)
1143
+
1144
+ vision_pos = np.stack([t_indices, h_indices, w_indices]) + start_idx
1145
+ pos_ids_list.append(vision_pos)
1146
+
1147
+ # Audio tokens for this chunk
1148
+ audio_chunk_size = min(t_ntoken_per_chunk, pure_audio_len - added_audio_len)
1149
+ if audio_chunk_size > 0:
1150
+ audio_pos = (
1151
+ np.broadcast_to(np.arange(audio_chunk_size), (3, audio_chunk_size))
1152
+ + audio_start_idx
1153
)
1012
- llm_pos_ids_list.append(llm_pos_ids)
1013
- vision_seqlen = video_grid_thw[video_idx].prod() // (
1014
- spatial_merge_size**2
1154
+ pos_ids_list.append(audio_pos)
1155
+ audio_start_idx = audio_start_idx + audio_chunk_size
1156
+ added_audio_len += audio_chunk_size
1157
+
1158
+ # Handle remaining audio that doesn't fit in chunks
1159
+ if added_audio_len < pure_audio_len:
1160
+ remaining = pure_audio_len - added_audio_len
1161
+ remaining_audio_pos = (
1162
+ np.broadcast_to(np.arange(remaining), (3, remaining)) + audio_start_idx
1163
+ )
1164
+ pos_ids_list.append(remaining_audio_pos)
1165
+
1166
+ # Calculate total token count
1167
+ vision_tokens = grid_t * grid_h * grid_w
1168
+ total_tokens = vision_tokens + pure_audio_len
1169
+
1170
+ return np.concatenate(pos_ids_list, axis=1), total_tokens
1171
+
1172
+ def get_mrope_input_positions(
1173
+ self,
1174
+ input_tokens: list[int],
1175
+ mm_features: list[MultiModalFeatureSpec],
1176
+ ) -> tuple[torch.Tensor, int]:
1177
+ """
1178
+ Compute M-RoPE input positions using mm_features directly.
1179
+
1180
+ Example for use_audio_in_video case:
1181
+ (V_i are vision position ids, A_i are audio position ids)
1182
+
1183
+ |V_1 ... V_n|A_1 ... A_n|V_n+1 ... V_2n|A_n+1 ... A_2n|...
1184
+ |vision chunk 1|audio chunk 1|vision chunk 2|audio chunk 2 |...
1185
+ """
1186
+ llm_pos_ids_list: list[np.ndarray] = []
1187
+ st = 0
1188
+
1189
+ for offset, modality, data in self.iter_mm_features(mm_features):
1190
+ # Add text segment before this feature
1191
+ text_len = offset - st
1192
+ st_idx = int(llm_pos_ids_list[-1].max()) + 1 if llm_pos_ids_list else 0
1193
+ if text_len > 0:
1194
+ llm_pos_ids_list.append(
1195
+ np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx
1196
)
1016
- new_src_item.extend([video_token_id] * vision_seqlen)
1017
- video_idx += 1
1018
- else:
1019
- # read audio from video
1020
- assert audio_seqlens is not None
1021
- audio_seqlen = audio_seqlens[audio_idx]
1022
- vision_seqlen = video_grid_thw[video_idx].prod() // (
1023
- spatial_merge_size**2
1197
+ st_idx += text_len
1198
+
1199
+ if modality == "audio":
1200
+ # Standalone audio positions
1201
+ audio_tokens = self._compute_audio_token_count(
1202
+ data["audio_feature_length"]
1203
)
1025
- grid_t = video_grid_thw[video_idx][0]
1026
- grid_h = video_grid_thw[video_idx][1]
1027
- grid_w = video_grid_thw[video_idx][2]
1028
- grid_hs = video_grid_thw[:, 1]
1029
- grid_ws = video_grid_thw[:, 2]
1030
- t_ntoken_per_chunk = int(tokens_per_second * seconds_per_chunk)
1031
- t_index = (
1032
- torch.arange(grid_t)
1033
- * second_per_grid_ts[video_idx]
1034
- * tokens_per_second
1035
- ).long()
1036
- t_index_split_chunk = split_list_into_ranges(
1037
- t_index, t_ntoken_per_chunk
1204
+ llm_pos_ids_list.append(
1205
+ np.broadcast_to(np.arange(audio_tokens), (3, audio_tokens)) + st_idx
1206
)
1039
- place_num = (((audio_seqlen - 1) // 2 + 1 - 2) // 2 + 1) + 2
1040
- pure_audio_len = place_num - 2
1041
- added_audio_len = 0
1042
- audio_llm_pos_ids_list: list[torch.Tensor] = []
1043
- for t_chunk in t_index_split_chunk:
1044
- vision_ntoken_per_chunk = (
1045
- len(t_chunk) * grid_h * grid_w // (spatial_merge_size**2)
1046
- )
1047
- new_src_item.extend([video_token_id] * vision_ntoken_per_chunk)
1048
- vision_llm_pos_ids_list = get_llm_pos_ids_for_vision(
1049
- start_idx,
1050
- video_idx,
1051
- spatial_merge_size,
1052
- t_chunk,
1053
- grid_hs,
1054
- grid_ws,
1055
- ).split(1, dim=1)
1056
- llm_pos_ids_list.extend(vision_llm_pos_ids_list)
1057
- new_src_item.extend(
1058
- min(t_ntoken_per_chunk, pure_audio_len - added_audio_len)
1059
- * [audio_token_id]
1207
+ st = offset + audio_tokens
1208
+
1209
+ elif modality == "image":
1210
+ # Image uses np.indices like Qwen2-VL
1211
+ grid_t = data["grid_t"]
1212
+ grid_h = data["grid_h"]
1213
+ grid_w = data["grid_w"]
1214
+ t_factor = data["t_factor"]
1215
+
1216
+ grid_indices = np.indices((grid_t, grid_h, grid_w))
1217
+ if t_factor != 1.0:
1218
+ grid_indices[0] = (grid_indices[0] * t_factor).astype(np.int64)
1219
+ llm_pos_ids_list.append(grid_indices.reshape(3, -1) + st_idx)
1220
+ st = offset + grid_t * grid_h * grid_w
1221
+
1222
+ elif modality == "video":
1223
+ grid_t = data["grid_t"]
1224
+ grid_h = data["grid_h"]
1225
+ grid_w = data["grid_w"]
1226
+ t_factor = data["t_factor"]
1227
+
1228
+ if not data["use_audio_in_video"]:
1229
+ # Simple video (same as Qwen2-VL)
1230
+ grid_indices = np.indices((grid_t, grid_h, grid_w))
1231
+ if t_factor != 1.0:
1232
+ grid_indices[0] = (grid_indices[0] * t_factor).astype(np.int64)
1233
+ llm_pos_ids_list.append(grid_indices.reshape(3, -1) + st_idx)
1234
+ st = offset + grid_t * grid_h * grid_w
1235
+ else:
1236
+ # Interleaved video+audio
1237
+ pos_ids, token_count = self._compute_interleaved_positions(
1238
+ st_idx, data
1239
)
1061
- audio_start_idx = (
1062
- start_idx
1063
- if len(audio_llm_pos_ids_list) == 0
1064
- else audio_llm_pos_ids_list[-1][0].item() + 1
1065
- )
1066
- if min(t_ntoken_per_chunk, pure_audio_len - added_audio_len) > 0:
1067
- audio_llm_pos_ids_list = (
1068
- torch.arange(
1069
- min(
1070
- t_ntoken_per_chunk, pure_audio_len - added_audio_len
1071
- )
1072
- ).expand(3, -1)
1073
- + audio_start_idx
1074
- ).split(1, dim=1)
1075
- else:
1076
- audio_llm_pos_ids_list = []
1077
- added_audio_len += min(
1078
- t_ntoken_per_chunk, pure_audio_len - added_audio_len
1079
- )
1080
- llm_pos_ids_list.extend(audio_llm_pos_ids_list)
1081
- if added_audio_len < pure_audio_len:
1082
- new_src_item.extend(
1083
- (pure_audio_len - added_audio_len) * [audio_token_id]
1084
- )
1085
- audio_llm_pos_ids_list = (
1086
- torch.arange(pure_audio_len - added_audio_len).expand(3, -1)
1087
- + llm_pos_ids_list[-1].max()
1088
- + 1
1089
- ).split(1, dim=1)
1090
- llm_pos_ids_list.extend(audio_llm_pos_ids_list)
1091
- audio_idx += 1
1092
- video_idx += 1
1093
- # move to the next token
1094
- idx += len(new_src_item) - new_src_item_len
1095
-
1096
- llm_positions = torch.cat(llm_pos_ids_list, dim=1)
1097
- mrope_position_delta = (
1098
- torch.cat(llm_pos_ids_list, dim=1).max() + 1 - len(src_item)
1099
- )
1240
+ llm_pos_ids_list.append(pos_ids)
1241
+ st = offset + token_count
1242
+
1243
+ # Add trailing text
1244
+ if st < len(input_tokens):
1245
+ st_idx = int(llm_pos_ids_list[-1].max()) + 1 if llm_pos_ids_list else 0
1246
+ text_len = len(input_tokens) - st
1247
+ llm_pos_ids_list.append(
1248
+ np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx
1249
+ )
1250
+
1251
+ llm_positions = np.concatenate(llm_pos_ids_list, axis=1).reshape(3, -1)
1252
+ mrope_position_delta = int(llm_positions.max()) + 1 - len(input_tokens)
1253
1101
- return llm_positions, mrope_position_delta
1254
+ return torch.from_numpy(llm_positions), mrope_position_delta
1255
1256
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
1257
mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)
vllm/v1/worker/gpu_model_runner.py
+9
-3
@@ -2474,9 +2474,15 @@ class GPUModelRunner(
2474
mm_embeds_item = encoder_output[start_idx:end_idx]
2475
2476
req_start_pos = req_start_idx + start_pos - num_computed_tokens
2477
- is_mm_embed[req_start_pos + start_idx : req_start_pos + end_idx] = (
2478
- True if is_embed is None else is_embed
2479
- )
2477
+ # OR mask for overlapping mm_features (use_audio_in_video)
2478
+ if is_embed is None:
2479
+ is_mm_embed[req_start_pos + start_idx : req_start_pos + end_idx] = (
2480
+ True
2481
+ )
2482
+ else:
2483
+ is_mm_embed[
2484
+ req_start_pos + start_idx : req_start_pos + end_idx
2485
+ ] |= is_embed
2486
mm_embeds_req.append(mm_embeds_item)
2487
2488
if self.is_multimodal_pruning_enabled and self.uses_mrope: