A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 14:45 UTC
[Multimodal][Qwen3 Omni] Make Qwen3 Omni work with audio-in-video inputs in V1 engine. (#27721)
Signed-off-by: Chenheli Hua <huachenheli@outlook.com> Signed-off-by: Roger Wang <hey@rogerw.io> Co-authored-by: Roger Wang <hey@rogerw.io>
Chenheli Hua committed
Nov 24, 2025 at 11:24 UTC
839c6b7b72bcc7197443019aae32be409f1c0363
4 files changed
+467
-59
examples/offline_inference/qwen3_omni/only_thinker.py
new
+170
@@ -0,0 +1,170 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""
4
+This example shows how to use vLLM for running offline inference
5
+with the correct prompt format on Qwen2.5-Omni (thinker only).
6
+"""
7
+
8
+from typing import NamedTuple
9
+
10
+from vllm import LLM, SamplingParams
11
+from vllm.assets.audio import AudioAsset
12
+from vllm.assets.image import ImageAsset
13
+from vllm.assets.video import VideoAsset
14
+from vllm.multimodal.image import convert_image_mode
15
+from vllm.utils.argparse_utils import FlexibleArgumentParser
16
+
17
+
18
+class QueryResult(NamedTuple):
19
+ inputs: dict
20
+ limit_mm_per_prompt: dict[str, int]
21
+
22
+
23
+# NOTE: The default `max_num_seqs` and `max_model_len` may result in OOM on
24
+# lower-end GPUs.
25
+# Unless specified, these settings have been tested to work on a single L4.
26
+
27
+default_system = (
28
+ "You are Qwen, a virtual human developed by the Qwen Team, Alibaba "
29
+ "Group, capable of perceiving auditory and visual inputs, as well as "
30
+ "generating text and speech."
31
+)
32
+
33
+
34
+def get_mixed_modalities_query() -> QueryResult:
35
+ question = (
36
+ "What is recited in the audio? "
37
+ "What is the content of this image? Why is this video funny?"
38
+ )
39
+ prompt = (
40
+ f"<|im_start|>system\n{default_system}<|im_end|>\n"
41
+ "<|im_start|>user\n<|audio_start|><|audio_pad|><|audio_end|>"
42
+ "<|vision_start|><|image_pad|><|vision_end|>"
43
+ "<|vision_start|><|video_pad|><|vision_end|>"
44
+ f"{question}<|im_end|>\n"
45
+ f"<|im_start|>assistant\n"
46
+ )
47
+ return QueryResult(
48
+ inputs={
49
+ "prompt": prompt,
50
+ "multi_modal_data": {
51
+ "audio": AudioAsset("mary_had_lamb").audio_and_sample_rate,
52
+ "image": convert_image_mode(
53
+ ImageAsset("cherry_blossom").pil_image, "RGB"
54
+ ),
55
+ "video": VideoAsset(name="baby_reading", num_frames=16).np_ndarrays,
56
+ },
57
+ },
58
+ limit_mm_per_prompt={"audio": 1, "image": 1, "video": 1},
59
+ )
60
+
61
+
62
+def get_use_audio_in_video_query() -> QueryResult:
63
+ question = (
64
+ "Describe the content of the video in details, then convert what the "
65
+ "baby say into text."
66
+ )
67
+ prompt = (
68
+ f"<|im_start|>system\n{default_system}<|im_end|>\n"
69
+ "<|im_start|>user\n<|vision_start|><|video_pad|><|vision_end|>"
70
+ f"{question}<|im_end|>\n"
71
+ f"<|im_start|>assistant\n"
72
+ )
73
+ asset = VideoAsset(name="baby_reading", num_frames=16)
74
+ audio = asset.get_audio(sampling_rate=16000)
75
+ return QueryResult(
76
+ inputs={
77
+ "prompt": prompt,
78
+ "multi_modal_data": {
79
+ "video": asset.np_ndarrays,
80
+ "audio": audio,
81
+ },
82
+ "mm_processor_kwargs": {
83
+ "use_audio_in_video": True,
84
+ },
85
+ },
86
+ limit_mm_per_prompt={"audio": 1, "video": 1},
87
+ )
88
+
89
+
90
+def get_multi_audios_query() -> QueryResult:
91
+ question = "Are these two audio clips the same?"
92
+ prompt = (
93
+ f"<|im_start|>system\n{default_system}<|im_end|>\n"
94
+ "<|im_start|>user\n<|audio_start|><|audio_pad|><|audio_end|>"
95
+ "<|audio_start|><|audio_pad|><|audio_end|>"
96
+ f"{question}<|im_end|>\n"
97
+ f"<|im_start|>assistant\n"
98
+ )
99
+ return QueryResult(
100
+ inputs={
101
+ "prompt": prompt,
102
+ "multi_modal_data": {
103
+ "audio": [
104
+ AudioAsset("winning_call").audio_and_sample_rate,
105
+ AudioAsset("mary_had_lamb").audio_and_sample_rate,
106
+ ],
107
+ },
108
+ },
109
+ limit_mm_per_prompt={
110
+ "audio": 2,
111
+ },
112
+ )
113
+
114
+
115
+query_map = {
116
+ "mixed_modalities": get_mixed_modalities_query,
117
+ "use_audio_in_video": get_use_audio_in_video_query,
118
+ "multi_audios": get_multi_audios_query,
119
+}
120
+
121
+
122
+def main(args):
123
+ model_name = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
124
+ query_result = query_map[args.query_type]()
125
+
126
+ llm = LLM(
127
+ model=model_name,
128
+ max_model_len=12800,
129
+ max_num_seqs=5,
130
+ limit_mm_per_prompt=query_result.limit_mm_per_prompt,
131
+ seed=args.seed,
132
+ )
133
+
134
+ # We set temperature to 0.2 so that outputs can be different
135
+ # even when all prompts are identical when running batch inference.
136
+ sampling_params = SamplingParams(temperature=0.2, max_tokens=256)
137
+
138
+ outputs = llm.generate(query_result.inputs, sampling_params=sampling_params)
139
+
140
+ for o in outputs:
141
+ generated_text = o.outputs[0].text
142
+ print(generated_text)
143
+
144
+
145
+def parse_args():
146
+ parser = FlexibleArgumentParser(
147
+ description="Demo on using vLLM for offline inference with "
148
+ "audio language models"
149
+ )
150
+ parser.add_argument(
151
+ "--query-type",
152
+ "-q",
153
+ type=str,
154
+ default="mixed_modalities",
155
+ choices=query_map.keys(),
156
+ help="Query type.",
157
+ )
158
+ parser.add_argument(
159
+ "--seed",
160
+ type=int,
161
+ default=None,
162
+ help="Set the seed when initializing `vllm.LLM`.",
163
+ )
164
+
165
+ return parser.parse_args()
166
+
167
+
168
+if __name__ == "__main__":
169
+ args = parse_args()
170
+ main(args)
tests/model_executor/test_qwen3_omni.py
new
+221
@@ -0,0 +1,221 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from unittest.mock import Mock
5
+
6
+import pytest
7
+from transformers import PretrainedConfig
8
+
9
+from vllm.multimodal.processing import InputProcessingContext
10
+
11
+
12
+# Helper function to print input IDs with coalesced audio/video tokens.
13
+def print_input_ids(input_ids):
14
+ """
15
+ Print input IDs, compressing consecutive special tokens.
16
+ - 151675: <|audio_pad|>
17
+ - 151656: <|video_pad|>
18
+ """
19
+ if not input_ids:
20
+ print("[]")
21
+ return
22
+
23
+ result = []
24
+ i = 0
25
+
26
+ while i < len(input_ids):
27
+ current_id = input_ids[i]
28
+
29
+ # Check if it's a special token that should be compressed
30
+ if current_id in [151675, 151656]:
31
+ # Count consecutive occurrences
32
+ count = 1
33
+ while i + count < len(input_ids) and input_ids[i + count] == current_id:
34
+ count += 1
35
+
36
+ # Add compressed representation
37
+ token_name = "<|audio_pad|>" if current_id == 151675 else "<|video_pad|>"
38
+ result.append(f"{token_name} * {count}")
39
+ i += count
40
+ else:
41
+ # Regular token, just add it
42
+ result.append(str(current_id))
43
+ i += 1
44
+
45
+ print(", ".join(result))
46
+
47
+
48
+@pytest.fixture
49
+def mock_qwen3_omni_config():
50
+ """Create a mock Qwen3OmniMoeThinker config."""
51
+ config = Mock(spec=PretrainedConfig)
52
+ # Token IDs from https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct/blob/main/tokenizer_config.json
53
+ config.audio_token_id = 151675 # <|audio_pad|>
54
+ config.video_token_id = 151656 # <|video_pad|>
55
+ config.image_token_id = 151655 # <|image_pad|>
56
+ config.audio_start_token_id = 151669 # <|audio_start|>
57
+ config.audio_end_token_id = 151670 # <|audio_end|>
58
+ config.vision_start_token_id = 151652 # <|vision_start|>
59
+ config.position_id_per_seconds = 12.5
60
+
61
+ # Vision config
62
+ vision_config = Mock()
63
+ vision_config.spatial_merge_size = 2
64
+ config.vision_config = vision_config
65
+
66
+ return config
67
+
68
+
69
+@pytest.fixture
70
+def mock_processor():
71
+ """Create a mock HF processor."""
72
+ from transformers.models.whisper import WhisperFeatureExtractor
73
+
74
+ processor = Mock()
75
+ processor.audio_token = "<|audio_pad|>"
76
+ processor.image_token = "<|image_pad|>"
77
+ processor.video_token = "<|video_pad|>"
78
+
79
+ # Create a real WhisperFeatureExtractor instance for the feature_extractor attribute
80
+ feature_extractor = WhisperFeatureExtractor()
81
+ processor.feature_extractor = feature_extractor
82
+
83
+ return processor
84
+
85
+
86
+@pytest.fixture
87
+def mock_tokenizer():
88
+ """Create a mock tokenizer."""
89
+ tokenizer = Mock()
90
+ # Token IDs from https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct/blob/main/tokenizer_config.json
91
+ tokenizer.get_vocab = Mock(
92
+ return_value={
93
+ "<|audio_pad|>": 151675,
94
+ "<|video_pad|>": 151656,
95
+ "<|image_pad|>": 151655,
96
+ "<|audio_start|>": 151669,
97
+ "<|audio_end|>": 151670,
98
+ "<|vision_start|>": 151652,
99
+ "<|vision_end|>": 151653,
100
+ }
101
+ )
102
+ tokenizer.encode = Mock(
103
+ side_effect=lambda x: {
104
+ "<|vision_start|>": [151652],
105
+ "<|vision_end|>": [151653],
106
+ "<|audio_start|>": [151669],
107
+ "<|audio_end|>": [151670],
108
+ "<|audio_pad|>": [151675],
109
+ "<|image_pad|>": [151655],
110
+ "<|video_pad|>": [151656],
111
+ }.get(x, [0])
112
+ )
113
+ tokenizer.vision_bos_token = "<|vision_start|>"
114
+ tokenizer.vision_eos_token = "<|vision_end|>"
115
+ tokenizer.audio_bos_token = "<|audio_start|>"
116
+ tokenizer.audio_eos_token = "<|audio_end|>"
117
+ return tokenizer
118
+
119
+
120
+@pytest.fixture
121
+def mock_image_processor():
122
+ """Create a mock image processor."""
123
+ image_processor = Mock()
124
+ image_processor.merge_size = 2
125
+ return image_processor
126
+
127
+
128
+def test_qwen3_omni_get_updates_use_audio_in_video(
129
+ mock_qwen3_omni_config,
130
+ mock_processor,
131
+ mock_tokenizer,
132
+ mock_image_processor,
133
+):
134
+ """Test the get_updates_use_audio_in_video method directly."""
135
+
136
+ from vllm.model_executor.models.qwen3_omni_moe_thinker import (
137
+ Qwen3OmniMoeThinkerMultiModalProcessor,
138
+ Qwen3OmniMoeThinkerProcessingInfo,
139
+ )
140
+
141
+ # Create a mock context
142
+ mock_ctx = Mock(spec=InputProcessingContext)
143
+
144
+ # Create processing info
145
+ info = Qwen3OmniMoeThinkerProcessingInfo(mock_ctx)
146
+ info.get_hf_config = Mock(return_value=mock_qwen3_omni_config)
147
+ info.get_hf_processor = Mock(return_value=mock_processor)
148
+ info.get_tokenizer = Mock(return_value=mock_tokenizer)
149
+ info.get_image_processor = Mock(return_value=mock_image_processor)
150
+
151
+ # Create a mock dummy_inputs builder
152
+ mock_dummy_inputs = Mock()
153
+
154
+ # Create the processor
155
+ processor = Qwen3OmniMoeThinkerMultiModalProcessor(info, mock_dummy_inputs)
156
+
157
+ # Test parameters from reference video
158
+ # https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Omni/demo/draw.mp4
159
+ audio_len = 85
160
+ video_grid_thw = [6, 36, 64]
161
+ video_second_per_grid_t = 2.0
162
+
163
+ # Call the method
164
+ updates = processor.get_updates_use_audio_in_video(
165
+ thinker_config=mock_qwen3_omni_config,
166
+ audio_len=audio_len,
167
+ video_grid_thw=video_grid_thw,
168
+ video_second_per_grid_t=video_second_per_grid_t,
169
+ )
170
+
171
+ # Updated input ids should align with HF implementation.
172
+ # 151669,
173
+ # <|video_pad|> * 576, <|audio_pad|> * 25,
174
+ # <|video_pad|> * 576, <|audio_pad|> * 25,
175
+ # <|video_pad|> * 576, <|audio_pad|> * 25,
176
+ # <|video_pad|> * 576, <|audio_pad|> * 10,
177
+ # <|video_pad|> * 1152,
178
+ # 151670
179
+ print_input_ids(updates)
180
+
181
+ # Verify structure
182
+ assert isinstance(updates, list)
183
+ assert len(updates) > 0
184
+
185
+ # Verify start and end tokens
186
+ audio_start_token_id = mock_qwen3_omni_config.audio_start_token_id
187
+ audio_end_token_id = mock_qwen3_omni_config.audio_end_token_id
188
+
189
+ assert updates[0] == audio_start_token_id
190
+ assert updates[-1] == audio_end_token_id
191
+
192
+ # Verify both audio and video tokens are present
193
+ audio_token_id = mock_qwen3_omni_config.audio_token_id
194
+ video_token_id = mock_qwen3_omni_config.video_token_id
195
+
196
+ audio_count = updates.count(audio_token_id)
197
+ video_count = updates.count(video_token_id)
198
+
199
+ assert audio_count == audio_len, (
200
+ f"Expected {audio_len} audio tokens, got {audio_count}"
201
+ )
202
+
203
+ # Calculate expected video token count
204
+ spatial_merge_size = mock_qwen3_omni_config.vision_config.spatial_merge_size
205
+ height = video_grid_thw[1] // spatial_merge_size
206
+ width = video_grid_thw[2] // spatial_merge_size
207
+ expected_video_count = video_grid_thw[0] * height * width
208
+
209
+ assert video_count == expected_video_count, (
210
+ f"Expected {expected_video_count} video tokens, got {video_count}"
211
+ )
212
+
213
+ # Total tokens should be: 1 (start) + audio_len + video_count + 1 (end)
214
+ expected_total = 1 + audio_len + expected_video_count + 1
215
+ assert len(updates) == expected_total, (
216
+ f"Expected {expected_total} total tokens, got {len(updates)}"
217
+ )
218
+
219
+
220
+if __name__ == "__main__":
221
+ pytest.main([__file__, "-v"])
vllm/model_executor/models/qwen2_5_omni_thinker.py
-25
@@ -23,7 +23,6 @@
23
"""Inference-only Qwen2.5-Omni model (thinker part)."""
24
25
from collections.abc import Callable, Iterable, Mapping, Sequence
26
-from copy import copy
26
from functools import partial
27
from typing import Annotated, Any, Literal
28
@@ -387,15 +386,6 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
386
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
387
self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
388
390
- use_audio_in_video = False
391
- if "video" in mm_kwargs:
392
- video_items = [item for item in mm_kwargs["video"] if item is not None]
393
- # only check video items (if there are any)
394
- if video_items:
395
- use_audio_in_video = all(
396
- item["use_audio_in_video"].data for item in video_items
397
- )
398
-
389
if is_update_applied:
390
mm_placeholders = self._find_mm_placeholders(
391
prompt_ids,
@@ -404,7 +394,6 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
394
self._validate_mm_placeholders(
395
mm_placeholders,
396
mm_item_counts,
407
- use_audio_in_video=use_audio_in_video,
397
)
398
else:
399
prompt_ids, mm_placeholders = self._apply_prompt_updates(
@@ -414,7 +403,6 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
403
self._validate_mm_placeholders(
404
mm_placeholders,
405
mm_item_counts,
417
- use_audio_in_video=use_audio_in_video,
406
)
407
408
return prompt_ids, mm_placeholders
@@ -640,19 +628,6 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
628
629
return mm_processed_data
630
643
- def _validate_mm_placeholders(
644
- self,
645
- mm_placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
646
- mm_item_counts: Mapping[str, int],
647
- use_audio_in_video: bool = False,
648
- ) -> None:
649
- if use_audio_in_video:
650
- mm_item_counts = copy(mm_item_counts)
651
- if "video" in mm_item_counts:
652
- assert "audio" in mm_item_counts
653
- mm_item_counts["audio"] -= mm_item_counts["video"]
654
- super()._validate_mm_placeholders(mm_placeholders, mm_item_counts)
655
-
631
632
class Qwen2_5OmniConditionalGenerationMixin:
633
def _parse_and_validate_audio_input(
vllm/model_executor/models/qwen3_omni_moe_thinker.py
+76
-34
@@ -68,11 +68,11 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
68
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalKwargsItems
69
from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataItems
70
from vllm.multimodal.processing import (
71
- BaseMultiModalProcessor,
71
MultiModalPromptUpdates,
72
PlaceholderFeaturesInfo,
73
PromptReplacement,
74
PromptUpdate,
75
+ PromptUpdateDetails,
76
)
77
from vllm.sequence import IntermediateTensors
78
@@ -87,7 +87,6 @@ from .qwen2_5_omni_thinker import (
87
Qwen2_5OmniConditionalGenerationMixin,
88
Qwen2_5OmniThinkerDummyInputsBuilder,
89
Qwen2_5OmniThinkerMultiModalProcessor,
90
- Qwen2_5OmniThinkerProcessingInfo,
90
)
91
from .qwen2_5_vl import (
92
Qwen2_5_VisionAttention,
@@ -807,24 +806,8 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
806
else:
807
use_audio_in_video = False
808
810
- if use_audio_in_video and "video" in mm_item_counts:
811
- assert "audio" in mm_item_counts
812
- mm_item_counts["audio"] -= mm_item_counts["video"]
813
-
814
- # Special case with `use_audio_in_video=True`
815
- if use_audio_in_video:
816
- if is_update_applied:
817
- prompt_ids = self._get_raw_input_ids(prompt_ids, use_audio_in_video)
818
- (
819
- prompt_ids,
820
- mm_placeholders,
821
- ) = self._apply_prompt_updates(
822
- prompt_ids,
823
- mm_prompt_updates,
824
- )
825
- self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
809
# normal case with `use_audio_in_video=False`
827
- elif is_update_applied:
810
+ if is_update_applied:
811
mm_placeholders = self._find_mm_placeholders(
812
prompt_ids,
813
mm_prompt_updates,
@@ -834,10 +817,24 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
817
mm_item_counts,
818
)
819
else:
837
- prompt_ids, mm_placeholders = self._apply_prompt_updates(
838
- prompt_ids,
839
- mm_prompt_updates,
840
- )
820
+ if use_audio_in_video and "audio" in mm_prompt_updates:
821
+ filtered_updates = {
822
+ k: v for k, v in mm_prompt_updates.items() if k != "audio"
823
+ }
824
+ prompt_ids, mm_placeholders = self._apply_prompt_updates(
825
+ prompt_ids,
826
+ filtered_updates,
827
+ )
828
+ # Derive audio placeholders from video placeholders
829
+ mm_placeholders = self._derive_audio_from_video_placeholders(
830
+ mm_placeholders, mm_prompt_updates
831
+ )
832
+ else:
833
+ prompt_ids, mm_placeholders = self._apply_prompt_updates(
834
+ prompt_ids,
835
+ mm_prompt_updates,
836
+ )
837
+
838
self._validate_mm_placeholders(
839
mm_placeholders,
840
mm_item_counts,
@@ -962,7 +959,9 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
959
960
def get_replacement_qwen2_use_audio_in_video(item_idx: int):
961
nonlocal audio_in_video_item_idx
965
- audio_num_features = audio_output_lengths[audio_item_idx + item_idx]
962
+ audio_num_features = audio_output_lengths[
963
+ audio_in_video_item_idx + item_idx
964
+ ]
965
video_grid_thw = out_mm_data["video_grid_thw"][item_idx]
966
967
audio_in_video_item_idx += 1
@@ -971,14 +970,17 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
970
if second_per_grid_ts:
971
video_second_per_grid_t = second_per_grid_ts[item_idx]
972
else:
974
- video_second_per_grid_t = 1.0
973
+ video_second_per_grid_t = 2.0
974
976
- return self.get_updates_use_audio_in_video(
975
+ placeholder = self.get_updates_use_audio_in_video(
976
thinker_config=thinker_config,
977
audio_len=audio_num_features,
978
video_grid_thw=video_grid_thw,
979
video_second_per_grid_t=video_second_per_grid_t,
980
)
981
+ return PromptUpdateDetails.select_token_id(
982
+ placeholder, embed_token_id=video_token_id
983
+ )
984
985
video_replacement_fn = (
986
get_replacement_qwen2_use_audio_in_video
@@ -1004,14 +1006,50 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
1006
),
1007
]
1008
1007
- def _validate_mm_placeholders(
1009
+ def _derive_audio_from_video_placeholders(
1010
self,
1009
- mm_placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
1010
- mm_item_counts: Mapping[str, int],
1011
- ) -> None:
1012
- BaseMultiModalProcessor[
1013
- Qwen2_5OmniThinkerProcessingInfo
1014
- ]._validate_mm_placeholders(self, mm_placeholders, mm_item_counts)
1011
+ placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
1012
+ mm_prompt_updates: MultiModalPromptUpdates,
1013
+ ) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
1014
+ """
1015
+ Helper to derive audio placeholders from video placeholders when
1016
+ use_audio_in_video=True.
1017
+ """
1018
+ if "video" not in placeholders:
1019
+ return placeholders
1020
+
1021
+ # Validate audio and video counts match
1022
+ num_videos = len(placeholders["video"])
1023
+ num_audios = len(mm_prompt_updates.get("audio", []))
1024
+ if num_audios != num_videos:
1025
+ raise ValueError(
1026
+ f"use_audio_in_video requires equal number of audio and video items, "
1027
+ f"got {num_audios=}, {num_videos=}"
1028
+ )
1029
+
1030
+ tokenizer = self.info.get_tokenizer()
1031
+ processor = self.info.get_hf_processor()
1032
+ audio_token_id = tokenizer.get_vocab()[processor.audio_token]
1033
+
1034
+ result_placeholders = dict(placeholders)
1035
+ audio_placeholders = []
1036
+
1037
+ # Each video is paired with one audio
1038
+ for video_idx, video_placeholder in enumerate(placeholders["video"]):
1039
+ # Create is_embed mask selecting only audio tokens
1040
+ audio_is_embed = torch.tensor(video_placeholder.tokens) == audio_token_id
1041
+
1042
+ audio_placeholder = PlaceholderFeaturesInfo(
1043
+ modality="audio",
1044
+ item_idx=video_idx,
1045
+ start_idx=video_placeholder.start_idx,
1046
+ tokens=video_placeholder.tokens,
1047
+ is_embed=audio_is_embed,
1048
+ )
1049
+ audio_placeholders.append(audio_placeholder)
1050
+
1051
+ result_placeholders["audio"] = audio_placeholders
1052
+ return result_placeholders
1053
1054
def _get_raw_input_ids(
1055
self,
@@ -1454,7 +1492,11 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
1492
)
1493
1494
if not len(second_per_grid_ts) and len(video_grid_thw):
1457
- second_per_grids = torch.ones(len(video_grid_thw), dtype=torch.float32)
1495
+ second_per_grid_ts = 2.0
1496
+ second_per_grids = (
1497
+ torch.ones(len(video_grid_thw), dtype=torch.float32)
1498
+ * second_per_grid_ts
1499
+ )
1500
else:
1501
second_per_grids = torch.tensor(second_per_grid_ts, dtype=torch.float32)
1502