A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 05:15 UTC
[BUGFIX]Fix Qwen-Omni models audio max_token_per_item estimation error leading to encoder_cache_size is 0 (#35994)
Signed-off-by: Miao, Avery <avery.miao@intel.com>
Avery Miao committed
Mar 6, 2026 at 01:16 UTC
e998fa76b99a73ba923adeb7457376228269cc9c
3 files changed
+86
vllm/model_executor/models/qwen2_5_omni_thinker.py
+33
@@ -353,6 +353,39 @@ class Qwen2_5OmniThinkerProcessingInfo(
353
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
354
return {"audio": None, "image": None, "video": None}
355
356
+ def get_mm_max_tokens_per_item(
357
+ self,
358
+ seq_len: int,
359
+ mm_counts: Mapping[str, int] | None = None,
360
+ ) -> Mapping[str, int] | None:
361
+ mm_counts = mm_counts or {}
362
+ requested_modalities = {m for m, c in mm_counts.items() if c > 0}
363
+ mm_max_tokens: dict[str, int] = {}
364
+
365
+ if requested_modalities & {"image", "video"}:
366
+ vl_tokens = Qwen2_5_VLProcessingInfo.get_mm_max_tokens_per_item(
367
+ self,
368
+ seq_len=seq_len,
369
+ mm_counts=mm_counts,
370
+ )
371
+ mm_max_tokens.update(
372
+ {
373
+ m: vl_tokens[m]
374
+ for m in ["image", "video"]
375
+ if m in requested_modalities
376
+ }
377
+ )
378
+
379
+ if "audio" in requested_modalities:
380
+ audio_tokens = Qwen2AudioProcessingInfo.get_mm_max_tokens_per_item(
381
+ self,
382
+ seq_len=seq_len,
383
+ mm_counts=mm_counts,
384
+ )
385
+ mm_max_tokens["audio"] = audio_tokens["audio"]
386
+
387
+ return mm_max_tokens
388
+
389
390
class Qwen2_5OmniThinkerDummyInputsBuilder(
391
BaseDummyInputsBuilder[Qwen2_5OmniThinkerProcessingInfo]
vllm/model_executor/models/qwen2_audio.py
+20
@@ -179,6 +179,26 @@ class Qwen2AudioProcessingInfo(BaseProcessingInfo):
179
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
180
return {"audio": None}
181
182
+ def get_mm_max_tokens_per_item(
183
+ self,
184
+ seq_len: int,
185
+ mm_counts: Mapping[str, int] | None = None,
186
+ ) -> Mapping[str, int]:
187
+ mm_counts = mm_counts or {}
188
+ if mm_counts.get("audio", 0) <= 0:
189
+ return {}
190
+
191
+ feature_extractor = self.get_feature_extractor()
192
+ chunk_length = min(feature_extractor.chunk_length, 30)
193
+ audio_len = int(chunk_length * feature_extractor.sampling_rate)
194
+ hop_length = feature_extractor.hop_length
195
+ max_mel_seq_len = audio_len // hop_length
196
+
197
+ input_lengths = torch.tensor([max_mel_seq_len], dtype=torch.long)
198
+ _, output_lengths = _get_feat_extract_output_lengths(input_lengths)
199
+
200
+ return {"audio": int(output_lengths.item())}
201
+
202
203
class Qwen2AudioDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2AudioProcessingInfo]):
204
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
vllm/model_executor/models/qwen3_omni_moe_thinker.py
+33
@@ -1163,6 +1163,39 @@ class Qwen3OmniMoeThinkerProcessingInfo(
1163
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
1164
return {"audio": None, "image": None, "video": None}
1165
1166
+ def get_mm_max_tokens_per_item(
1167
+ self,
1168
+ seq_len: int,
1169
+ mm_counts: Mapping[str, int] | None = None,
1170
+ ) -> Mapping[str, int] | None:
1171
+ mm_counts = mm_counts or {}
1172
+ requested_modalities = {m for m, c in mm_counts.items() if c > 0}
1173
+ mm_max_tokens: dict[str, int] = {}
1174
+
1175
+ if requested_modalities & {"image", "video"}:
1176
+ vl_tokens = Qwen2_5_VLProcessingInfo.get_mm_max_tokens_per_item(
1177
+ self,
1178
+ seq_len=seq_len,
1179
+ mm_counts=mm_counts,
1180
+ )
1181
+ mm_max_tokens.update(
1182
+ {
1183
+ m: vl_tokens[m]
1184
+ for m in ["image", "video"]
1185
+ if m in requested_modalities
1186
+ }
1187
+ )
1188
+
1189
+ if "audio" in requested_modalities:
1190
+ audio_tokens = Qwen2AudioProcessingInfo.get_mm_max_tokens_per_item(
1191
+ self,
1192
+ seq_len=seq_len,
1193
+ mm_counts=mm_counts,
1194
+ )
1195
+ mm_max_tokens["audio"] = audio_tokens["audio"]
1196
+
1197
+ return mm_max_tokens
1198
+
1199
1200
Qwen3OmniMoeThinkerDummyInputsBuilder = Qwen2_5OmniThinkerDummyInputsBuilder
1201