@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 5113cf9d16

[Mypy] Fix mypy typing for Qwen and Qianfan models (#58046)

Signed-off-by: Ashraf Bhuiyan <mbhuiyan@redhat.com>

Ashraf Bhuiyan committed Sep 26, 2026 at 10:31 UTC 5113cf9d168cf71b1573c69500f884b0bccbad89
23 files changed +302 -157
tools/pre_commit/mypy.py
-1
@@ -113,7 +113,6 @@ SEPARATE_GROUPS = [
113
114 EXCLUDE = [
115 r"vllm/model_executor/models/[kK]",
116 - r"vllm/model_executor/models/[qQ]",
116 r"vllm/model_executor/models/[uU]",
117 r"vllm/model_executor/models/[vV]",
118 r"vllm/model_executor/models/[wW]",
vllm/model_executor/models/interfaces.py
+1 -1
@@ -1262,7 +1262,7 @@ class SupportsRealtime(Protocol):
1262 Override in subclasses based on the model's expected output length."""
1263
1264 @classmethod
1265 - async def buffer_realtime_audio(
1265 + def buffer_realtime_audio(
1266 cls,
1267 audio_stream: AsyncGenerator[np.ndarray, None],
1268 input_stream: asyncio.Queue[list[int]],
vllm/model_executor/models/qianfan_ocr.py
+8 -8
@@ -36,14 +36,14 @@ class QianfanOCRProcessingInfo(BaseInternVLProcessingInfo):
36 config = self.get_hf_config()
37 vision_config = config.vision_config
38
39 - kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
40 - kwargs.setdefault("image_size", vision_config.image_size)
41 - kwargs.setdefault("min_dynamic_patch", config.min_dynamic_patch)
42 - kwargs.setdefault("max_dynamic_patch", config.max_dynamic_patch)
43 - kwargs.setdefault("dynamic_image_size", config.dynamic_image_size)
44 - kwargs.setdefault("use_thumbnail", config.use_thumbnail)
39 + merged_kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
40 + merged_kwargs.setdefault("image_size", vision_config.image_size)
41 + merged_kwargs.setdefault("min_dynamic_patch", config.min_dynamic_patch)
42 + merged_kwargs.setdefault("max_dynamic_patch", config.max_dynamic_patch)
43 + merged_kwargs.setdefault("dynamic_image_size", config.dynamic_image_size)
44 + merged_kwargs.setdefault("use_thumbnail", config.use_thumbnail)
45
46 - image_processor = InternVLImageProcessor(**kwargs)
46 + image_processor = InternVLImageProcessor(**merged_kwargs)
47 image_size = image_processor.image_size
48 patch_size = vision_config.patch_size
49 downsample_ratio = config.downsample_ratio
@@ -73,7 +73,7 @@ class QianfanOCRForConditionalGeneration(InternVLChatModel):
73 """
74
75 def _patch_quant_config(
76 - self, config: PreTrainedConfig, quant_config: QuantizationConfig
76 + self, config: PreTrainedConfig, quant_config: QuantizationConfig | None
77 ) -> None:
78 super()._patch_quant_config(config, quant_config)
79 # ignore vit layers to preserve model performance
vllm/model_executor/models/qwen2.py
+7 -6
@@ -187,6 +187,12 @@ class Qwen2Attention(nn.Module):
187 if attn_type == AttentionType.ENCODER_ONLY
188 else Attention
189 )
190 + attention_kwargs: dict[str, Any] = {}
191 + if dual_chunk_attention_config:
192 + attention_kwargs = {
193 + "layer_idx": extract_layer_index(prefix),
194 + "dual_chunk_attention_config": dual_chunk_attention_config,
195 + }
196 self.attn = attn_cls(
197 self.num_heads,
198 self.head_dim,
@@ -196,12 +202,7 @@ class Qwen2Attention(nn.Module):
202 quant_config=quant_config,
203 attn_type=attn_type,
204 prefix=f"{prefix}.attn",
199 - **{
200 - "layer_idx": extract_layer_index(prefix),
201 - "dual_chunk_attention_config": dual_chunk_attention_config,
202 - }
203 - if dual_chunk_attention_config
204 - else {},
205 + **attention_kwargs,
206 )
207
208 def forward(
vllm/model_executor/models/qwen2_5_omni_thinker.py
+58 -24
@@ -42,6 +42,7 @@ from transformers.models.qwen2_5_omni.processing_qwen2_5_omni import (
42 Qwen2_5OmniProcessor,
43 )
44 from transformers.models.whisper import WhisperFeatureExtractor
45 +from typing_extensions import TypeVar
46
47 from vllm.config import VllmConfig
48 from vllm.config.multimodal import MultiModalDummyOptions
@@ -63,7 +64,10 @@ from vllm.model_executor.models.qwen2_audio import (
64 Qwen2AudioProcessingInfo,
65 _get_feat_extract_output_lengths,
66 )
66 -from vllm.model_executor.models.qwen2_vl import Qwen2VLMultiModalDataParser
67 +from vllm.model_executor.models.qwen2_vl import (
68 + Qwen2VLMultiModalDataParser,
69 + Qwen2VLProcessingInfo,
70 +)
71 from vllm.multimodal import MULTIMODAL_REGISTRY
72 from vllm.multimodal.inputs import (
73 ImageItem,
@@ -79,6 +83,7 @@ from vllm.multimodal.parse import (
83 )
84 from vllm.multimodal.processing import (
85 BaseDummyInputsBuilder,
86 + BaseProcessingInfo,
87 )
88 from vllm.multimodal.processing.processor import (
89 BaseMultiModalProcessor,
@@ -118,6 +123,14 @@ except (ImportError, ModuleNotFoundError):
123 logger = init_logger(__name__)
124
125
126 +def _get_feature_tensor(feature: MultiModalFeatureSpec, key: str) -> torch.Tensor:
127 + data = feature.data
128 + assert data is not None
129 + tensor = data[key].data
130 + assert isinstance(tensor, torch.Tensor)
131 + return tensor
132 +
133 +
134 def unpad_and_flat_audio_features(
135 input_audio_features: torch.Tensor,
136 audio_feature_lengths: torch.Tensor,
@@ -347,7 +360,7 @@ class Qwen2_5OmniThinkerMultiModalDataParser(Qwen2VLMultiModalDataParser):
360 def _parse_audio_data(
361 self,
362 data: dict[str, torch.Tensor] | ModalityData[ImageItem],
350 - ) -> ModalityDataItems[Any, Any]:
363 + ) -> ModalityDataItems[Any, Any] | None:
364 if isinstance(data, dict):
365 required, optional = self.embedding_field_sets("audio")
366 data = dict(data)
@@ -406,7 +419,7 @@ class Qwen2_5OmniThinkerProcessingInfo(
419 self,
420 seq_len: int,
421 mm_counts: Mapping[str, int] | None = None,
409 - ) -> Mapping[str, int] | None:
422 + ) -> Mapping[str, int]:
423 mm_counts = mm_counts or {}
424 requested_modalities = {m for m, c in mm_counts.items() if c > 0}
425 mm_max_tokens: dict[str, int] = {}
@@ -436,9 +449,13 @@ class Qwen2_5OmniThinkerProcessingInfo(
449 return mm_max_tokens
450
451
439 -class Qwen2_5OmniThinkerDummyInputsBuilder(
440 - BaseDummyInputsBuilder[Qwen2_5OmniThinkerProcessingInfo]
441 -):
452 +_I = TypeVar("_I", bound=BaseProcessingInfo, default=Qwen2_5OmniThinkerProcessingInfo)
453 +_VI = TypeVar(
454 + "_VI", bound=Qwen2VLProcessingInfo, default=Qwen2_5OmniThinkerProcessingInfo
455 +)
456 +
457 +
458 +class Qwen2_5OmniThinkerDummyInputsBuilder(BaseDummyInputsBuilder[_VI]):
459 def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
460 num_audios = mm_counts.get("audio", 0)
461 num_images = mm_counts.get("image", 0)
@@ -462,7 +479,7 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(
479 mm_counts: Mapping[str, int],
480 mm_options: MultiModalDummyOptions,
481 ) -> MultiModalDataDict:
465 - feature_extractor = self.info.get_feature_extractor()
482 + feature_extractor = self.info.get_hf_processor().feature_extractor
483
484 target_audio_length = (
485 min(
@@ -501,9 +518,7 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(
518 return mm_data
519
520
504 -class Qwen2_5OmniThinkerMultiModalProcessor(
505 - BaseMultiModalProcessor[Qwen2_5OmniThinkerProcessingInfo]
506 -):
521 +class Qwen2_5OmniThinkerMultiModalProcessor(BaseMultiModalProcessor[_I]):
522 def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
523 return self.dummy_inputs.get_dummy_text(mm_counts)
524
@@ -651,6 +666,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
666 for item in mm_kwargs["video"]:
667 if item and item.get("use_audio_in_video"):
668 use_audio_in_video_tensor = item["use_audio_in_video"].data
669 + assert isinstance(use_audio_in_video_tensor, torch.Tensor)
670 if use_audio_in_video_tensor.numel() > 0:
671 use_audio_in_video = bool(use_audio_in_video_tensor.item())
672 break
@@ -749,6 +765,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
765 ) -> Sequence[PromptUpdate]:
766 processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
767 tokenizer = self.info.get_tokenizer()
768 + assert isinstance(self.info, Qwen2VLProcessingInfo)
769 image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs)
770 vocab = tokenizer.get_vocab()
771
@@ -786,6 +803,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
803 if num_features == 0:
804 audios = mm_items.get_items("audio", AudioProcessorItems)
805 audio = audios.get(item_idx)
806 + assert audio is not None
807 raise ValueError(
808 f"The audio {audio} (len={len(audio)}) is too short "
809 "to be represented inside the model"
@@ -887,6 +905,10 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
905
906
907 class Qwen2_5OmniConditionalGenerationMixin:
908 + audio_tower: nn.Module
909 + visual: nn.Module
910 + vllm_config: VllmConfig
911 +
912 def _parse_and_validate_audio_input(
913 self, **kwargs: object
914 ) -> Qwen2_5OmniAudioFeatureInputs | None:
@@ -916,7 +938,7 @@ class Qwen2_5OmniConditionalGenerationMixin:
938
939 def _parse_and_validate_image_input(
940 self,
919 - **kwargs: dict[str, Any],
941 + **kwargs: object,
942 ) -> Qwen2_5_VLImageInputs | None:
943 pixel_values = kwargs.pop("pixel_values", None)
944 image_embeds = kwargs.pop("image_embeds", None)
@@ -932,7 +954,7 @@ class Qwen2_5OmniConditionalGenerationMixin:
954 image_grid_thw=image_grid_thw,
955 )
956
935 - if image_embeds is not None:
957 + else:
958 return Qwen2_5_VLImageEmbeddingInputs(
959 type="image_embeds",
960 image_embeds=image_embeds,
@@ -941,7 +963,7 @@ class Qwen2_5OmniConditionalGenerationMixin:
963
964 def _parse_and_validate_video_input(
965 self,
944 - **kwargs: dict[str, Any],
966 + **kwargs: object,
967 ) -> Qwen2_5_VLVideoInputs | None:
968 pixel_values_videos = kwargs.pop("pixel_values_videos", None)
969 video_embeds = kwargs.pop("video_embeds", None)
@@ -957,7 +979,7 @@ class Qwen2_5OmniConditionalGenerationMixin:
979 video_grid_thw=video_grid_thw,
980 )
981
960 - if video_embeds is not None:
982 + else:
983 if not isinstance(video_embeds, torch.Tensor):
984 raise ValueError(
985 "Incorrect type of video embeddings. "
@@ -1130,7 +1152,13 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
1152 )
1153
1154 def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
1133 - mm_input_by_modality = {}
1155 + mm_input_by_modality: dict[
1156 + str,
1157 + Qwen2_5_VLImageInputs
1158 + | Qwen2_5_VLVideoInputs
1159 + | Qwen2_5OmniAudioFeatureInputs
1160 + | None,
1161 + ] = {}
1162
1163 # Preserve the order of modalities if there are multiple of them
1164 # from the order of kwargs.
@@ -1175,8 +1203,9 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
1203 f
1204 for f in mm_features
1205 if f.modality == "video"
1206 + and f.data is not None
1207 and f.data.get("use_audio_in_video")
1179 - and f.data["use_audio_in_video"].data.item()
1208 + and _get_feature_tensor(f, "use_audio_in_video").item()
1209 ]
1210 audios = [f for f in mm_features if f.modality == "audio"]
1211
@@ -1185,7 +1214,9 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
1214 paired_audio_offsets: set[int] = set()
1215 for i, video_f in enumerate(videos_with_audio):
1216 if i < len(audios):
1188 - audio_len = audios[i].data["audio_feature_lengths"].data.item()
1217 + audio_len = int(
1218 + _get_feature_tensor(audios[i], "audio_feature_lengths").item()
1219 + )
1220 mapping[video_f.mm_position.offset] = audio_len
1221 paired_audio_offsets.add(audios[i].mm_position.offset)
1222 return mapping, paired_audio_offsets
@@ -1220,9 +1251,10 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
1251 for mm_feature in sorted_features:
1252 offset = mm_feature.mm_position.offset
1253 modality = mm_feature.modality
1254 + assert mm_feature.data is not None
1255
1256 if modality == "image":
1225 - t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
1257 + t, h, w = _get_feature_tensor(mm_feature, "image_grid_thw").tolist()
1258 yield (
1259 offset,
1260 "image",
@@ -1234,16 +1266,16 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
1266 },
1267 )
1268 elif modality == "video":
1237 - t, h, w = mm_feature.data["video_grid_thw"].data.tolist()
1269 + t, h, w = _get_feature_tensor(mm_feature, "video_grid_thw").tolist()
1270 second_per_grid_ts = 1.0
1271 if mm_feature.data.get("second_per_grid_ts"):
1240 - second_per_grid_ts = mm_feature.data[
1241 - "second_per_grid_ts"
1242 - ].data.item()
1272 + second_per_grid_ts = _get_feature_tensor(
1273 + mm_feature, "second_per_grid_ts"
1274 + ).item()
1275 use_audio_in_video = False
1276 if mm_feature.data.get("use_audio_in_video"):
1277 use_audio_in_video = bool(
1246 - mm_feature.data["use_audio_in_video"].data.item()
1278 + _get_feature_tensor(mm_feature, "use_audio_in_video").item()
1279 )
1280
1281 yield (
@@ -1261,7 +1293,9 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
1293 elif modality == "audio":
1294 # Skip audio that's paired with video (handled in video case)
1295 if offset not in paired_audio_offsets:
1264 - audio_len = mm_feature.data["audio_feature_lengths"].data.item()
1296 + audio_len = _get_feature_tensor(
1297 + mm_feature, "audio_feature_lengths"
1298 + ).item()
1299 yield offset, "audio", {"audio_feature_length": audio_len}
1300
1301 def _compute_interleaved_positions(
vllm/model_executor/models/qwen2_5_vl.py
+37 -24
@@ -654,7 +654,10 @@ class Qwen2_5_VisionTransformer(nn.Module):
654 self.spatial_merge_size = vision_config.spatial_merge_size
655 self.fullatt_block_indexes = vision_config.fullatt_block_indexes
656 self.spatial_merge_unit = self.spatial_merge_size**2
657 - self._rope_by_thw_cache = LRUCache(capacity=1024)
657 + self._rope_by_thw_cache: LRUCache[
658 + tuple[int, int, int],
659 + tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor],
660 + ] = LRUCache(capacity=1024)
661 use_data_parallel = is_vit_use_data_parallel()
662 self.tp_size = (
663 1
@@ -885,7 +888,7 @@ class Qwen2_5_VisionTransformer(nn.Module):
888
889 def prepare_encoder_metadata(
890 self,
888 - grid_thw: list[list[int]],
891 + grid_thw: Sequence[Sequence[int]],
892 *,
893 max_batch_size: int | None = None,
894 max_frames_per_batch: int | None = None,
@@ -927,9 +930,9 @@ class Qwen2_5_VisionTransformer(nn.Module):
930 # patchify
931 rotary_pos_emb_cos = []
932 rotary_pos_emb_sin = []
930 - window_index: list = []
931 - cu_window_seqlens: list = [torch.tensor([0], dtype=torch.int32)]
932 - cu_seqlens: list = []
933 + window_indices: list[torch.Tensor] = []
934 + cu_window_seqlens_parts = [torch.tensor([0], dtype=torch.int32)]
935 + cu_seqlens_parts: list[torch.Tensor] = []
936
937 window_index_id = 0
938 cu_window_seqlens_last = 0
@@ -946,26 +949,26 @@ class Qwen2_5_VisionTransformer(nn.Module):
949 cu_seqlens_thw,
950 ) = self.get_rope_by_thw(t, h, w)
951
949 - window_index.append(window_index_thw + window_index_id)
952 + window_indices.append(window_index_thw + window_index_id)
953 window_index_id += t * llm_h * llm_w
954
955 cu_seqlens_window_thw = cu_seqlens_window_thw + cu_window_seqlens_last
956 cu_window_seqlens_last = cu_seqlens_window_thw[-1]
954 - cu_window_seqlens.append(cu_seqlens_window_thw)
957 + cu_window_seqlens_parts.append(cu_seqlens_window_thw)
958
959 rotary_pos_emb_cos.append(cos_thw)
960 rotary_pos_emb_sin.append(sin_thw)
961
959 - cu_seqlens.append(cu_seqlens_thw)
962 + cu_seqlens_parts.append(cu_seqlens_thw)
963
964 rotary_pos_emb_cos = torch.cat(rotary_pos_emb_cos)
965 rotary_pos_emb_sin = torch.cat(rotary_pos_emb_sin)
963 - window_index = torch.cat(window_index)
966 + window_index = torch.cat(window_indices)
967 # compute reverse indices
968 reverse_indices = self.invert_permutation(window_index)
966 - cu_window_seqlens = torch.cat(cu_window_seqlens)
969 + cu_window_seqlens = torch.cat(cu_window_seqlens_parts)
970 cu_window_seqlens = torch.unique_consecutive(cu_window_seqlens)
968 - cu_seqlens = torch.cat(cu_seqlens)
971 + cu_seqlens = torch.cat(cu_seqlens_parts)
972 cu_seqlens = torch.cumsum(cu_seqlens, dim=0, dtype=torch.int32)
973 cu_seqlens = F.pad(cu_seqlens, (1, 0), "constant", 0)
974
@@ -1088,6 +1091,7 @@ class Qwen2_5_VisionTransformer(nn.Module):
1091
1092 seq_len = hidden_states.shape[0]
1093 if encoder_metadata is None:
1094 + assert grid_thw is not None
1095 encoder_metadata = self.prepare_encoder_metadata(grid_thw)
1096
1097 rotary_pos_emb_cos = encoder_metadata["rotary_pos_emb_cos"]
@@ -1162,7 +1166,7 @@ class Qwen2_5_VLDummyInputsBuilder(Qwen2VLDummyInputsBuilder):
1166 )
1167 videos = [v.copy() for v in videos]
1168
1165 - video_items = []
1169 + video_items: list[VideoItem] = []
1170 for video in videos:
1171 video_num_frames = video.shape[0]
1172 video_metadata = {
@@ -1216,6 +1220,7 @@ class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):
1220 hf_data = hf_inputs.hf_data
1221
1222 if videos := hf_data.get("videos"):
1223 + assert isinstance(videos, list)
1224 # HF expects video metadata as a separate argument.
1225 hf_data["videos"] = [video for video, _ in videos]
1226 hf_data["video_metadata"] = [
@@ -1336,17 +1341,22 @@ class Qwen2_5_VLForConditionalGeneration(
1341 tokens_per_second = getattr(self.config.vision_config, "tokens_per_second", 1.0)
1342 for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
1343 offset = mm_feature.mm_position.offset
1344 + data = mm_feature.data
1345 + assert data is not None
1346 if mm_feature.modality == "image":
1340 - t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
1347 + grid = data["image_grid_thw"].data
1348 + assert isinstance(grid, torch.Tensor)
1349 + t, h, w = grid.tolist()
1350 assert t == 1, f"Image must have 1 frame, got {t}"
1351 yield offset, 1, h // spatial_merge_size, w // spatial_merge_size, 1.0
1352 elif mm_feature.modality == "video":
1344 - t, h, w = mm_feature.data["video_grid_thw"].data.tolist()
1353 + grid = data["video_grid_thw"].data
1354 + assert isinstance(grid, torch.Tensor)
1355 + t, h, w = grid.tolist()
1356 second_per_grid_ts = 1.0
1346 - if mm_feature.data.get("second_per_grid_ts", None):
1347 - second_per_grid_ts = mm_feature.data[
1348 - "second_per_grid_ts"
1349 - ].data.item()
1357 + if seconds := data.get("second_per_grid_ts"):
1358 + assert isinstance(seconds.data, torch.Tensor)
1359 + second_per_grid_ts = seconds.data.item()
1360 t_factor = second_per_grid_ts * tokens_per_second
1361 yield (
1362 offset,
@@ -1409,7 +1419,7 @@ class Qwen2_5_VLForConditionalGeneration(
1419 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1420 super().__init__()
1421 config: Qwen2_5_VLConfig = vllm_config.model_config.hf_config
1412 - multimodal_config = vllm_config.model_config.multimodal_config
1422 + multimodal_config = vllm_config.model_config.get_multimodal_config()
1423
1424 self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
1425 self.config = config
@@ -1458,7 +1468,7 @@ class Qwen2_5_VLForConditionalGeneration(
1468 image_grid_thw=image_grid_thw,
1469 )
1470
1461 - if image_embeds is not None:
1471 + else:
1472 return Qwen2_5_VLImageEmbeddingInputs(
1473 type="image_embeds",
1474 image_embeds=image_embeds,
@@ -1484,7 +1494,7 @@ class Qwen2_5_VLForConditionalGeneration(
1494 second_per_grid_ts=second_per_grid_ts,
1495 )
1496
1487 - if video_embeds is not None:
1497 + else:
1498 return Qwen2_5_VLVideoEmbeddingInputs(
1499 type="video_embeds",
1500 video_embeds=video_embeds,
@@ -1549,8 +1559,7 @@ class Qwen2_5_VLForConditionalGeneration(
1559 )
1560 emb = torch.cat([emb, positions], dim=1)
1561 image_embeds_out.append(emb)
1552 - image_embeds_split = image_embeds_out
1553 - return tuple(image_embeds_split)
1562 + return tuple(image_embeds_out)
1563
1564 def _process_video_input(
1565 self, video_input: Qwen2_5_VLVideoInputs
@@ -1614,6 +1623,7 @@ class Qwen2_5_VLForConditionalGeneration(
1623 tokens_per_second = self.config.vision_config.tokens_per_second
1624
1625 video_embeds_out = []
1626 + assert self.video_pruning_rate is not None
1627 for emb, size, video_second_per_grid_t in zip(
1628 video_embeds_split, grid_thw_list, second_per_grid_ts
1629 ):
@@ -1702,7 +1712,9 @@ class Qwen2_5_VLForConditionalGeneration(
1712 return mm_embeddings_out, positions, mrope_positions_delta
1713
1714 def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
1705 - mm_input_by_modality = {}
1715 + mm_input_by_modality: dict[
1716 + str, Qwen2_5_VLImageInputs | Qwen2_5_VLVideoInputs | None
1717 + ] = {}
1718
1719 # Preserve the order of modalities if there are multiple of them
1720 # from the order of kwargs.
@@ -1808,6 +1820,7 @@ class Qwen2_5_VLForConditionalGeneration(
1820 def get_max_frames_per_video(self) -> int:
1821 mm_registry = MULTIMODAL_REGISTRY
1822 info = mm_registry.get_processing_info(self.model_config)
1823 + assert isinstance(info, Qwen2VLProcessingInfo)
1824 max_frames_per_video = info.get_num_frames_with_most_features(
1825 seq_len=self.model_config.max_model_len,
1826 mm_counts={"video": self.multimodal_config.get_limit_per_prompt("video")},
vllm/model_executor/models/qwen2_audio.py
+4 -1
@@ -286,11 +286,14 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessing
286
287 def get_replacement_qwen2_audio(item_idx: int):
288 if "audio_num_tokens" in out_mm_data:
289 - num_features = int(out_mm_data["audio_num_tokens"][item_idx])
289 + audio_num_tokens = out_mm_data["audio_num_tokens"]
290 + assert isinstance(audio_num_tokens, torch.Tensor)
291 + num_features = int(audio_num_tokens[item_idx])
292 elif audio_output_lengths:
293 num_features = audio_output_lengths[item_idx]
294 else:
295 audio_embeds = out_mm_data["audio_embeds"][item_idx]
296 + assert isinstance(audio_embeds, torch.Tensor)
297 assert len(audio_embeds.shape) == 2, "audio_embeds must be a 2D tensor"
298 num_features = audio_embeds.shape[0]
299
vllm/model_executor/models/qwen2_moe.py
+8 -6
@@ -153,6 +153,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
153 prefix=f"{prefix}.shared_expert_gate",
154 )
155
156 + self.shared_expert: Qwen2MoeMLP | None
157 if config.shared_expert_intermediate_size > 0:
158 self.shared_expert = Qwen2MoeMLP(
159 hidden_size=config.hidden_size,
@@ -252,6 +253,12 @@ class Qwen2MoeAttention(nn.Module):
253 rope_parameters=rope_parameters,
254 dual_chunk_attention_config=dual_chunk_attention_config,
255 )
256 + attention_kwargs: dict[str, Any] = {}
257 + if dual_chunk_attention_config:
258 + attention_kwargs = {
259 + "layer_idx": extract_layer_index(prefix),
260 + "dual_chunk_attention_config": dual_chunk_attention_config,
261 + }
262 self.attn = Attention(
263 self.num_heads,
264 self.head_dim,
@@ -260,12 +267,7 @@ class Qwen2MoeAttention(nn.Module):
267 cache_config=cache_config,
268 quant_config=quant_config,
269 prefix=f"{prefix}.attn",
263 - **{
264 - "layer_idx": extract_layer_index(prefix),
265 - "dual_chunk_attention_config": dual_chunk_attention_config,
266 - }
267 - if dual_chunk_attention_config
268 - else {},
270 + **attention_kwargs,
271 )
272
273 def forward(
vllm/model_executor/models/qwen2_vl.py
+17 -10
@@ -1211,17 +1211,23 @@ class Qwen2VLForConditionalGeneration(
1211 tokens_per_second = getattr(self.config.vision_config, "tokens_per_second", 1.0)
1212 for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
1213 offset = mm_feature.mm_position.offset
1214 + data = mm_feature.data
1215 + assert data is not None
1216 if mm_feature.modality == "image":
1215 - t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
1217 + image_grid_thw = data["image_grid_thw"]
1218 + assert isinstance(image_grid_thw.data, torch.Tensor)
1219 + t, h, w = image_grid_thw.data.tolist()
1220 assert t == 1, f"Image must have 1 frame, got {t}"
1221 yield offset, 1, h // spatial_merge_size, w // spatial_merge_size, 1.0
1222 elif mm_feature.modality == "video":
1219 - t, h, w = mm_feature.data["video_grid_thw"].data.tolist()
1223 + video_grid_thw = data["video_grid_thw"]
1224 + assert isinstance(video_grid_thw.data, torch.Tensor)
1225 + t, h, w = video_grid_thw.data.tolist()
1226 second_per_grid_ts = 1.0
1221 - if mm_feature.data.get("second_per_grid_ts", None):
1222 - second_per_grid_ts = mm_feature.data[
1223 - "second_per_grid_ts"
1224 - ].data.item()
1227 + second_per_grid_ts_field = data.get("second_per_grid_ts")
1228 + if second_per_grid_ts_field is not None:
1229 + assert isinstance(second_per_grid_ts_field.data, torch.Tensor)
1230 + second_per_grid_ts = second_per_grid_ts_field.data.item()
1231 t_factor = second_per_grid_ts * tokens_per_second
1232 yield (
1233 offset,
@@ -1285,7 +1291,7 @@ class Qwen2VLForConditionalGeneration(
1291 super().__init__()
1292 config: Qwen2VLConfig = vllm_config.model_config.hf_config
1293 quant_config = vllm_config.quant_config
1288 - multimodal_config = vllm_config.model_config.multimodal_config
1294 + multimodal_config = vllm_config.model_config.get_multimodal_config()
1295 self.model_config = vllm_config.model_config
1296 self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
1297 self.config = config
@@ -1328,7 +1334,7 @@ class Qwen2VLForConditionalGeneration(
1334 image_grid_thw=image_grid_thw,
1335 )
1336
1331 - if image_embeds is not None:
1337 + else:
1338 return Qwen2VLImageEmbeddingInputs(
1339 type="image_embeds",
1340 image_embeds=image_embeds,
@@ -1352,7 +1358,7 @@ class Qwen2VLForConditionalGeneration(
1358 video_grid_thw=video_grid_thw,
1359 )
1360
1355 - if video_embeds is not None:
1361 + else:
1362 return Qwen2VLVideoEmbeddingInputs(
1363 type="video_embeds",
1364 video_embeds=video_embeds,
@@ -1411,7 +1417,7 @@ class Qwen2VLForConditionalGeneration(
1417 return video_embeds.split(sizes)
1418
1419 def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
1414 - modalities = {}
1420 + modalities: dict[str, Qwen2VLImageInputs | Qwen2VLVideoInputs | None] = {}
1421
1422 # Preserve the order of modalities if there are multiple of them
1423 # from the order of kwargs.
@@ -1481,6 +1487,7 @@ class Qwen2VLForConditionalGeneration(
1487 def get_max_frames_per_video(self) -> int:
1488 mm_registry = MULTIMODAL_REGISTRY
1489 info = mm_registry.get_processing_info(self.model_config)
1490 + assert isinstance(info, Qwen2VLProcessingInfo)
1491 max_frames_per_video = info.get_num_frames_with_most_features(
1492 seq_len=self.model_config.max_model_len,
1493 mm_counts={"video": self.multimodal_config.get_limit_per_prompt("video")},
vllm/model_executor/models/qwen3.py
+7 -6
@@ -130,6 +130,12 @@ class Qwen3Attention(nn.Module):
130 if attn_type == AttentionType.ENCODER_ONLY
131 else Attention
132 )
133 + attention_kwargs: dict[str, Any] = {}
134 + if dual_chunk_attention_config:
135 + attention_kwargs = {
136 + "layer_idx": extract_layer_index(prefix),
137 + "dual_chunk_attention_config": dual_chunk_attention_config,
138 + }
139 self.attn = attn_cls(
140 self.num_heads,
141 self.head_dim,
@@ -140,12 +146,7 @@ class Qwen3Attention(nn.Module):
146 per_layer_sliding_window=per_layer_sliding_window,
147 prefix=f"{prefix}.attn",
148 attn_type=attn_type,
143 - **{
144 - "layer_idx": extract_layer_index(prefix),
145 - "dual_chunk_attention_config": dual_chunk_attention_config,
146 - }
147 - if dual_chunk_attention_config
148 - else {},
149 + **attention_kwargs,
150 )
151 self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
152 self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
vllm/model_executor/models/qwen3_5.py
+9 -4
@@ -25,6 +25,7 @@
25 """Inference-only Qwen3.5 Series compatible with HuggingFace weights."""
26
27 from collections.abc import Iterable
28 +from typing import ClassVar
29
30 import torch
31 from torch import nn
@@ -54,6 +55,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
55 VocabParallelEmbedding,
56 )
57 from vllm.multimodal import MULTIMODAL_REGISTRY
58 +from vllm.multimodal.inputs import MultiModalFeatureSpec
59 from vllm.sequence import IntermediateTensors
60 from vllm.tokenizers.registry import cached_tokenizer_from_config
61 from vllm.transformers_utils.configs.qwen3_5 import Qwen3_5Config, Qwen3_5TextConfig
@@ -442,7 +444,7 @@ class Qwen3_5ForCausalLMBase(
444 def get_mrope_input_positions(
445 self,
446 input_tokens: list[int],
445 - mm_features: list[object],
447 + mm_features: list[MultiModalFeatureSpec],
448 ) -> tuple[torch.Tensor, int]:
449 positions = torch.arange(len(input_tokens), dtype=torch.long)
450 return positions.unsqueeze(0).expand(3, -1), 0
@@ -488,7 +490,7 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid)
490 nn.Module.__init__(self)
491 config: Qwen3_5Config = vllm_config.model_config.hf_config
492 quant_config = vllm_config.quant_config
491 - multimodal_config = vllm_config.model_config.multimodal_config
493 + multimodal_config = vllm_config.model_config.get_multimodal_config()
494
495 self.config = config
496 self.model_config = vllm_config.model_config
@@ -648,6 +650,9 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid)
650
651
652 class Qwen3_5_MoeMixtureOfExperts(MixtureOfExperts):
653 + language_model: Qwen3_5ForCausalLM
654 + num_local_physical_experts: int
655 +
656 def update_physical_experts_metadata(
657 self,
658 num_physical_experts: int,
@@ -700,14 +705,14 @@ class Qwen3_5MoeForConditionalGeneration(
705 Qwen3_5ForConditionalGeneration, Qwen3_5_MoeMixtureOfExperts
706 ):
707 # For MoE LoRA weights loading
703 - is_3d_moe_weight: bool = True
708 + is_3d_moe_weight: ClassVar[bool] = True
709
710 def __init__(self, *, vllm_config: VllmConfig, prefix: str = "model"):
711 # protocols have not __init__ method, so we need to use nn.Module.__init__
712 nn.Module.__init__(self)
713 config: Qwen3_5MoeConfig = vllm_config.model_config.hf_config
714 quant_config = vllm_config.quant_config
710 - multimodal_config = vllm_config.model_config.multimodal_config
715 + multimodal_config = vllm_config.model_config.get_multimodal_config()
716
717 self.config = config
718 self.model_config = vllm_config.model_config
vllm/model_executor/models/qwen3_5_mtp.py
+1 -1
@@ -286,7 +286,7 @@ class Qwen3_5MTP(LocalArgmaxMixin, nn.Module, SupportsMultiModal, SupportsPP):
286
287 return inputs_embeds
288
289 - def forward(
289 + def forward( # type: ignore[override]
290 self,
291 input_ids: torch.Tensor,
292 positions: torch.Tensor,
vllm/model_executor/models/qwen3_asr.py
+2 -1
@@ -274,7 +274,7 @@ class Qwen3ASRMultiModalDataParser(MultiModalDataParser):
274
275
276 class Qwen3ASRMultiModalProcessor(
277 - Qwen3OmniMoeThinkerMultiModalProcessor,
277 + Qwen3OmniMoeThinkerMultiModalProcessor[Qwen3ASRProcessingInfo]
278 ):
279 def _get_mm_fields_config(
280 self,
@@ -316,6 +316,7 @@ class Qwen3ASRMultiModalProcessor(
316 if num_features == 0:
317 audios = mm_items.get_items("audio", AudioProcessorItems)
318 audio = audios.get(item_idx)
319 + assert audio is not None
320 raise ValueError(
321 f"The audio {audio} (len={len(audio)}) is too short "
322 "to be represented inside the model"
vllm/model_executor/models/qwen3_asr_forced_aligner.py
+3 -2
@@ -72,8 +72,9 @@ class Qwen3ASRForcedAlignerForTokenClassification(
72
73 # Remove the unused generation head created by the base class;
74 # the forced aligner uses a classifier head instead.
75 - self.language_model.lm_head = None
76 - self.language_model.logits_processor = None
75 + language_model: nn.Module = self.language_model
76 + language_model.lm_head = None
77 + language_model.logits_processor = None
78
79 self.classify_num = thinker_config.classify_num
80
vllm/model_executor/models/qwen3_asr_realtime.py
+1
@@ -120,6 +120,7 @@ class Qwen3ASRRealtimeMultiModalProcessor(Qwen3ASRMultiModalProcessor):
120 )
121
122 audio_data = audios[0]
123 + assert audio_data is not None
124 audio_feature_lengths = audio_data.get("audio_feature_lengths")
125 if audio_feature_lengths is not None:
126 if isinstance(audio_feature_lengths.data, torch.Tensor):
vllm/model_executor/models/qwen3_dflash.py
+12 -5
@@ -92,7 +92,9 @@ def _dflash_layer_causal(config: Qwen3Config, layer_idx: int) -> bool:
92 if override is not None:
93 return bool(override)
94 layer_types = getattr(config, "layer_types", None)
95 - return bool(layer_types) and layer_types[layer_idx] == _SLIDING_ATTENTION
95 + if not layer_types:
96 + return False
97 + return layer_types[layer_idx] == _SLIDING_ATTENTION
98
99
100 def dflash_has_any_non_causal(config: Qwen3Config) -> bool:
@@ -105,6 +107,7 @@ def dflash_has_any_non_causal(config: Qwen3Config) -> bool:
107
108 def _get_dflash_fc_input_size(vllm_config: VllmConfig) -> int:
109 spec_config = vllm_config.speculative_config
110 + assert spec_config is not None
111 config = spec_config.draft_model_config.hf_config
112 aux_layers = get_eagle3_aux_layers_from_config(spec_config)
113 num_features_to_use = len(aux_layers) if aux_layers else config.num_hidden_layers
@@ -418,7 +421,9 @@ class DFlashQwen3Model(nn.Module):
421 prefix: str = "",
422 ) -> None:
423 super().__init__()
421 - self.config = vllm_config.speculative_config.draft_model_config.hf_config
424 + speculative_config = vllm_config.speculative_config
425 + assert speculative_config is not None
426 + self.config = speculative_config.draft_model_config.hf_config
427 self.vocab_size = self.config.vocab_size
428 self.quant_config = get_draft_quant_config(vllm_config)
429 _add_global_draft_layer_exclusions(
@@ -759,7 +764,9 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
764
765 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
766 nn.Module.__init__(self)
762 - self.draft_model_config = vllm_config.speculative_config.draft_model_config
767 + speculative_config = vllm_config.speculative_config
768 + assert speculative_config is not None
769 + self.draft_model_config = speculative_config.draft_model_config
770 self.config = self.draft_model_config.hf_config
771 if getattr(self.config, "draft_vocab_size", None) is None:
772 self.config.draft_vocab_size = getattr(self.config, "vocab_size", None)
@@ -801,7 +808,7 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
808 ) -> torch.Tensor:
809 return self.model.embed_input_ids(input_ids)
810
804 - def forward(
811 + def forward( # type: ignore[override]
812 self,
813 input_ids: torch.Tensor,
814 positions: torch.Tensor,
@@ -896,7 +903,7 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
903 model_weights["model.mask_embedding"] = mask_embedding
904 self.model.has_separate_mask_embedding = True
905
899 - orig_to_new_substr = {}
906 + orig_to_new_substr: dict[str, None] = {}
907 if not includes_draft_id_mapping:
908 orig_to_new_substr["draft_id_to_target_id"] = None
909 if not includes_embed_tokens:
vllm/model_executor/models/qwen3_dspark.py
+3 -1
@@ -233,7 +233,9 @@ class Qwen3DSparkModel(DFlashQwen3Model):
233 class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM):
234 def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
235 nn.Module.__init__(self)
236 - self.draft_model_config = vllm_config.speculative_config.draft_model_config
236 + speculative_config = vllm_config.speculative_config
237 + assert speculative_config is not None
238 + self.draft_model_config = speculative_config.draft_model_config
239 self.config = self.draft_model_config.hf_config
240 if getattr(self.config, "draft_vocab_size", None) is None:
241 self.config.draft_vocab_size = getattr(self.config, "vocab_size", None)
vllm/model_executor/models/qwen3_eagle3.py
+10 -5
@@ -101,7 +101,7 @@ class Qwen3Eagle3DecoderLayer(Qwen3DecoderLayer):
101 hidden_states = self.hidden_norm(hidden_states)
102 return hidden_states, residual
103
104 - def forward(
104 + def forward( # type: ignore[override]
105 self,
106 positions: torch.Tensor,
107 embeds: torch.Tensor,
@@ -159,7 +159,9 @@ class Qwen3Eagle3Model(nn.Module):
159 prefix: str = "",
160 ) -> None:
161 super().__init__()
162 - self.config = vllm_config.speculative_config.draft_model_config.hf_config
162 + speculative_config = vllm_config.speculative_config
163 + assert speculative_config is not None
164 + self.config = speculative_config.draft_model_config.hf_config
165 self.vocab_size = self.config.vocab_size
166
167 # Get drafter's quantization config
@@ -210,6 +212,7 @@ class Qwen3Eagle3Model(nn.Module):
212 self.config, "target_hidden_size", self.config.hidden_size
213 )
214 self.fc_input_size = target_hidden_size * num_aux_features
215 + self.input_norm: RMSNorm | None
216 if self.norm_before_fc:
217 self.input_norm = RMSNorm(
218 self.fc_input_size,
@@ -282,7 +285,9 @@ class Qwen3Eagle3Model(nn.Module):
285 class Eagle3Qwen3ForCausalLM(Qwen3ForCausalLM):
286 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
287 nn.Module.__init__(self)
285 - self.config = vllm_config.speculative_config.draft_model_config.hf_config
288 + speculative_config = vllm_config.speculative_config
289 + assert speculative_config is not None
290 + self.config = speculative_config.draft_model_config.hf_config
291 # Ensure draft_vocab_size is set
292 # default to the base vocab size when absent
293 if getattr(self.config, "draft_vocab_size", None) is None:
@@ -318,7 +323,7 @@ class Eagle3Qwen3ForCausalLM(Qwen3ForCausalLM):
323 else:
324 self.draft_id_to_target_id = None
325
321 - self.use_parallel_drafting = vllm_config.speculative_config.parallel_drafting
326 + self.use_parallel_drafting = speculative_config.parallel_drafting
327
328 if self.use_parallel_drafting:
329 self.register_buffer(
@@ -335,7 +340,7 @@ class Eagle3Qwen3ForCausalLM(Qwen3ForCausalLM):
340 ) -> torch.Tensor:
341 return self.model.embed_input_ids(input_ids)
342
338 - def forward(
343 + def forward( # type: ignore[override]
344 self,
345 input_ids: torch.Tensor,
346 positions: torch.Tensor,
vllm/model_executor/models/qwen3_moe.py
+9 -6
@@ -177,6 +177,8 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
177 shared_expert_intermediate_size = getattr(
178 config, "shared_expert_intermediate_size", 0
179 )
180 + self.shared_expert_gate: ReplicatedLinear | None
181 + self.shared_expert: Qwen3MoeMLP | None
182 if shared_expert_intermediate_size > 0:
183 self.shared_expert_gate = ReplicatedLinear(
184 config.hidden_size,
@@ -302,6 +304,12 @@ class Qwen3MoeAttention(nn.Module):
304 rope_parameters=rope_parameters,
305 dual_chunk_attention_config=dual_chunk_attention_config,
306 )
307 + attention_kwargs: dict[str, Any] = {}
308 + if dual_chunk_attention_config:
309 + attention_kwargs = {
310 + "layer_idx": extract_layer_index(prefix),
311 + "dual_chunk_attention_config": dual_chunk_attention_config,
312 + }
313 self.attn = Attention(
314 self.num_heads,
315 self.head_dim,
@@ -310,12 +318,7 @@ class Qwen3MoeAttention(nn.Module):
318 cache_config=cache_config,
319 quant_config=quant_config,
320 prefix=f"{prefix}.attn",
313 - **{
314 - "layer_idx": extract_layer_index(prefix),
315 - "dual_chunk_attention_config": dual_chunk_attention_config,
316 - }
317 - if dual_chunk_attention_config
318 - else {},
321 + **attention_kwargs,
322 )
323
324 self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
vllm/model_executor/models/qwen3_next.py
+3
@@ -757,6 +757,9 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
757
758
759 class QwenNextMixtureOfExperts(MixtureOfExperts):
760 + model: Qwen3NextModel
761 + num_local_physical_experts: int
762 +
763 def update_physical_experts_metadata(
764 self,
765 num_physical_experts: int,
vllm/model_executor/models/qwen3_omni_moe_thinker.py
+60 -25
@@ -24,7 +24,7 @@
24
25 from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
26 from functools import partial
27 -from typing import Any, cast
27 +from typing import Any, ClassVar
28
29 import numpy as np
30 import torch
@@ -41,12 +41,13 @@ from transformers.models.qwen3_omni_moe.processing_qwen3_omni_moe import (
41 Qwen3OmniMoeProcessor,
42 )
43 from transformers.models.whisper import WhisperFeatureExtractor
44 +from typing_extensions import TypeVar
45
46 from vllm.compilation.decorators import support_torch_compile
47 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
48 from vllm.config.speech_to_text import SpeechToTextParams
49 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
49 -from vllm.inputs import PromptType
50 +from vllm.inputs import PromptType, TextPrompt
51 from vllm.logger import init_logger
52 from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY
53 from vllm.model_executor.layers.attention.mm_encoder_attention import (
@@ -73,6 +74,7 @@ from vllm.multimodal.inputs import (
74 )
75 from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataItems
76 from vllm.multimodal.processing.processor import (
77 + BaseProcessingInfo,
78 HFMultiModalInputs,
79 MultiModalProcessingResult,
80 MultiModalPromptUpdates,
@@ -102,12 +104,15 @@ from .qwen2_5_omni_thinker import (
104 Qwen2_5OmniThinkerDummyInputsBuilder,
105 Qwen2_5OmniThinkerMultiModalDataParser,
106 Qwen2_5OmniThinkerMultiModalProcessor,
107 + _get_feature_tensor,
108 check_interleaved_audio_video,
109 merge_interleaved_embeddings,
110 )
111 from .qwen2_5_vl import (
112 Qwen2_5_VisionAttention,
113 + Qwen2_5_VLImageInputs,
114 Qwen2_5_VLProcessingInfo,
115 + Qwen2_5_VLVideoInputs,
116 )
117 from .qwen3_moe import Qwen3MoeForCausalLM, Qwen3MoeModel
118 from .utils import (
@@ -963,7 +968,7 @@ class Qwen3Omni_VisionTransformer(nn.Module):
968 def forward(
969 self,
970 x: torch.Tensor,
966 - grid_thw: list[list[int]],
971 + grid_thw: torch.Tensor,
972 ) -> torch.Tensor:
973 hidden_states = x.to(device=self.device, dtype=self.dtype)
974 hidden_states = self.patch_embed(hidden_states)
@@ -1201,7 +1206,7 @@ class Qwen3OmniMoeThinkerProcessingInfo(
1206 self,
1207 seq_len: int,
1208 mm_counts: Mapping[str, int] | None = None,
1204 - ) -> Mapping[str, int] | None:
1209 + ) -> Mapping[str, int]:
1210 mm_counts = mm_counts or {}
1211 requested_modalities = {m for m, c in mm_counts.items() if c > 0}
1212 mm_max_tokens: dict[str, int] = {}
@@ -1231,11 +1236,15 @@ class Qwen3OmniMoeThinkerProcessingInfo(
1236 return mm_max_tokens
1237
1238
1234 -Qwen3OmniMoeThinkerDummyInputsBuilder = Qwen2_5OmniThinkerDummyInputsBuilder
1239 +Qwen3OmniMoeThinkerDummyInputsBuilder = Qwen2_5OmniThinkerDummyInputsBuilder[
1240 + Qwen3OmniMoeThinkerProcessingInfo
1241 +]
1242 +
1243 +_I = TypeVar("_I", bound=BaseProcessingInfo, default=Qwen3OmniMoeThinkerProcessingInfo)
1244
1245
1246 class Qwen3OmniMoeThinkerMultiModalProcessor(
1238 - Qwen2_5OmniThinkerMultiModalProcessor,
1247 + Qwen2_5OmniThinkerMultiModalProcessor[_I],
1248 ):
1249 def _get_hf_mm_inputs(
1250 self,
@@ -1246,7 +1255,8 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
1255
1256 audios = hf_inputs.hf_data.get("audio")
1257 if audios:
1249 - feature_extractor = self.info.get_feature_extractor()
1258 + assert isinstance(audios, (list, tuple))
1259 + feature_extractor = self.info.get_hf_processor().feature_extractor
1260 hop_length = feature_extractor.hop_length
1261
1262 def pad_to_hop_length(x: np.ndarray) -> np.ndarray:
@@ -1283,9 +1293,10 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
1293 ):
1294 audios = mm_data.get("audio", [])
1295 if audios:
1286 - feature_extractor = self.info.get_feature_extractor(
1296 + assert isinstance(audios, (list, tuple))
1297 + feature_extractor = self.info.get_hf_processor(
1298 **hf_processor_mm_kwargs
1288 - )
1299 + ).feature_extractor
1300 hop_length = feature_extractor.hop_length
1301
1302 audio_num_frames = []
@@ -1421,6 +1432,7 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
1432 ) -> Sequence[PromptUpdate]:
1433 processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
1434 tokenizer = self.info.get_tokenizer()
1435 + assert isinstance(self.info, Qwen2_5_VLProcessingInfo)
1436 image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs)
1437 vocab = tokenizer.get_vocab()
1438
@@ -1460,6 +1472,7 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
1472 if num_features == 0:
1473 audios = mm_items.get_items("audio", AudioProcessorItems)
1474 audio = audios.get(item_idx)
1475 + assert audio is not None
1476 raise ValueError(
1477 f"The audio {audio} (len={len(audio)}) is too short "
1478 "to be represented inside the model"
@@ -1607,8 +1620,8 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
1620 Qwen3OmniMoeConditionalGenerationMixin,
1621 SupportsTranscription,
1622 ):
1610 - is_3d_moe_weight: bool = True
1611 - supports_tower_connector_lora: bool = True
1623 + is_3d_moe_weight: ClassVar[bool] = True
1624 + supports_tower_connector_lora: ClassVar[bool] = True
1625 hf_to_vllm_mapper = WeightsMapper(
1626 orig_to_new_prefix={
1627 "thinker.lm_head.": "language_model.lm_head.",
@@ -1769,7 +1782,13 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
1782 self.deepstack_input_embeds_num_tokens = 0
1783
1784 def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
1772 - mm_input_by_modality = {}
1785 + mm_input_by_modality: dict[
1786 + str,
1787 + Qwen2_5_VLImageInputs
1788 + | Qwen2_5_VLVideoInputs
1789 + | Qwen2_5OmniAudioFeatureInputs
1790 + | None,
1791 + ] = {}
1792
1793 # Preserve the order of modalities if there are multiple of them
1794 # from the order of kwargs.
@@ -1862,6 +1881,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
1881 if self.visual.deepstack_visual_indexes is not None and any(
1882 has_vision_embeddings
1883 ):
1884 + multimodal_embeddings = list(multimodal_embeddings)
1885 multiscale_len = len(self.visual.deepstack_visual_indexes)
1886 multimodal_embeddings_multiscale = []
1887
@@ -2006,8 +2026,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
2026 f
2027 for f in mm_features
2028 if f.modality == "video"
2029 + and f.data is not None
2030 and f.data.get("use_audio_in_video")
2010 - and f.data["use_audio_in_video"].data.item()
2031 + and _get_feature_tensor(f, "use_audio_in_video").item()
2032 ]
2033 audios = [f for f in mm_features if f.modality == "audio"]
2034
@@ -2015,7 +2036,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
2036 paired_audio_offsets: set[int] = set()
2037 for i, video_f in enumerate(videos_with_audio):
2038 if i < len(audios):
2018 - audio_len = audios[i].data["audio_feature_lengths"].data.item()
2039 + audio_len = int(
2040 + _get_feature_tensor(audios[i], "audio_feature_lengths").item()
2041 + )
2042 mapping[video_f.mm_position.offset] = audio_len
2043 paired_audio_offsets.add(audios[i].mm_position.offset)
2044 return mapping, paired_audio_offsets
@@ -2043,9 +2066,10 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
2066 for mm_feature in sorted_features:
2067 offset = mm_feature.mm_position.offset
2068 modality = mm_feature.modality
2069 + assert mm_feature.data is not None
2070
2071 if modality == "image":
2048 - t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
2072 + t, h, w = _get_feature_tensor(mm_feature, "image_grid_thw").tolist()
2073 yield (
2074 offset,
2075 "image",
@@ -2057,15 +2081,15 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
2081 },
2082 )
2083 elif modality == "video":
2060 - t, h, w = mm_feature.data["video_grid_thw"].data.tolist()
2084 + t, h, w = _get_feature_tensor(mm_feature, "video_grid_thw").tolist()
2085 second_per_grid_ts = 2.0
2086 if mm_feature.data.get("second_per_grid_ts"):
2063 - second_per_grid_ts = mm_feature.data[
2064 - "second_per_grid_ts"
2065 - ].data.item()
2087 + second_per_grid_ts = _get_feature_tensor(
2088 + mm_feature, "second_per_grid_ts"
2089 + ).item()
2090 use_audio_in_video = bool(
2091 mm_feature.data.get("use_audio_in_video")
2068 - and mm_feature.data["use_audio_in_video"].data.item()
2092 + and _get_feature_tensor(mm_feature, "use_audio_in_video").item()
2093 )
2094
2095 yield (
@@ -2082,7 +2106,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
2106 )
2107 elif modality == "audio":
2108 if offset not in paired_audio_offsets:
2085 - audio_len = mm_feature.data["audio_feature_lengths"].data.item()
2109 + audio_len = _get_feature_tensor(
2110 + mm_feature, "audio_feature_lengths"
2111 + ).item()
2112 yield offset, "audio", {"audio_feature_length": audio_len}
2113
2114 def _compute_interleaved_positions(
@@ -2170,8 +2196,14 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
2196 to_language = "en"
2197
2198 # Get full language names from supported_languages mapping
2173 - full_lang_name = cls.supported_languages.get(language, "")
2174 - full_lang_name_to = cls.supported_languages.get(to_language, "")
2199 + full_lang_name = (
2200 + cls.supported_languages.get(language, "") if language is not None else ""
2201 + )
2202 + full_lang_name_to = (
2203 + cls.supported_languages.get(to_language, "")
2204 + if to_language is not None
2205 + else ""
2206 + )
2207
2208 if task_type == "transcribe" and full_lang_name:
2209 instruction += f" into {full_lang_name}"
@@ -2201,8 +2233,11 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
2233 )
2234
2235 audio_data = (audio, stt_config.sample_rate)
2204 - prompts_dict = {"multi_modal_data": {"audio": audio_data}, "prompt": prompt}
2205 - return cast(PromptType, prompts_dict)
2236 + prompts_dict: TextPrompt = {
2237 + "multi_modal_data": {"audio": audio_data},
2238 + "prompt": prompt,
2239 + }
2240 + return prompts_dict
2241
2242 def get_mrope_input_positions(
2243 self,
vllm/model_executor/models/qwen3_vl.py
+34 -16
@@ -582,7 +582,9 @@ class Qwen3_VisionTransformer(nn.Module):
582 else []
583 )
584 self.num_grid_per_side = int(self.num_position_embeddings**0.5)
585 - self._rot_pos_ids_cache = LRUCache(capacity=1024)
585 + self._rot_pos_ids_cache: LRUCache[tuple[int, int, int], torch.Tensor] = (
586 + LRUCache(capacity=1024)
587 + )
588
589 use_data_parallel = is_vit_use_data_parallel()
590 self.tp_size = (
@@ -705,7 +707,7 @@ class Qwen3_VisionTransformer(nn.Module):
707 self._rot_pos_ids_cache[cache_key] = result
708 return result
709
708 - def rot_pos_emb(self, grid_thw: list[list[int]]):
710 + def rot_pos_emb(self, grid_thw: Sequence[Sequence[int]]):
711 max_grid_size = max(max(h, w) for _, h, w in grid_thw)
712 pos_ids = [
713 self.rot_pos_ids(h, w, self.spatial_merge_size)
@@ -731,7 +733,9 @@ class Qwen3_VisionTransformer(nn.Module):
733
734 return cos_combined, sin_combined
735
734 - def fast_pos_embed_interpolate(self, grid_thw: list[list[int]]) -> torch.Tensor:
736 + def fast_pos_embed_interpolate(
737 + self, grid_thw: Sequence[Sequence[int]]
738 + ) -> torch.Tensor:
739 interpolate_fn = (
740 triton_pos_embed_interpolate if HAS_TRITON else pos_embed_interpolate_native
741 )
@@ -752,7 +756,7 @@ class Qwen3_VisionTransformer(nn.Module):
756
757 def prepare_encoder_metadata(
758 self,
755 - grid_thw_list: list[list[int]],
759 + grid_thw_list: Sequence[Sequence[int]],
760 *,
761 max_batch_size: int | None = None,
762 max_frames_per_batch: int | None = None,
@@ -1005,9 +1009,10 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
1009 self,
1010 seq_len: int,
1011 mm_counts: Mapping[str, int],
1012 + max_frames_per_video: int = DUMMY_VIDEO_NUM_FRAMES,
1013 ) -> int:
1014 return super().get_num_frames_with_most_features(
1010 - seq_len, mm_counts, max_frames_per_video=DUMMY_VIDEO_NUM_FRAMES
1015 + seq_len, mm_counts, max_frames_per_video=max_frames_per_video
1016 )
1017
1018 def get_max_video_tokens(
@@ -1253,7 +1258,7 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
1258 )
1259 videos = [v.copy() for v in videos]
1260
1256 - video_items = []
1261 + video_items: list[VideoItem] = []
1262 for video in videos:
1263 video_num_frames = video.shape[0]
1264 video_metadata = {
@@ -1339,6 +1344,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
1344 # are processed into several image patches
1345 video_input_ids_lst: list[list[int]] = []
1346 if videos := hf_data.pop("videos", []):
1347 + assert isinstance(videos, list)
1348 video_grid_thw_lst = []
1349 pixel_values_videos_lst = []
1350 timestamps_per_video = []
@@ -1361,7 +1367,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
1367
1368 # NOTE: a copy of is created to update do_sample_frames,
1369 # otherwise mm_hash for the object will be incorrect.
1364 - video_mm_kwargs = dict(**hf_kwargs)
1370 + video_mm_kwargs: dict[str, Any] = dict(hf_kwargs)
1371 merged = self.info.ctx.get_merged_mm_kwargs(hf_kwargs, modality="video")
1372 if merged.keys() & {"size", "min_pixels", "max_pixels"}:
1373 video_size = dict(self.info.get_video_processor().size)
@@ -1551,6 +1557,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
1557 sampled_fps = sampled_fps[item_idx]
1558
1559 timestamps = out_item["timestamps"].data
1560 + assert isinstance(timestamps, (list, torch.Tensor))
1561 assert len(timestamps) == grid_thw[0], (
1562 f"The timestamps length({len(timestamps)}) should be equal "
1563 f"video length ({grid_thw[0]})."
@@ -1617,7 +1624,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
1624 def get_video_repl(
1625 *,
1626 tokens_per_frame: list[int],
1620 - timestamps: list[float | int] | torch.Tensor,
1627 + timestamps: Sequence[float] | torch.Tensor,
1628 tokenizer: TokenizerLike,
1629 vision_start_token_id: int,
1630 vision_end_token_id: int,
@@ -1842,7 +1849,7 @@ class Qwen3VLForConditionalGeneration(
1849 super().__init__()
1850 config: Qwen3VLConfig = vllm_config.model_config.hf_config
1851 quant_config = vllm_config.quant_config
1845 - multimodal_config = vllm_config.model_config.multimodal_config
1852 + multimodal_config = vllm_config.model_config.get_multimodal_config()
1853
1854 self.config = config
1855 self.model_config = vllm_config.model_config
@@ -2006,6 +2013,7 @@ class Qwen3VLForConditionalGeneration(
2013 def get_max_frames_per_video(self) -> int:
2014 mm_registry = MULTIMODAL_REGISTRY
2015 info = mm_registry.get_processing_info(self.model_config)
2016 + assert isinstance(info, Qwen3VLProcessingInfo)
2017 max_frames_per_video = info.get_num_frames_with_most_features(
2018 seq_len=self.model_config.max_model_len,
2019 mm_counts={"video": self.multimodal_config.get_limit_per_prompt("video")},
@@ -2254,7 +2262,7 @@ class Qwen3VLForConditionalGeneration(
2262 image_grid_thw=image_grid_thw,
2263 )
2264
2257 - if image_embeds is not None:
2265 + else:
2266 return Qwen2_5_VLImageEmbeddingInputs(
2267 type="image_embeds",
2268 image_embeds=image_embeds,
@@ -2282,7 +2290,7 @@ class Qwen3VLForConditionalGeneration(
2290 timestamps=timestamps,
2291 )
2292
2285 - if video_embeds is not None:
2293 + else:
2294 return Qwen2_5_VLVideoEmbeddingInputs(
2295 type="video_embeds",
2296 video_embeds=video_embeds,
@@ -2405,6 +2413,7 @@ class Qwen3VLForConditionalGeneration(
2413
2414 # Apply EVS to each video.
2415 video_embeds_out = []
2416 + assert video_input.timestamps is not None
2417 for video_idx, (emb, size) in enumerate(zip(video_embeds_split, grid_thw_list)):
2418 # Compute positions.
2419 timestamps = video_input.timestamps[video_idx]
@@ -2412,6 +2421,7 @@ class Qwen3VLForConditionalGeneration(
2421
2422 t, h, w = size
2423 if self.is_multimodal_pruning_enabled:
2424 + assert self.video_pruning_rate is not None
2425 # Compute the retention mask for each video (EVS or VidCom2).
2426 if self.video_pruning_method == "vidcom2":
2427 mask_fn = vidcom2_compute_retention_mask
@@ -2583,7 +2593,7 @@ class Qwen3VLForConditionalGeneration(
2593 {
2594 "video_grid_thw": MultiModalFieldElem(
2595 data=torch.tensor(video_grid_thw),
2586 - field=None, # HACK.
2596 + field=MultiModalFieldConfig.batched("video").field,
2597 ),
2598 }
2599 ),
@@ -2611,7 +2621,9 @@ class Qwen3VLForConditionalGeneration(
2621 return expanded_positions
2622
2623 def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
2614 - mm_input_by_modality = {}
2624 + mm_input_by_modality: dict[
2625 + str, Qwen2_5_VLImageInputs | Qwen2_5_VLVideoInputs | None
2626 + ] = {}
2627 for input_key in kwargs:
2628 if (
2629 input_key in ("pixel_values", "image_embeds")
@@ -2659,14 +2671,20 @@ class Qwen3VLForConditionalGeneration(
2671 """
2672 for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
2673 offset = mm_feature.mm_position.offset
2674 + data = mm_feature.data
2675 + assert data is not None
2676 if mm_feature.modality == "image":
2663 - t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
2677 + grid = data["image_grid_thw"].data
2678 + assert isinstance(grid, torch.Tensor)
2679 + t, h, w = grid.tolist()
2680 assert t == 1, f"Image must have 1 frame, got {t}"
2681 llm_grid_h = h // spatial_merge_size
2682 llm_grid_w = w // spatial_merge_size
2683 yield offset, llm_grid_h, llm_grid_w, llm_grid_h * llm_grid_w
2684 elif mm_feature.modality == "video":
2669 - t, h, w = mm_feature.data["video_grid_thw"].data.tolist()
2685 + grid = data["video_grid_thw"].data
2686 + assert isinstance(grid, torch.Tensor)
2687 + t, h, w = grid.tolist()
2688 llm_grid_h = h // spatial_merge_size
2689 llm_grid_w = w // spatial_merge_size
2690
@@ -2721,7 +2739,7 @@ class Qwen3VLForConditionalGeneration(
2739 mm_features: list[MultiModalFeatureSpec],
2740 config: Qwen3VLConfig,
2741 ):
2724 - llm_pos_ids_list = []
2742 + llm_pos_ids_list: list[np.ndarray] = []
2743 st = 0
2744 for (
2745 offset,
vllm/model_executor/models/qwen3_vl_moe.py
+8 -4
@@ -25,6 +25,7 @@
25 """Inference-only Qwen3-VL-MoE model compatible with HuggingFace weights."""
26
27 from itertools import islice
28 +from typing import ClassVar
29
30 import torch
31 from transformers.models.qwen3_vl_moe.configuration_qwen3_vl_moe import (
@@ -176,6 +177,9 @@ class Qwen3MoeLLMForCausalLM(Qwen3MoeForCausalLM):
177
178
179 class Qwen3VLMoeMixtureOfExperts(MixtureOfExperts):
180 + language_model: Qwen3MoeLLMForCausalLM
181 + num_local_physical_experts: int
182 +
183 def update_physical_experts_metadata(
184 self,
185 num_physical_experts: int,
@@ -223,7 +227,7 @@ class Qwen3VLMoeMixtureOfExperts(MixtureOfExperts):
227 class Qwen3VLMoeForConditionalGeneration(
228 Qwen3VLForConditionalGeneration, Qwen3VLMoeMixtureOfExperts
229 ):
226 - is_3d_moe_weight: bool = True
230 + is_3d_moe_weight: ClassVar[bool] = True
231 packed_modules_mapping = {
232 "qkv_proj": [
233 "q_proj",
@@ -236,7 +240,7 @@ class Qwen3VLMoeForConditionalGeneration(
240 super(Qwen3VLForConditionalGeneration, self).__init__()
241 config: Qwen3VLMoeConfig = vllm_config.model_config.hf_config
242 quant_config = vllm_config.quant_config
239 - multimodal_config = vllm_config.model_config.multimodal_config
243 + multimodal_config = vllm_config.model_config.get_multimodal_config()
244
245 self.config = config
246 self.model_config = vllm_config.model_config
@@ -293,8 +297,8 @@ class Qwen3VLMoeForConditionalGeneration(
297 )
298
299 # Whether to include the gate_up_proj mapping is determined by
296 - # the language model.
297 - self.packed_modules_mapping = (
300 + # the language model. Keep this override local to the instance.
301 + self.packed_modules_mapping = ( # type: ignore[misc]
302 self.packed_modules_mapping | self.language_model.packed_modules_mapping
303 )
304