A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix][Perf] Misc fixes for Qwen3 VL (#25238)
Signed-off-by: Roger Wang <hey@rogerw.io>
Roger Wang committed
Sep 19, 2025 at 03:46 UTC
1dfea5f4a95df8d14b46433a479a28d56e60494c
2 files changed
+12
-13
vllm/model_executor/models/qwen3_vl.py
+10
-13
@@ -1075,6 +1075,8 @@ class Qwen3VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1075
config.text_config.hidden_size)
1076
for _ in range(self.deepstack_num_level)
1077
] if self.use_deepstack else None
1078
+ self.visual_dim = config.vision_config.out_hidden_size
1079
+ self.multiscale_dim = self.visual_dim * self.deepstack_num_level
1080
1081
def _get_deepstack_input_embeds(self,
1082
num_tokens: int) -> IntermediateTensors:
@@ -1313,12 +1315,8 @@ class Qwen3VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1315
]
1316
multimodal_embeddings_cat = torch.cat(multimodal_embeddings, dim=0)
1317
1316
- visual_dim = multimodal_embeddings_cat.shape[-1] // (
1317
- self.deepstack_num_level + 1)
1318
-
1319
- main_dim, multi_dim = visual_dim, visual_dim * self.deepstack_num_level
1318
multimodal_embeddings_main, multimodal_embeddings_multiscale = torch.split( # noqa:E501
1321
- multimodal_embeddings_cat, [main_dim, multi_dim],
1319
+ multimodal_embeddings_cat, [self.visual_dim, self.multiscale_dim],
1320
dim=-1)
1321
1322
multimodal_embeddings = torch.split(multimodal_embeddings_main,
@@ -1340,10 +1338,8 @@ class Qwen3VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1338
],
1339
)
1340
deepstack_input_embeds = deepstack_input_embeds.view(
1343
- inputs_embeds.shape[0], self.deepstack_num_level,
1344
- visual_dim).contiguous()
1345
- deepstack_input_embeds = deepstack_input_embeds.permute(
1346
- 1, 0, 2).contiguous()
1341
+ inputs_embeds.shape[0], self.deepstack_num_level, self.visual_dim)
1342
+ deepstack_input_embeds = deepstack_input_embeds.permute(1, 0, 2)
1343
return deepstack_input_embeds, multimodal_embeddings
1344
1345
def get_input_embeddings(
@@ -1353,9 +1349,10 @@ class Qwen3VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1349
) -> torch.Tensor:
1350
deepstack_input_embeds = None
1351
inputs_embeds = self.language_model.get_input_embeddings(input_ids)
1356
- if multimodal_embeddings is not None and self.use_deepstack:
1357
- deepstack_input_embeds, multimodal_embeddings = self._compute_deepstack_embeds( # noqa:E501
1358
- input_ids, inputs_embeds, multimodal_embeddings)
1352
+ if multimodal_embeddings is not None:
1353
+ if self.use_deepstack:
1354
+ deepstack_input_embeds, multimodal_embeddings = self._compute_deepstack_embeds( # noqa:E501
1355
+ input_ids, inputs_embeds, multimodal_embeddings)
1356
inputs_embeds = merge_multimodal_embeddings(
1357
input_ids, inputs_embeds, multimodal_embeddings,
1358
[self.config.image_token_id, self.config.video_token_id])
@@ -1531,4 +1528,4 @@ class Qwen3VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1528
language_model="language_model",
1529
connector="model.visual.merger",
1530
tower_model="model.visual.",
1534
- )
1531
+ )
\ No newline at end of file
vllm/model_executor/models/qwen3_vl_moe.py
+2
@@ -344,3 +344,5 @@ class Qwen3VLMoeForConditionalGeneration(Qwen3VLForConditionalGeneration):
344
config.text_config.hidden_size)
345
for _ in range(self.deepstack_num_level)
346
] if self.use_deepstack else None
347
+ self.visual_dim = config.vision_config.out_hidden_size
348
+ self.multiscale_dim = self.visual_dim * self.deepstack_num_level
\ No newline at end of file