A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 04:15 UTC
[Model] Fix Qwen3VL and Qwen3Omni after torch.compile changes (#27705)
Signed-off-by: Lukas Geiger <lukas.geiger94@gmail.com> Signed-off-by: Roger Wang <hey@rogerw.io> Co-authored-by: Roger Wang <hey@rogerw.io>
Lukas Geiger committed
Oct 29, 2025 at 05:28 UTC
0d8161b075044168a8d6528ec4ba5ba81ee72ec3
3 files changed
+17
-16
vllm/model_executor/models/qwen2_5_vl.py
+2
-4
@@ -836,10 +836,8 @@ class Qwen2_5_VisionTransformer(nn.Module):
836
self,
837
cu_seqlens: torch.Tensor,
838
) -> tuple[torch.Tensor, torch.Tensor]:
839
- max_seqlen, seqlens = (
840
- torch.zeros(1, device=cu_seqlens.device),
841
- torch.zeros(1, device=cu_seqlens.device),
842
- )
839
+ max_seqlen = torch.zeros([], device=cu_seqlens.device)
840
+ seqlens = torch.zeros(1, device=cu_seqlens.device)
841
if (
842
self.attn_backend == _Backend.FLASH_ATTN
843
or self.attn_backend == _Backend.ROCM_AITER_FA
vllm/model_executor/models/qwen3_omni_moe_thinker.py
+8
-6
@@ -223,8 +223,8 @@ class Qwen3_VisionBlock(nn.Module):
223
x: torch.Tensor,
224
cu_seqlens: torch.Tensor,
225
rotary_pos_emb: torch.Tensor,
226
- max_seqlen: int | None = None, # Only used for Flash Attention
227
- seqlens: list[int] | None = None, # Only used for xFormers
226
+ max_seqlen: torch.Tensor, # Only used for Flash Attention
227
+ seqlens: torch.Tensor, # Only used for xFormers
228
) -> torch.Tensor:
229
x = x + self.attn(
230
self.norm1(x),
@@ -488,12 +488,13 @@ class Qwen3Omni_VisionTransformer(nn.Module):
488
def compute_attn_mask_seqlen(
489
self,
490
cu_seqlens: torch.Tensor,
491
- ) -> tuple[int | None, list[int] | None]:
492
- max_seqlen, seqlens = None, None
491
+ ) -> tuple[torch.Tensor, torch.Tensor]:
492
+ max_seqlen = torch.zeros([], device=cu_seqlens.device)
493
+ seqlens = torch.zeros(1, device=cu_seqlens.device)
494
if self.attn_backend == _Backend.FLASH_ATTN:
494
- max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max().item()
495
+ max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max()
496
elif self.attn_backend == _Backend.XFORMERS:
496
- seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()
497
+ seqlens = cu_seqlens[1:] - cu_seqlens[:-1]
498
return max_seqlen, seqlens
499
500
def forward(
@@ -1114,6 +1115,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
1115
1116
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1117
super().__init__()
1118
+ self.vllm_config = vllm_config # needed for torch compile forward context
1119
thinker_config: Qwen3OmniMoeThinkerConfig = (
1120
vllm_config.model_config.hf_config.thinker_config
1121
)
vllm/model_executor/models/qwen3_vl.py
+7
-6
@@ -231,8 +231,8 @@ class Qwen3_VisionBlock(nn.Module):
231
x: torch.Tensor,
232
cu_seqlens: torch.Tensor,
233
rotary_pos_emb: torch.Tensor,
234
- max_seqlen: int | None = None, # Only used for Flash Attention
235
- seqlens: list[int] | None = None, # Only used for xFormers
234
+ max_seqlen: torch.Tensor, # Only used for Flash Attention
235
+ seqlens: torch.Tensor, # Only used for xFormers
236
) -> torch.Tensor:
237
x = x + self.attn(
238
self.norm1(x),
@@ -512,15 +512,16 @@ class Qwen3_VisionTransformer(nn.Module):
512
def compute_attn_mask_seqlen(
513
self,
514
cu_seqlens: torch.Tensor,
515
- ) -> tuple[int | None, list[int] | None]:
516
- max_seqlen, seqlens = None, None
515
+ ) -> tuple[torch.Tensor, torch.Tensor]:
516
+ max_seqlen = torch.zeros([], device=cu_seqlens.device)
517
+ seqlens = torch.zeros(1, device=cu_seqlens.device)
518
if (
519
self.attn_backend == _Backend.FLASH_ATTN
520
or self.attn_backend == _Backend.ROCM_AITER_FA
521
):
521
- max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max().item()
522
+ max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max()
523
elif self.attn_backend == _Backend.XFORMERS:
523
- seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()
524
+ seqlens = cu_seqlens[1:] - cu_seqlens[:-1]
525
return max_seqlen, seqlens
526
527
def forward(