A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 11:30 UTC
[Bugfix] Fix missing sequence_lengths in qwen3_omni_moe_thinker (#35741)
Signed-off-by: Ye (Charlotte) Qi <yeq@meta.com>
Ye (Charlotte) Qi committed
Mar 2, 2026 at 13:11 UTC
fa6a6be51978bd4b49ba0da17039e60f96dc5b13
1 file changed
+17
vllm/model_executor/models/qwen3_omni_moe_thinker.py
+17
@@ -648,6 +648,7 @@ class Qwen3_VisionBlock(nn.Module):
648
rotary_pos_emb_cos: torch.Tensor,
649
rotary_pos_emb_sin: torch.Tensor,
650
max_seqlen: torch.Tensor | None, # Only used for Flash Attention
651
+ sequence_lengths: torch.Tensor | None, # Only used for FlashInfer CuDNN backend
652
) -> torch.Tensor:
653
x = x + self.attn(
654
self.norm1(x),
@@ -655,6 +656,7 @@ class Qwen3_VisionBlock(nn.Module):
656
rotary_pos_emb_cos=rotary_pos_emb_cos,
657
rotary_pos_emb_sin=rotary_pos_emb_sin,
658
max_seqlen=max_seqlen,
659
+ sequence_lengths=sequence_lengths,
660
)
661
662
x = x + self.mlp(self.norm2(x))
@@ -975,6 +977,20 @@ class Qwen3Omni_VisionTransformer(nn.Module):
977
rotary_pos_emb_sin = rotary_pos_emb_sin.to(hidden_states.device)
978
max_seqlen = self.compute_attn_mask_seqlen(cu_seqlens)
979
980
+ # Recompute cu_seqlens in numpy from grid_thw to avoid GPU->CPU sync
981
+ grid_thw_np = grid_thw.cpu().numpy()
982
+ cu_seqlens_np = np.repeat(
983
+ grid_thw_np[:, 1] * grid_thw_np[:, 2], grid_thw_np[:, 0]
984
+ ).cumsum(axis=0, dtype=np.int32)
985
+ cu_seqlens_np = np.concatenate([np.zeros(1, dtype=np.int32), cu_seqlens_np])
986
+ sequence_lengths = MMEncoderAttention.maybe_compute_sequence_lengths(
987
+ self.attn_backend, cu_seqlens_np
988
+ )
989
+ if sequence_lengths is not None:
990
+ sequence_lengths = torch.from_numpy(sequence_lengths).to(
991
+ self.device, non_blocking=True
992
+ )
993
+
994
hidden_states_list = []
995
deepstack_visual_indexes = self.deepstack_visual_indexes
996
@@ -985,6 +1001,7 @@ class Qwen3Omni_VisionTransformer(nn.Module):
1001
rotary_pos_emb_cos=rotary_pos_emb_cos,
1002
rotary_pos_emb_sin=rotary_pos_emb_sin,
1003
max_seqlen=max_seqlen,
1004
+ sequence_lengths=sequence_lengths,
1005
)
1006
if (
1007
deepstack_visual_indexes is not None