A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 23:30 UTC
[Model][QwenVL] Simplify cos/sin rotary embedding indexing (#28962)
Signed-off-by: Lukas Geiger <lukas.geiger94@gmail.com>
Lukas Geiger committed
Nov 19, 2025 at 05:43 UTC
3d4e7d34be856cc4f54033e6a019059afacb5e76
5 files changed
+11
-42
vllm/model_executor/models/glm4_1v.py
+2
-7
@@ -797,13 +797,8 @@ class Glm4vVisionTransformer(nn.Module):
797
# Use pre-computed cos_sin_cache from RotaryEmbedding
798
cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size)
799
800
- cos_h = cos[pos_ids[:, 0]] # (num_tokens, rotary_dim // 2)
801
- cos_w = cos[pos_ids[:, 1]]
802
- sin_h = sin[pos_ids[:, 0]]
803
- sin_w = sin[pos_ids[:, 1]]
804
-
805
- cos_combined = torch.cat([cos_h, cos_w], dim=-1)
806
- sin_combined = torch.cat([sin_h, sin_w], dim=-1)
800
+ cos_combined = cos[pos_ids].flatten(1)
801
+ sin_combined = sin[pos_ids].flatten(1)
802
return cos_combined, sin_combined, pos_ids
803
804
def compute_attn_mask_seqlen(
vllm/model_executor/models/qwen2_5_vl.py
+2
-7
@@ -738,13 +738,8 @@ class Qwen2_5_VisionTransformer(nn.Module):
738
# Use pre-computed cos_sin_cache from RotaryEmbedding
739
cos, sin = self.rotary_pos_emb.get_cos_sin(max_size)
740
741
- cos_h = cos[pos_ids[:, 0]] # (num_tokens, rotary_dim // 2)
742
- cos_w = cos[pos_ids[:, 1]]
743
- sin_h = sin[pos_ids[:, 0]]
744
- sin_w = sin[pos_ids[:, 1]]
745
-
746
- cos_combined = torch.cat([cos_h, cos_w], dim=-1)
747
- sin_combined = torch.cat([sin_h, sin_w], dim=-1)
741
+ cos_combined = cos[pos_ids].flatten(1)
742
+ sin_combined = sin[pos_ids].flatten(1)
743
744
cos_combined = cos_combined.reshape(
745
cos_combined.shape[0] // self.spatial_merge_unit,
vllm/model_executor/models/qwen2_vl.py
+2
-7
@@ -724,13 +724,8 @@ class Qwen2VisionTransformer(nn.Module):
724
# Use pre-computed cos_sin_cache from RotaryEmbedding
725
cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size)
726
727
- cos_h = cos[pos_ids[:, 0]] # (num_tokens, rotary_dim // 2)
728
- cos_w = cos[pos_ids[:, 1]]
729
- sin_h = sin[pos_ids[:, 0]]
730
- sin_w = sin[pos_ids[:, 1]]
731
-
732
- cos_combined = torch.cat([cos_h, cos_w], dim=-1)
733
- sin_combined = torch.cat([sin_h, sin_w], dim=-1)
727
+ cos_combined = cos[pos_ids].flatten(1)
728
+ sin_combined = sin[pos_ids].flatten(1)
729
return cos_combined, sin_combined
730
731
def compute_attn_mask_seqlen(
vllm/model_executor/models/qwen3_omni_moe_thinker.py
+2
-7
@@ -428,13 +428,8 @@ class Qwen3Omni_VisionTransformer(nn.Module):
428
# Use pre-computed cos_sin_cache from RotaryEmbedding
429
cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size)
430
431
- cos_h = cos[pos_ids[:, 0]] # (num_tokens, rotary_dim // 2)
432
- cos_w = cos[pos_ids[:, 1]]
433
- sin_h = sin[pos_ids[:, 0]]
434
- sin_w = sin[pos_ids[:, 1]]
435
-
436
- cos_combined = torch.cat([cos_h, cos_w], dim=-1)
437
- sin_combined = torch.cat([sin_h, sin_w], dim=-1)
431
+ cos_combined = cos[pos_ids].flatten(1)
432
+ sin_combined = sin[pos_ids].flatten(1)
433
434
return cos_combined, sin_combined
435
vllm/model_executor/models/qwen3_vl.py
+3
-14
@@ -459,18 +459,13 @@ class Qwen3_VisionTransformer(nn.Module):
459
else self.rot_pos_ids(h, w, self.spatial_merge_size).repeat(t, 1)
460
for t, h, w in grid_thw
461
]
462
- pos_ids = torch.cat(pos_ids, dim=0)
462
+ pos_ids = torch.cat(pos_ids, dim=0).to(self.device, non_blocking=True)
463
464
# Use pre-computed cos_sin_cache from RotaryEmbedding
465
cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size)
466
467
- cos_h = cos[pos_ids[:, 0]] # (num_tokens, rotary_dim // 2)
468
- cos_w = cos[pos_ids[:, 1]]
469
- sin_h = sin[pos_ids[:, 0]]
470
- sin_w = sin[pos_ids[:, 1]]
471
-
472
- cos_combined = torch.cat([cos_h, cos_w], dim=-1)
473
- sin_combined = torch.cat([sin_h, sin_w], dim=-1)
467
+ cos_combined = cos[pos_ids].flatten(1)
468
+ sin_combined = sin[pos_ids].flatten(1)
469
470
return cos_combined, sin_combined
471
@@ -566,12 +561,6 @@ class Qwen3_VisionTransformer(nn.Module):
561
pos_embeds = self.fast_pos_embed_interpolate(grid_thw_list)
562
hidden_states = hidden_states + pos_embeds
563
rotary_pos_emb_cos, rotary_pos_emb_sin = self.rot_pos_emb(grid_thw_list)
569
- rotary_pos_emb_cos = rotary_pos_emb_cos.to(
570
- hidden_states.device, non_blocking=True
571
- )
572
- rotary_pos_emb_sin = rotary_pos_emb_sin.to(
573
- hidden_states.device, non_blocking=True
574
- )
564
565
cu_seqlens = torch.repeat_interleave(
566
grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]