A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Model][Qwen3VL] Compute `cu_seqlens` on CPU to remove (#26496)
Signed-off-by: Lukas Geiger <lukas.geiger94@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Lukas Geiger committed
Oct 10, 2025 at 17:42 UTC
b2155ed3175f203c9ef7de66636ffbfde1d541b7
1 file changed
+7
-4
vllm/model_executor/models/qwen3_vl.py
+7
-4
@@ -488,7 +488,9 @@ class Qwen3_VisionTransformer(nn.Module):
488
489
indices = torch.stack([idx00, idx01, idx10, idx11], dim=0).reshape(4, -1)
490
weights = torch.stack([w00, w01, w10, w11], dim=0).reshape(4, -1, 1)
491
- weights = weights.to(dtype=self.dtype, device=self.device)
491
+ weights = weights.to(
492
+ dtype=self.dtype, device=self.device, non_blocking=True
493
+ )
494
495
embeds = self.pos_embed(indices)
496
weighted_embeds = embeds * weights
@@ -524,14 +526,15 @@ class Qwen3_VisionTransformer(nn.Module):
526
x: torch.Tensor,
527
grid_thw: list[list[int]],
528
) -> torch.Tensor:
527
- hidden_states = x.to(device=self.device, dtype=self.dtype)
529
+ hidden_states = x.to(device=self.device, dtype=self.dtype, non_blocking=True)
530
hidden_states = self.patch_embed(hidden_states)
531
532
pos_embeds = self.fast_pos_embed_interpolate(grid_thw)
533
hidden_states = hidden_states + pos_embeds
534
rotary_pos_emb = self.rot_pos_emb(grid_thw)
535
+ rotary_pos_emb = rotary_pos_emb.to(hidden_states.device, non_blocking=True)
536
534
- grid_thw_tensor = torch.tensor(grid_thw, device=self.device, dtype=torch.int32)
537
+ grid_thw_tensor = torch.tensor(grid_thw, dtype=torch.int32)
538
539
cu_seqlens = torch.repeat_interleave(
540
grid_thw_tensor[:, 1] * grid_thw_tensor[:, 2], grid_thw_tensor[:, 0]
@@ -542,8 +545,8 @@ class Qwen3_VisionTransformer(nn.Module):
545
cu_seqlens = torch.cat([cu_seqlens.new_zeros(1), cu_seqlens])
546
547
hidden_states = hidden_states.unsqueeze(1)
545
- rotary_pos_emb = rotary_pos_emb.to(hidden_states.device)
548
max_seqlen, seqlens = self.compute_attn_mask_seqlen(cu_seqlens)
549
+ cu_seqlens = cu_seqlens.to(self.device, non_blocking=True)
550
551
deepstack_feature_lists = []
552
for layer_num, blk in enumerate(self.blocks):