A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Models][Qwen3VL] Speedup `fast_pos_embed_interpolate` (#26647)
Signed-off-by: Lukas Geiger <lukas.geiger94@gmail.com>
Lukas Geiger committed
Oct 12, 2025 at 18:20 UTC
a6049be73cb965bad04f6657de6c4d98261a5237
1 file changed
+11
-18
vllm/model_executor/models/qwen3_vl.py
+11
-18
@@ -467,8 +467,6 @@ class Qwen3_VisionTransformer(nn.Module):
467
dh_grid, dw_grid = torch.meshgrid(dh, dw, indexing="ij")
468
h_floor_grid, w_floor_grid = torch.meshgrid(h_floor, w_floor, indexing="ij")
469
h_ceil_grid, w_ceil_grid = torch.meshgrid(h_ceil, w_ceil, indexing="ij")
470
- h_floor_grid_idx = h_floor_grid * num_grid_per_side
471
- h_ceil_grid_idx = h_ceil_grid * num_grid_per_side
470
471
# original computation of weights
472
# w00 = (1 - dh_grid) * (1 - dw_grid)
@@ -480,30 +478,25 @@ class Qwen3_VisionTransformer(nn.Module):
478
w11 = dh_grid * dw_grid
479
w10 = dh_grid - w11
480
w01 = dw_grid - w11
483
- w00 = 1 - dh_grid - dw_grid + w11
481
+ w00 = 1 - dh_grid - w01
482
485
- idx00 = h_floor_grid_idx + w_floor_grid
486
- idx01 = h_floor_grid_idx + w_ceil_grid
487
- idx10 = h_ceil_grid_idx + w_floor_grid
488
- idx11 = h_ceil_grid_idx + w_ceil_grid
483
+ h_grid = torch.stack([h_floor_grid, h_floor_grid, h_ceil_grid, h_ceil_grid])
484
+ w_grid = torch.stack([w_floor_grid, w_ceil_grid, w_floor_grid, w_ceil_grid])
485
+ h_grid_idx = h_grid * num_grid_per_side
486
490
- indices = torch.stack([idx00, idx01, idx10, idx11], dim=0).reshape(4, -1)
487
+ indices = (h_grid_idx + w_grid).reshape(4, -1)
488
weights = torch.stack([w00, w01, w10, w11], dim=0).reshape(4, -1, 1)
492
- weights = weights.to(
493
- dtype=self.dtype, device=self.device, non_blocking=True
494
- )
489
+ weights = weights.to(dtype=self.dtype)
490
491
embeds = self.pos_embed(indices)
492
weighted_embeds = embeds * weights
498
- p0, p1, p2, p3 = weighted_embeds.unbind(dim=0)
499
- combined = p0 + p1 + p2 + p3
493
+ combined = weighted_embeds.sum(dim=0)
494
501
- combined = combined.view(h * w, hidden_dim)
502
- repeated = combined.unsqueeze(0).expand(t, -1, -1).contiguous()
503
- repeated = repeated.view(
504
- t, h // m_size, m_size, w // m_size, m_size, hidden_dim
495
+ combined = combined.reshape(
496
+ h // m_size, m_size, w // m_size, m_size, hidden_dim
497
)
506
- repeated = repeated.permute(0, 1, 3, 2, 4, 5).reshape(-1, hidden_dim)
498
+ combined = combined.permute(0, 2, 1, 3, 4).reshape(1, -1, hidden_dim)
499
+ repeated = combined.expand(t, -1, -1).reshape(-1, hidden_dim)
500
outputs.append(repeated)
501
502
return torch.cat(outputs, dim=0)