A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Model] Switch to Fused RMS norm in Qwen2.5_VL model. (#22184)
Signed-off-by: kf <kuanfu.liu@embeddedllm.com> Signed-off-by: tjtanaavllm <tunjian.tan@amd.com> Signed-off-by: vllmellm <vllm.ellm@embeddedllm.com> Co-authored-by: kf <kuanfu.liu@embeddedllm.com>
vllmellm committed
Aug 7, 2025 at 14:05 UTC
cbc8457b2663e66beb2dedb20f3f0728b82ae603
1 file changed
+7
-7
vllm/model_executor/models/qwen2_5_vl.py
+7
-7
@@ -396,13 +396,13 @@ class Qwen2_5_VisionBlock(nn.Module):
396
max_seqlen: Optional[int] = None, # Only used for Flash Attention
397
seqlens: Optional[list[int]] = None, # Only used for xFormers
398
) -> torch.Tensor:
399
- x = x + self.attn(self.norm1(x),
400
- cu_seqlens=cu_seqlens,
401
- rotary_pos_emb=rotary_pos_emb,
402
- max_seqlen=max_seqlen,
403
- seqlens=seqlens)
404
-
405
- x = x + self.mlp(self.norm2(x))
399
+ x_attn = self.attn(self.norm1(x),
400
+ cu_seqlens=cu_seqlens,
401
+ rotary_pos_emb=rotary_pos_emb,
402
+ max_seqlen=max_seqlen,
403
+ seqlens=seqlens)
404
+ x_fused_norm, residual = self.norm2(x, residual=x_attn)
405
+ x = residual + self.mlp(x_fused_norm)
406
return x
407
408