A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 15:15 UTC
[Bug] correct out dtype of rms_norm_gated native path (#35369)
Signed-off-by: Zhu, Zufang <zufang.zhu@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
zofia committed
Feb 27, 2026 at 13:19 UTC
516cf26698f07d2c92dd61bd5541ceb1376401bc
1 file changed
+1
-1
vllm/model_executor/layers/layernorm.py
+1
-1
@@ -577,7 +577,7 @@ class RMSNormGated(CustomOp):
577
if z is not None and self.norm_before_gate:
578
out = out * F.silu(z)
579
580
- return out
580
+ return out.to(x.dtype)
581
582
def forward_cuda(
583
self, x: torch.Tensor, z: torch.Tensor | None = None