A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 12:15 UTC
fix gemma3 results all zero (#17364)
Signed-off-by: mayuyuace <qiming1.zhang@intel.com>
Qiming Zhang committed
Apr 29, 2025 at 09:40 UTC
d3cf61b89bc53aa7709932ab43e7630b9a71f2b3
1 file changed
+4
-1
vllm/model_executor/layers/layernorm.py
+4
-1
@@ -241,7 +241,10 @@ class GemmaRMSNorm(CustomOp):
241
"""PyTorch-native implementation equivalent to forward()."""
242
orig_dtype = x.dtype
243
if residual is not None:
244
- x = x + residual
244
+ if orig_dtype == torch.float16:
245
+ x = x + residual.float()
246
+ else:
247
+ x = x + residual
248
residual = x
249
250
x = x.float()