A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 19:00 UTC
[CPU][BugFix] Fix loading of w8a8int models with bias (#33582)
Signed-off-by: Fadi Arafeh <fadi.arafeh@arm.com>
Fadi Arafeh committed
Feb 6, 2026 at 11:59 UTC
f79d9dce16781b403713b8813df04d57c053bdbf
1 file changed
+8
-3
vllm/model_executor/layers/quantization/kernels/mixed_precision/dynamic_4bit.py
+8
-3
@@ -86,9 +86,14 @@ class Dynamic4bitLinearKernel(MPLinearKernel):
86
) # Float32 & Bfloat16 variants requires float32 scales
87
scales = scales.view(-1, 1) # Channel-wise scales
88
if layer.bias is not None:
89
- layer.bias = layer.bias.to(
90
- torch.float32
91
- ) # Float32 & Bfloat16 variants requires float32 bias
89
+ # Float32 & Bfloat16 variants requires float32 bias
90
+ replace_parameter(
91
+ layer,
92
+ "bias",
93
+ torch.nn.Parameter(
94
+ layer.bias.to(torch.float32), requires_grad=False
95
+ ),
96
+ )
97
else:
98
# KleidiAI kernel requires bfloat16 scales with groupwise scheme
99
scales = scales.to(torch.bfloat16)