A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
Bugfix for whisper quantization due to fake k_proj bias (#12524)
Signed-off-by: mgoin <michael@neuralmagic.com>
Michael Goin committed
Jan 28, 2025 at 23:49 UTC
bd02164cf9eeed8436b26d62c37c1d792e97f9e8
1 file changed
+1
-1
vllm/model_executor/models/whisper.py
+1
-1
@@ -743,7 +743,7 @@ def _create_fake_bias_for_k_proj(
743
So that the bias for k_proj in qkv_proj can be initialized with zeros.
744
"""
745
for name, weight in weights:
746
- if ".self_attn.k_proj.weight" in name:
746
+ if name.endswith(".self_attn.k_proj.weight"):
747
bias = torch.zeros(weight.size(0))
748
bias_name = name.replace("weight", "bias")
749
yield from [(name, weight), (bias_name, bias)]