@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bd02164cf9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

Bugfix for whisper quantization due to fake k_proj bias (#12524)

Signed-off-by: mgoin <michael@neuralmagic.com>

Michael Goin committed Jan 28, 2025 at 23:49 UTC bd02164cf9eeed8436b26d62c37c1d792e97f9e8
1 file changed +1 -1
vllm/model_executor/models/whisper.py
+1 -1
@@ -743,7 +743,7 @@ def _create_fake_bias_for_k_proj(
743 So that the bias for k_proj in qkv_proj can be initialized with zeros.
744 """
745 for name, weight in weights:
746 - if ".self_attn.k_proj.weight" in name:
746 + if name.endswith(".self_attn.k_proj.weight"):
747 bias = torch.zeros(weight.size(0))
748 bias_name = name.replace("weight", "bias")
749 yield from [(name, weight), (bias_name, bias)]