@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c5b4b11d7f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Bugfix] Fix k_proj's bias for whisper self attention (#12342)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Jan 23, 2025 at 18:15 UTC c5b4b11d7f4b69160d6a0d99771cb5c04e923a8d
1 file changed +18 -3
vllm/model_executor/models/whisper.py
+18 -3
@@ -729,7 +729,22 @@ class WhisperForConditionalGeneration(nn.Module, SupportsMultiModal):
729 def load_weights(self, weights: Iterable[Tuple[str,
730 torch.Tensor]]) -> Set[str]:
731 loader = AutoWeightsLoader(self, skip_prefixes=["proj_out."])
732 - loaded_weights = [(name, loaded_weight)
733 - for name, loaded_weight in weights]
732 mapper = WeightsMapper({".fc1.": ".mlp.fc1.", ".fc2.": ".mlp.fc2."})
735 - return loader.load_weights(loaded_weights, mapper=mapper)
733 + # add fake zeros bias for k_proj to state_dict
734 + weights = _create_fake_bias_for_k_proj(weights)
735 + return loader.load_weights(weights, mapper=mapper)
736 +
737 +
738 +def _create_fake_bias_for_k_proj(
739 + weights: Iterable[Tuple[str, torch.Tensor]]
740 +) -> Iterable[Tuple[str, torch.Tensor]]:
741 + """
742 + Create full zeros bias for k_proj weight in self-attention layers.
743 + So that the bias for k_proj in qkv_proj can be initialized with zeros.
744 + """
745 + for name, weight in weights:
746 + if ".self_attn.k_proj.weight" in name:
747 + bias = torch.zeros(weight.size(0))
748 + bias_name = name.replace("weight", "bias")
749 + yield from [(name, weight), (bias_name, bias)]
750 + yield name, weight