A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Bugfix] Fix k_proj's bias for whisper self attention (#12342)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Jan 23, 2025 at 18:15 UTC
c5b4b11d7f4b69160d6a0d99771cb5c04e923a8d
1 file changed
+18
-3
vllm/model_executor/models/whisper.py
+18
-3
@@ -729,7 +729,22 @@ class WhisperForConditionalGeneration(nn.Module, SupportsMultiModal):
729
def load_weights(self, weights: Iterable[Tuple[str,
730
torch.Tensor]]) -> Set[str]:
731
loader = AutoWeightsLoader(self, skip_prefixes=["proj_out."])
732
- loaded_weights = [(name, loaded_weight)
733
- for name, loaded_weight in weights]
732
mapper = WeightsMapper({".fc1.": ".mlp.fc1.", ".fc2.": ".mlp.fc2."})
735
- return loader.load_weights(loaded_weights, mapper=mapper)
733
+ # add fake zeros bias for k_proj to state_dict
734
+ weights = _create_fake_bias_for_k_proj(weights)
735
+ return loader.load_weights(weights, mapper=mapper)
736
+
737
+
738
+def _create_fake_bias_for_k_proj(
739
+ weights: Iterable[Tuple[str, torch.Tensor]]
740
+) -> Iterable[Tuple[str, torch.Tensor]]:
741
+ """
742
+ Create full zeros bias for k_proj weight in self-attention layers.
743
+ So that the bias for k_proj in qkv_proj can be initialized with zeros.
744
+ """
745
+ for name, weight in weights:
746
+ if ".self_attn.k_proj.weight" in name:
747
+ bias = torch.zeros(weight.size(0))
748
+ bias_name = name.replace("weight", "bias")
749
+ yield from [(name, weight), (bias_name, bias)]
750
+ yield name, weight