@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9b4b150395
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Bugfix] Ignore `lm_head` when loading embedding models (#10719)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Nov 28, 2024 at 03:05 UTC 9b4b150395d509a35031e58fb6e0f3331b532055
4 files changed +8
vllm/model_executor/models/bert.py
+2
@@ -443,6 +443,8 @@ class BertEmbeddingModel(nn.Module):
443 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
444 hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
445 weights = hf_to_vllm_mapper.apply(weights)
446 + weights = ((name, data) for name, data in weights
447 + if not name.startswith("lm_head."))
448 self.model.load_weights(weights)
449
450 def _build_model(self,
vllm/model_executor/models/gemma2.py
+2
@@ -504,4 +504,6 @@ class Gemma2EmbeddingModel(nn.Module, SupportsPP):
504 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
505 hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
506 weights = hf_to_vllm_mapper.apply(weights)
507 + weights = ((name, data) for name, data in weights
508 + if not name.startswith("lm_head."))
509 self.model.load_weights(weights)
vllm/model_executor/models/llama.py
+2
@@ -689,6 +689,8 @@ class LlamaEmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
689 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
690 hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
691 weights = hf_to_vllm_mapper.apply(weights)
692 + weights = ((name, data) for name, data in weights
693 + if not name.startswith("lm_head."))
694 self.model.load_weights(weights)
695
696 def load_kv_cache_scales(self, quantization_param_path: str) -> None:
vllm/model_executor/models/qwen2.py
+2
@@ -580,4 +580,6 @@ class Qwen2EmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
580 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
581 hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
582 weights = hf_to_vllm_mapper.apply(weights)
583 + weights = ((name, data) for name, data in weights
584 + if not name.startswith("lm_head."))
585 self.model.load_weights(weights)