A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Bugfix] Ignore `lm_head` when loading embedding models (#10719)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Nov 28, 2024 at 03:05 UTC
9b4b150395d509a35031e58fb6e0f3331b532055
4 files changed
+8
vllm/model_executor/models/bert.py
+2
@@ -443,6 +443,8 @@ class BertEmbeddingModel(nn.Module):
443
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
444
hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
445
weights = hf_to_vllm_mapper.apply(weights)
446
+ weights = ((name, data) for name, data in weights
447
+ if not name.startswith("lm_head."))
448
self.model.load_weights(weights)
449
450
def _build_model(self,
vllm/model_executor/models/gemma2.py
+2
@@ -504,4 +504,6 @@ class Gemma2EmbeddingModel(nn.Module, SupportsPP):
504
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
505
hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
506
weights = hf_to_vllm_mapper.apply(weights)
507
+ weights = ((name, data) for name, data in weights
508
+ if not name.startswith("lm_head."))
509
self.model.load_weights(weights)
vllm/model_executor/models/llama.py
+2
@@ -689,6 +689,8 @@ class LlamaEmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
689
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
690
hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
691
weights = hf_to_vllm_mapper.apply(weights)
692
+ weights = ((name, data) for name, data in weights
693
+ if not name.startswith("lm_head."))
694
self.model.load_weights(weights)
695
696
def load_kv_cache_scales(self, quantization_param_path: str) -> None:
vllm/model_executor/models/qwen2.py
+2
@@ -580,4 +580,6 @@ class Qwen2EmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
580
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
581
hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
582
weights = hf_to_vllm_mapper.apply(weights)
583
+ weights = ((name, data) for name, data in weights
584
+ if not name.startswith("lm_head."))
585
self.model.load_weights(weights)