A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Bugfix] Fix lm_head weights tying with lora for llama (#9227)
Isotr0py committed
Oct 10, 2024 at 21:11 UTC
07c11cf4d4b9a913fa52142fe134849f1e25e393
2 files changed
+12
-2
vllm/model_executor/layers/vocab_parallel_embedding.py
+10
-1
@@ -443,7 +443,7 @@ class ParallelLMHead(VocabParallelEmbedding):
443
super().__init__(num_embeddings, embedding_dim, params_dtype,
444
org_num_embeddings, padding_size, quant_config,
445
prefix)
446
-
446
+ self.quant_config = quant_config
447
if bias:
448
self.bias = Parameter(
449
torch.empty(self.num_embeddings_per_partition,
@@ -455,6 +455,15 @@ class ParallelLMHead(VocabParallelEmbedding):
455
else:
456
self.register_parameter("bias", None)
457
458
+ def tie_weights(self, embed_tokens: VocabParallelEmbedding):
459
+ """Tie the weights with word embeddings."""
460
+ # GGUF quantized embed_tokens.
461
+ if self.quant_config and self.quant_config.get_name() == "gguf":
462
+ return embed_tokens
463
+ else:
464
+ self.weight = embed_tokens.weight
465
+ return self
466
+
467
def forward(self, input_):
468
del input_
469
raise RuntimeError("LMHead's weights should be used in the sampler.")
vllm/model_executor/models/llama.py
+2
-1
@@ -524,7 +524,8 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
524
quant_config=quant_config,
525
)
526
if config.tie_word_embeddings:
527
- self.lm_head = self.model.embed_tokens
527
+ self.lm_head = self.lm_head.tie_weights(
528
+ self.model.embed_tokens)
529
530
logit_scale = getattr(config, "logit_scale", 1.0)
531
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,