@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 07c11cf4d4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Bugfix] Fix lm_head weights tying with lora for llama (#9227)

Isotr0py committed Oct 10, 2024 at 21:11 UTC 07c11cf4d4b9a913fa52142fe134849f1e25e393
2 files changed +12 -2
vllm/model_executor/layers/vocab_parallel_embedding.py
+10 -1
@@ -443,7 +443,7 @@ class ParallelLMHead(VocabParallelEmbedding):
443 super().__init__(num_embeddings, embedding_dim, params_dtype,
444 org_num_embeddings, padding_size, quant_config,
445 prefix)
446 -
446 + self.quant_config = quant_config
447 if bias:
448 self.bias = Parameter(
449 torch.empty(self.num_embeddings_per_partition,
@@ -455,6 +455,15 @@ class ParallelLMHead(VocabParallelEmbedding):
455 else:
456 self.register_parameter("bias", None)
457
458 + def tie_weights(self, embed_tokens: VocabParallelEmbedding):
459 + """Tie the weights with word embeddings."""
460 + # GGUF quantized embed_tokens.
461 + if self.quant_config and self.quant_config.get_name() == "gguf":
462 + return embed_tokens
463 + else:
464 + self.weight = embed_tokens.weight
465 + return self
466 +
467 def forward(self, input_):
468 del input_
469 raise RuntimeError("LMHead's weights should be used in the sampler.")
vllm/model_executor/models/llama.py
+2 -1
@@ -524,7 +524,8 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
524 quant_config=quant_config,
525 )
526 if config.tie_word_embeddings:
527 - self.lm_head = self.model.embed_tokens
527 + self.lm_head = self.lm_head.tie_weights(
528 + self.model.embed_tokens)
529
530 logit_scale = getattr(config, "logit_scale", 1.0)
531 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,