@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / af9e53496f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[BugFix] Fix Falcon tied embeddings (#3590)

Co-authored-by: 44670 <44670@users.noreply.github.com>

Woosuk Kwon committed Mar 24, 2024 at 06:34 UTC af9e53496fc4dfc01b4680c1f16e38687cb3a91a
1 file changed +7 -7
vllm/model_executor/models/falcon.py
+7 -7
@@ -37,7 +37,7 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
37 from vllm.model_executor.layers.logits_processor import LogitsProcessor
38 from vllm.model_executor.layers.sampler import Sampler
39 from vllm.model_executor.layers.vocab_parallel_embedding import (
40 - VocabParallelEmbedding, ParallelLMHead)
40 + VocabParallelEmbedding)
41 from vllm.model_executor.parallel_utils.communication_op import (
42 tensor_model_parallel_all_reduce)
43 from vllm.model_executor.parallel_utils.parallel_state import (
@@ -370,10 +370,7 @@ class FalconForCausalLM(nn.Module):
370 self.config = config
371 self.linear_method = linear_method
372 self.transformer = FalconModel(config, linear_method)
373 - self.lm_head = ParallelLMHead(
374 - config.vocab_size,
375 - config.hidden_size,
376 - )
373 + self.lm_head_weight = self.transformer.word_embeddings.weight
374 self.logits_processor = LogitsProcessor(config.vocab_size)
375 self.sampler = Sampler()
376
@@ -394,7 +391,7 @@ class FalconForCausalLM(nn.Module):
391
392 def compute_logits(self, hidden_states: torch.Tensor,
393 sampling_metadata: SamplingMetadata) -> torch.Tensor:
397 - logits = self.logits_processor(self.lm_head.weight, hidden_states,
394 + logits = self.logits_processor(self.lm_head_weight, hidden_states,
395 sampling_metadata)
396 return logits
397
@@ -419,9 +416,12 @@ class FalconForCausalLM(nn.Module):
416 else:
417 total_num_kv_heads = total_num_heads
418 num_query_heads_per_kv_head = total_num_heads // total_num_kv_heads
422 - params_dict = dict(self.named_parameters())
419 + params_dict = dict(self.named_parameters(remove_duplicate=False))
420 for name, loaded_weight in hf_model_weights_iterator(
421 model_name_or_path, cache_dir, load_format, revision):
422 + if name == "lm_head.weight":
423 + # Falcon uses tied embeddings.
424 + continue
425 # Skip loading extra bias for GPTQ models.
426 if name.endswith(".bias") and name not in params_dict:
427 continue