A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[BugFix] Fix Falcon tied embeddings (#3590)
Co-authored-by: 44670 <44670@users.noreply.github.com>
Woosuk Kwon committed
Mar 24, 2024 at 06:34 UTC
af9e53496fc4dfc01b4680c1f16e38687cb3a91a
1 file changed
+7
-7
vllm/model_executor/models/falcon.py
+7
-7
@@ -37,7 +37,7 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
37
from vllm.model_executor.layers.logits_processor import LogitsProcessor
38
from vllm.model_executor.layers.sampler import Sampler
39
from vllm.model_executor.layers.vocab_parallel_embedding import (
40
- VocabParallelEmbedding, ParallelLMHead)
40
+ VocabParallelEmbedding)
41
from vllm.model_executor.parallel_utils.communication_op import (
42
tensor_model_parallel_all_reduce)
43
from vllm.model_executor.parallel_utils.parallel_state import (
@@ -370,10 +370,7 @@ class FalconForCausalLM(nn.Module):
370
self.config = config
371
self.linear_method = linear_method
372
self.transformer = FalconModel(config, linear_method)
373
- self.lm_head = ParallelLMHead(
374
- config.vocab_size,
375
- config.hidden_size,
376
- )
373
+ self.lm_head_weight = self.transformer.word_embeddings.weight
374
self.logits_processor = LogitsProcessor(config.vocab_size)
375
self.sampler = Sampler()
376
@@ -394,7 +391,7 @@ class FalconForCausalLM(nn.Module):
391
392
def compute_logits(self, hidden_states: torch.Tensor,
393
sampling_metadata: SamplingMetadata) -> torch.Tensor:
397
- logits = self.logits_processor(self.lm_head.weight, hidden_states,
394
+ logits = self.logits_processor(self.lm_head_weight, hidden_states,
395
sampling_metadata)
396
return logits
397
@@ -419,9 +416,12 @@ class FalconForCausalLM(nn.Module):
416
else:
417
total_num_kv_heads = total_num_heads
418
num_query_heads_per_kv_head = total_num_heads // total_num_kv_heads
422
- params_dict = dict(self.named_parameters())
419
+ params_dict = dict(self.named_parameters(remove_duplicate=False))
420
for name, loaded_weight in hf_model_weights_iterator(
421
model_name_or_path, cache_dir, load_format, revision):
422
+ if name == "lm_head.weight":
423
+ # Falcon uses tied embeddings.
424
+ continue
425
# Skip loading extra bias for GPTQ models.
426
if name.endswith(".bias") and name not in params_dict:
427
continue