A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Bugfix][Model] Skip loading lm_head weights if using tie_word_embeddings (#6758)
Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>
Travis Johnson committed
Jul 31, 2024 at 20:49 UTC
630dd9e0aea166085a4c897e21a98ec752954265
4 files changed
+22
-1
vllm/model_executor/models/chameleon.py
+7
@@ -998,6 +998,13 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsVision):
998
# Models trained using ColossalAI may include these tensors in
999
# the checkpoint. Skip them.
1000
continue
1001
+
1002
+ # With tie_word_embeddings, we can skip lm_head.weight
1003
+ # The weight might appear unnecessarily in the files if the model is
1004
+ # processed with quantization, LoRA, fine-tuning, etc.
1005
+ if self.config.tie_word_embeddings and "lm_head.weight" in name:
1006
+ continue
1007
+
1008
use_default_weight_loading = False
1009
if "vqmodel" in name:
1010
if self.model.vqmodel is not None:
vllm/model_executor/models/llama.py
+5
@@ -469,6 +469,11 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
469
# Models trained using ColossalAI may include these tensors in
470
# the checkpoint. Skip them.
471
continue
472
+ # With tie_word_embeddings, we can skip lm_head.weight
473
+ # The weight might appear unnecessarily in the files if the model is
474
+ # processed with quantization, LoRA, fine-tuning, etc.
475
+ if self.config.tie_word_embeddings and "lm_head.weight" in name:
476
+ continue
477
if scale_name := get_compressed_tensors_cache_scale(name):
478
# Loading kv cache scales for compressed-tensors quantization
479
param = params_dict[scale_name]
vllm/model_executor/models/minicpm.py
+5
-1
@@ -514,7 +514,11 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA):
514
# Models trained using ColossalAI may include these tensors in
515
# the checkpoint. Skip them.
516
continue
517
-
517
+ # With tie_word_embeddings, we can skip lm_head.weight
518
+ # The weight might appear unnecessarily in the files if the model is
519
+ # processed with quantization, LoRA, fine-tuning, etc.
520
+ if self.config.tie_word_embeddings and "lm_head.weight" in name:
521
+ continue
522
for (param_name, weight_name, shard_id) in stacked_params_mapping:
523
if weight_name not in name:
524
continue
vllm/model_executor/models/olmo.py
+5
@@ -343,6 +343,11 @@ class OlmoForCausalLM(nn.Module):
343
# Models trained using ColossalAI may include these tensors in
344
# the checkpoint. Skip them.
345
continue
346
+ # With tie_word_embeddings, we can skip lm_head.weight
347
+ # The weight might appear unnecessarily in the files if the model is
348
+ # processed with quantization, LoRA, fine-tuning, etc.
349
+ if self.config.tie_word_embeddings and "lm_head.weight" in name:
350
+ continue
351
for (param_name, weight_name, shard_id) in stacked_params_mapping:
352
if weight_name not in name:
353
continue