@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 630dd9e0ae
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Bugfix][Model] Skip loading lm_head weights if using tie_word_embeddings (#6758)

Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>

Travis Johnson committed Jul 31, 2024 at 20:49 UTC 630dd9e0aea166085a4c897e21a98ec752954265
4 files changed +22 -1
vllm/model_executor/models/chameleon.py
+7
@@ -998,6 +998,13 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsVision):
998 # Models trained using ColossalAI may include these tensors in
999 # the checkpoint. Skip them.
1000 continue
1001 +
1002 + # With tie_word_embeddings, we can skip lm_head.weight
1003 + # The weight might appear unnecessarily in the files if the model is
1004 + # processed with quantization, LoRA, fine-tuning, etc.
1005 + if self.config.tie_word_embeddings and "lm_head.weight" in name:
1006 + continue
1007 +
1008 use_default_weight_loading = False
1009 if "vqmodel" in name:
1010 if self.model.vqmodel is not None:
vllm/model_executor/models/llama.py
+5
@@ -469,6 +469,11 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
469 # Models trained using ColossalAI may include these tensors in
470 # the checkpoint. Skip them.
471 continue
472 + # With tie_word_embeddings, we can skip lm_head.weight
473 + # The weight might appear unnecessarily in the files if the model is
474 + # processed with quantization, LoRA, fine-tuning, etc.
475 + if self.config.tie_word_embeddings and "lm_head.weight" in name:
476 + continue
477 if scale_name := get_compressed_tensors_cache_scale(name):
478 # Loading kv cache scales for compressed-tensors quantization
479 param = params_dict[scale_name]
vllm/model_executor/models/minicpm.py
+5 -1
@@ -514,7 +514,11 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA):
514 # Models trained using ColossalAI may include these tensors in
515 # the checkpoint. Skip them.
516 continue
517 -
517 + # With tie_word_embeddings, we can skip lm_head.weight
518 + # The weight might appear unnecessarily in the files if the model is
519 + # processed with quantization, LoRA, fine-tuning, etc.
520 + if self.config.tie_word_embeddings and "lm_head.weight" in name:
521 + continue
522 for (param_name, weight_name, shard_id) in stacked_params_mapping:
523 if weight_name not in name:
524 continue
vllm/model_executor/models/olmo.py
+5
@@ -343,6 +343,11 @@ class OlmoForCausalLM(nn.Module):
343 # Models trained using ColossalAI may include these tensors in
344 # the checkpoint. Skip them.
345 continue
346 + # With tie_word_embeddings, we can skip lm_head.weight
347 + # The weight might appear unnecessarily in the files if the model is
348 + # processed with quantization, LoRA, fine-tuning, etc.
349 + if self.config.tie_word_embeddings and "lm_head.weight" in name:
350 + continue
351 for (param_name, weight_name, shard_id) in stacked_params_mapping:
352 if weight_name not in name:
353 continue