A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[BugFix] gemma loading after quantization or LoRA. (#3553)
Taemin Lee committed
Mar 22, 2024 at 05:16 UTC
b7050ca7df640326f53e89f518f3ee045dfbbdef
1 file changed
+4
vllm/model_executor/models/gemma.py
+4
@@ -340,6 +340,10 @@ class GemmaForCausalLM(nn.Module):
340
weight_loader(param, loaded_weight, shard_id)
341
break
342
else:
343
+ # lm_head is not used in vllm as it is tied with embed_token.
344
+ # To prevent errors, skip loading lm_head.weight.
345
+ if "lm_head.weight" in name:
346
+ continue
347
# Skip loading extra bias for GPTQ models.
348
if name.endswith(".bias") and name not in params_dict:
349
continue