A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Bugfix] Added Command-R GPTQ support (#3849)
Co-authored-by: Egor Tolmachev <t333ga@gmail.com>
egortolmachev committed
Apr 8, 2024 at 17:59 UTC
f46864d68dfb46ff88f574e6844f10fdb14cd3b5
1 file changed
+10
vllm/model_executor/models/commandr.py
+10
@@ -349,11 +349,21 @@ class CohereForCausalLM(nn.Module):
349
if shard_name not in name:
350
continue
351
name = name.replace(shard_name, param_name)
352
+ # Skip loading extra bias for GPTQ models.
353
+ if name.endswith(".bias") and name not in params_dict:
354
+ continue
355
param = params_dict[name]
356
weight_loader = param.weight_loader
357
weight_loader(param, loaded_weight, shard_id)
358
break
359
else:
360
+ # lm_head is not used in vllm as it is tied with embed_token.
361
+ # To prevent errors, skip loading lm_head.weight.
362
+ if "lm_head.weight" in name:
363
+ continue
364
+ # Skip loading extra bias for GPTQ models.
365
+ if name.endswith(".bias") and name not in params_dict:
366
+ continue
367
param = params_dict[name]
368
weight_loader = getattr(param, "weight_loader",
369
default_weight_loader)