@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f46864d68d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Bugfix] Added Command-R GPTQ support (#3849)

Co-authored-by: Egor Tolmachev <t333ga@gmail.com>

egortolmachev committed Apr 8, 2024 at 17:59 UTC f46864d68dfb46ff88f574e6844f10fdb14cd3b5
1 file changed +10
vllm/model_executor/models/commandr.py
+10
@@ -349,11 +349,21 @@ class CohereForCausalLM(nn.Module):
349 if shard_name not in name:
350 continue
351 name = name.replace(shard_name, param_name)
352 + # Skip loading extra bias for GPTQ models.
353 + if name.endswith(".bias") and name not in params_dict:
354 + continue
355 param = params_dict[name]
356 weight_loader = param.weight_loader
357 weight_loader(param, loaded_weight, shard_id)
358 break
359 else:
360 + # lm_head is not used in vllm as it is tied with embed_token.
361 + # To prevent errors, skip loading lm_head.weight.
362 + if "lm_head.weight" in name:
363 + continue
364 + # Skip loading extra bias for GPTQ models.
365 + if name.endswith(".bias") and name not in params_dict:
366 + continue
367 param = params_dict[name]
368 weight_loader = getattr(param, "weight_loader",
369 default_weight_loader)