A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Bugfix] Enable loading FP8 checkpoints for gpt_bigcode models (#5460)
Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com>
Thomas Parnell committed
Jun 14, 2024 at 22:28 UTC
e2afb03c92a06700d296a2e7f6565d4a4f05168c
1 file changed
+7
-1
vllm/model_executor/models/gpt_bigcode.py
+7
-1
@@ -299,4 +299,10 @@ class GPTBigCodeForCausalLM(nn.Module):
299
param = params_dict[name]
300
weight_loader = getattr(param, "weight_loader",
301
default_weight_loader)
302
- weight_loader(param, loaded_weight)
302
+ # TODO (@robertgshaw2-neuralmagic): move to fp8 linear method
303
+ if "c_attn.input_scale" in name or "c_attn.weight_scale" in name:
304
+ weight_loader(param, loaded_weight, 'q')
305
+ weight_loader(param, loaded_weight, 'k')
306
+ weight_loader(param, loaded_weight, 'v')
307
+ else:
308
+ weight_loader(param, loaded_weight)