@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d3c04b6a39
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Add GPTQ support for Gemma (#3200)

TechxGenus committed Mar 7, 2024 at 08:19 UTC d3c04b6a39df016504c28ec3fc27ea58ca802a28
1 file changed +6
vllm/model_executor/models/gemma.py
+6
@@ -325,11 +325,17 @@ class GemmaForCausalLM(nn.Module):
325 if shard_name not in name:
326 continue
327 name = name.replace(shard_name, param_name)
328 + # Skip loading extra bias for GPTQ models.
329 + if name.endswith(".bias") and name not in params_dict:
330 + continue
331 param = params_dict[name]
332 weight_loader = param.weight_loader
333 weight_loader(param, loaded_weight, shard_id)
334 break
335 else:
336 + # Skip loading extra bias for GPTQ models.
337 + if name.endswith(".bias") and name not in params_dict:
338 + continue
339 # GemmaRMSNorm is different from Llama's in that it multiplies
340 # (1 + weight) to the output, instead of just weight.
341 if "norm.weight" in name: