@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 06311e2956
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Misc] Skip loading extra bias for Qwen2-VL GPTQ-Int8 (#8442)

Jee Jee Li committed Sep 13, 2024 at 15:58 UTC 06311e295666916d3456a357cdd91dd2a03c34e2
1 file changed +6
vllm/model_executor/models/qwen2_vl.py
+6
@@ -1055,6 +1055,9 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal):
1055 if weight_name not in name:
1056 continue
1057 name = name.replace(weight_name, param_name)
1058 + # Skip loading extra bias for GPTQ models.
1059 + if name.endswith(".bias") and name not in params_dict:
1060 + continue
1061 param = params_dict[name]
1062 weight_loader = param.weight_loader
1063 weight_loader(param, loaded_weight, shard_id)
@@ -1078,6 +1081,9 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal):
1081 loaded_weight = loaded_weight.transpose(0, 1)
1082 loaded_weight = loaded_weight.reshape(-1)
1083 try:
1084 + # Skip loading extra bias for GPTQ models.
1085 + if name.endswith(".bias") and name not in params_dict:
1086 + continue
1087 param = params_dict[name]
1088 except KeyError:
1089 print(params_dict.keys())