A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Misc] Skip loading extra bias for Qwen2-VL GPTQ-Int8 (#8442)
Jee Jee Li committed
Sep 13, 2024 at 15:58 UTC
06311e295666916d3456a357cdd91dd2a03c34e2
1 file changed
+6
vllm/model_executor/models/qwen2_vl.py
+6
@@ -1055,6 +1055,9 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal):
1055
if weight_name not in name:
1056
continue
1057
name = name.replace(weight_name, param_name)
1058
+ # Skip loading extra bias for GPTQ models.
1059
+ if name.endswith(".bias") and name not in params_dict:
1060
+ continue
1061
param = params_dict[name]
1062
weight_loader = param.weight_loader
1063
weight_loader(param, loaded_weight, shard_id)
@@ -1078,6 +1081,9 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal):
1081
loaded_weight = loaded_weight.transpose(0, 1)
1082
loaded_weight = loaded_weight.reshape(-1)
1083
try:
1084
+ # Skip loading extra bias for GPTQ models.
1085
+ if name.endswith(".bias") and name not in params_dict:
1086
+ continue
1087
param = params_dict[name]
1088
except KeyError:
1089
print(params_dict.keys())