A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Misc] Skip loading extra bias for Qwen2-MOE GPTQ models (#8329)
Jee Jee Li committed
Sep 11, 2024 at 08:59 UTC
e497b8aeff5799d4ca2cfd6e01105194ebd39eac
1 file changed
+8
-2
vllm/model_executor/models/qwen2_moe.py
+8
-2
@@ -469,7 +469,8 @@ class Qwen2MoeForCausalLM(nn.Module):
469
continue
470
name = name.replace(weight_name, param_name)
471
# Skip loading extra bias for GPTQ models.
472
- if name.endswith(".bias") and name not in params_dict:
472
+ if ((name.endswith(".bias") or name.endswith("_bias"))
473
+ and name not in params_dict):
474
continue
475
# Skip layers on other devices.
476
if is_pp_missing_parameter(name, self):
@@ -490,6 +491,10 @@ class Qwen2MoeForCausalLM(nn.Module):
491
# Skip layers on other devices.
492
if is_pp_missing_parameter(name, self):
493
continue
494
+ # Skip loading extra bias for GPTQ models.
495
+ if ((name.endswith(".bias") or name.endswith("_bias"))
496
+ and name not in params_dict):
497
+ continue
498
param = params_dict[name]
499
weight_loader = param.weight_loader
500
weight_loader(param,
@@ -500,7 +505,8 @@ class Qwen2MoeForCausalLM(nn.Module):
505
break
506
else:
507
# Skip loading extra bias for GPTQ models.
503
- if name.endswith(".bias") and name not in params_dict:
508
+ if ((name.endswith(".bias") or name.endswith("_bias"))
509
+ and name not in params_dict):
510
continue
511
# Skip layers on other devices.
512
if is_pp_missing_parameter(name, self):