@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e497b8aeff
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Misc] Skip loading extra bias for Qwen2-MOE GPTQ models (#8329)

Jee Jee Li committed Sep 11, 2024 at 08:59 UTC e497b8aeff5799d4ca2cfd6e01105194ebd39eac
1 file changed +8 -2
vllm/model_executor/models/qwen2_moe.py
+8 -2
@@ -469,7 +469,8 @@ class Qwen2MoeForCausalLM(nn.Module):
469 continue
470 name = name.replace(weight_name, param_name)
471 # Skip loading extra bias for GPTQ models.
472 - if name.endswith(".bias") and name not in params_dict:
472 + if ((name.endswith(".bias") or name.endswith("_bias"))
473 + and name not in params_dict):
474 continue
475 # Skip layers on other devices.
476 if is_pp_missing_parameter(name, self):
@@ -490,6 +491,10 @@ class Qwen2MoeForCausalLM(nn.Module):
491 # Skip layers on other devices.
492 if is_pp_missing_parameter(name, self):
493 continue
494 + # Skip loading extra bias for GPTQ models.
495 + if ((name.endswith(".bias") or name.endswith("_bias"))
496 + and name not in params_dict):
497 + continue
498 param = params_dict[name]
499 weight_loader = param.weight_loader
500 weight_loader(param,
@@ -500,7 +505,8 @@ class Qwen2MoeForCausalLM(nn.Module):
505 break
506 else:
507 # Skip loading extra bias for GPTQ models.
503 - if name.endswith(".bias") and name not in params_dict:
508 + if ((name.endswith(".bias") or name.endswith("_bias"))
509 + and name not in params_dict):
510 continue
511 # Skip layers on other devices.
512 if is_pp_missing_parameter(name, self):