@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 48d5985a08
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

Sync huggingface modifications of qwen Moe model (#4774)

eigenLiu committed May 18, 2024 at 00:43 UTC 48d5985a088c6e13e9ad9b0c7a0ce846e30b529f
1 file changed +9 -2
vllm/model_executor/models/qwen2_moe.py
+9 -2
@@ -283,8 +283,9 @@ class Qwen2MoeDecoderLayer(nn.Module):
283 cache_config=cache_config,
284 quant_config=quant_config,
285 )
286 - if (config.num_experts is not None
287 - and (layer_idx + 1) % config.decoder_sparse_step == 0):
286 + if (layer_idx not in config.mlp_only_layers) and (
287 + config.num_experts > 0 and
288 + (layer_idx + 1) % config.decoder_sparse_step == 0):
289 self.mlp = Qwen2MoeSparseMoeBlock(config=config,
290 quant_config=quant_config)
291 else:
@@ -439,6 +440,9 @@ class Qwen2MoeForCausalLM(nn.Module):
440 if (("mlp.experts." in name or "mlp.shared_expert." in name)
441 and name not in params_dict):
442 continue
443 + if name not in params_dict:
444 + continue
445 +
446 param = params_dict[name]
447 weight_loader = param.weight_loader
448 weight_loader(param, loaded_weight, shard_id)
@@ -451,6 +455,9 @@ class Qwen2MoeForCausalLM(nn.Module):
455 if (("mlp.experts." in name or "mlp.shared_expert." in name)
456 and name not in params_dict):
457 continue
458 + if name not in params_dict:
459 + continue
460 +
461 param = params_dict[name]
462 weight_loader = getattr(param, "weight_loader",
463 default_weight_loader)