A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[EPLB] Support ernie4.5-moe (#22100)
Signed-off-by: Haisheng Chen <langzs335@outlook.com> Signed-off-by: Haisheng Chen <60504847+HsChen-sys@users.noreply.github.com> Signed-off-by: Haisheng Chen <hac048@ucsd.edu> Co-authored-by: Haisheng Chen <langzs335@outlook.com>
Haisheng Chen committed
Oct 11, 2025 at 19:40 UTC
c5c8f5ea59f060f68b2257bb4d2066a264fc865c
1 file changed
+132
-7
vllm/model_executor/models/ernie45_moe.py
+132
-7
@@ -33,8 +33,12 @@ from transformers import PretrainedConfig
33
34
from vllm.attention import Attention
35
from vllm.compilation.decorators import support_torch_compile
36
-from vllm.config import CacheConfig, VllmConfig
37
-from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
36
+from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config
37
+from vllm.distributed import (
38
+ get_ep_group,
39
+ get_pp_group,
40
+ get_tensor_model_parallel_world_size,
41
+)
42
from vllm.logger import init_logger
43
from vllm.model_executor.layers.activation import SiluAndMul
44
from vllm.model_executor.layers.fused_moe import SharedFusedMoE
@@ -58,7 +62,7 @@ from vllm.model_executor.model_loader.weight_utils import (
62
)
63
from vllm.sequence import IntermediateTensors
64
61
-from .interfaces import SupportsLoRA, SupportsPP
65
+from .interfaces import MixtureOfExperts, SupportsLoRA, SupportsPP
66
from .utils import (
67
AutoWeightsLoader,
68
PPMissingLayer,
@@ -118,12 +122,34 @@ class Ernie4_5_MoeMoE(nn.Module):
122
config: PretrainedConfig,
123
quant_config: Optional[QuantizationConfig] = None,
124
prefix: str = "",
125
+ enable_eplb: bool = False,
126
):
127
super().__init__()
128
129
layer_idx = extract_layer_index(prefix)
130
self.layer_idx = layer_idx
131
self.tp_size = get_tensor_model_parallel_world_size()
132
+
133
+ self.moe_num_shared_experts = getattr(config, "moe_num_shared_experts", None)
134
+ self.ep_group = get_ep_group().device_group
135
+ self.ep_rank = self.ep_group.rank()
136
+ self.ep_size = self.ep_group.size()
137
+ self.n_routed_experts: int = config.moe_num_experts
138
+ self.n_shared_experts: int = self.moe_num_shared_experts
139
+
140
+ # Load balancing settings.
141
+ vllm_config = get_current_vllm_config()
142
+ parallel_config = vllm_config.parallel_config
143
+ self.enable_eplb = enable_eplb
144
+
145
+ self.n_redundant_experts = parallel_config.num_redundant_experts
146
+ self.n_logical_experts = self.n_routed_experts
147
+ self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
148
+ self.n_local_physical_experts = self.n_physical_experts // self.ep_size
149
+ self.physical_expert_start = self.ep_rank * self.n_local_physical_experts
150
+ self.physical_expert_end = (
151
+ self.physical_expert_start + self.n_local_physical_experts
152
+ )
153
self.has_shared_experts = getattr(config, "moe_num_shared_experts", 0) > 0
154
155
if self.tp_size > config.moe_num_experts:
@@ -171,6 +197,8 @@ class Ernie4_5_MoeMoE(nn.Module):
197
quant_config=quant_config,
198
prefix=f"{prefix}.experts",
199
e_score_correction_bias=self.gate.e_score_correction_bias,
200
+ enable_eplb=self.enable_eplb,
201
+ num_redundant_experts=self.n_redundant_experts,
202
)
203
204
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -298,6 +326,7 @@ class Ernie4_5_MoeDecoderLayer(nn.Module):
326
cache_config: Optional[CacheConfig] = None,
327
quant_config: Optional[QuantizationConfig] = None,
328
prefix: str = "",
329
+ enable_eplb: bool = False,
330
) -> None:
331
super().__init__()
332
self.hidden_size = config.hidden_size
@@ -338,7 +367,10 @@ class Ernie4_5_MoeDecoderLayer(nn.Module):
367
and layer_idx <= moe_layer_end_index
368
):
369
self.mlp = Ernie4_5_MoeMoE(
341
- config=config, quant_config=quant_config, prefix=f"{prefix}.mlp"
370
+ config=config,
371
+ quant_config=quant_config,
372
+ prefix=f"{prefix}.mlp",
373
+ enable_eplb=enable_eplb,
374
)
375
else:
376
self.mlp = Ernie4_5_MoeMLP(
@@ -393,6 +425,9 @@ class Ernie4_5_MoeModel(nn.Module):
425
self.padding_idx = config.pad_token_id
426
self.vocab_size = config.vocab_size
427
self.config = config
428
+ parallel_config = vllm_config.parallel_config
429
+ enable_eplb = parallel_config.enable_eplb
430
+ self.num_redundant_experts = parallel_config.num_redundant_experts
431
432
if get_pp_group().is_first_rank:
433
self.embed_tokens = VocabParallelEmbedding(
@@ -411,6 +446,7 @@ class Ernie4_5_MoeModel(nn.Module):
446
cache_config=cache_config,
447
quant_config=quant_config,
448
prefix=prefix,
449
+ enable_eplb=enable_eplb,
450
),
451
prefix=f"{prefix}.layers",
452
)
@@ -465,6 +501,7 @@ class Ernie4_5_MoeModel(nn.Module):
501
ckpt_down_proj_name="down_proj",
502
ckpt_up_proj_name="up_proj",
503
num_experts=self.config.moe_num_experts,
504
+ num_redundant_experts=self.num_redundant_experts,
505
)
506
507
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
@@ -513,15 +550,22 @@ class Ernie4_5_MoeModel(nn.Module):
550
weight_loader(param, loaded_weight, shard_id)
551
break
552
else:
553
+ is_expert_weight = False
554
for mapping in expert_params_mapping:
555
param_name, weight_name, expert_id, shard_id = mapping
556
557
if weight_name not in name:
558
continue
559
522
- name = name.replace(weight_name, param_name)
560
+ # Anyway, this is an expert weight and should not be
561
+ # attempted to load as other weights later
562
+ is_expert_weight = True
563
+
564
+ # Do not modify `name` since the loop may continue here
565
+ # Instead, create a new variable
566
+ name_mapped = name.replace(weight_name, param_name)
567
# Skip layers on other devices.
524
- if is_pp_missing_parameter(name, self):
568
+ if is_pp_missing_parameter(name_mapped, self):
569
continue
570
571
# Skip loading extra bias for GPTQ models.
@@ -541,6 +585,12 @@ class Ernie4_5_MoeModel(nn.Module):
585
)
586
break
587
else:
588
+ if is_expert_weight:
589
+ # We've checked that this is an expert weight
590
+ # However it's not mapped locally to this rank
591
+ # So we simply skip it
592
+ continue
593
+
594
# Skip loading extra bias for GPTQ models.
595
if (
596
name.endswith(".bias") or name.endswith("_bias")
@@ -563,7 +613,7 @@ class Ernie4_5_MoeModel(nn.Module):
613
return loaded_params
614
615
566
-class Ernie4_5_MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
616
+class Ernie4_5_MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA, MixtureOfExperts):
617
packed_modules_mapping = {
618
"qkv_proj": [
619
"q_proj",
@@ -605,6 +655,81 @@ class Ernie4_5_MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
655
self.model.make_empty_intermediate_tensors
656
)
657
658
+ self.expert_weights = []
659
+
660
+ # Set MoE hyperparameters
661
+ moe_layers_indices = [
662
+ i
663
+ for i in range(config.num_hidden_layers)
664
+ if (
665
+ i >= config.moe_layer_start_index
666
+ and i <= config.moe_layer_end_index
667
+ and (i + 1) % config.moe_layer_interval == 0
668
+ )
669
+ ]
670
+ self.num_moe_layers = len(moe_layers_indices)
671
+ self.num_expert_groups = 1
672
+
673
+ self.moe_layers: list[SharedFusedMoE] = []
674
+ example_moe = None
675
+ for layer in self.model.layers:
676
+ if isinstance(layer, PPMissingLayer):
677
+ continue
678
+
679
+ assert isinstance(layer, Ernie4_5_MoeDecoderLayer)
680
+ if isinstance(layer.mlp, Ernie4_5_MoeMoE):
681
+ example_moe = layer.mlp
682
+ self.moe_layers.append(layer.mlp.experts)
683
+
684
+ if example_moe is None:
685
+ logger.warning("No Ernie4_5_MoeMoE layer found in model.layers.")
686
+ self.num_logical_experts = 0
687
+ self.num_physical_experts = 0
688
+ self.num_local_physical_experts = 0
689
+ self.num_routed_experts = 0
690
+ self.num_shared_experts = 0
691
+ self.num_redundant_experts = 0
692
+ else:
693
+ self.num_logical_experts = example_moe.n_logical_experts
694
+ self.num_physical_experts = example_moe.n_physical_experts
695
+ self.num_local_physical_experts = example_moe.n_local_physical_experts
696
+ self.num_routed_experts = example_moe.n_routed_experts
697
+ self.num_shared_experts = example_moe.n_shared_experts
698
+ self.num_redundant_experts = example_moe.n_redundant_experts
699
+
700
+ def set_eplb_state(
701
+ self,
702
+ expert_load_view: torch.Tensor,
703
+ logical_to_physical_map: torch.Tensor,
704
+ logical_replica_count: torch.Tensor,
705
+ ) -> None:
706
+ for layer_idx, layer in enumerate(self.moe_layers):
707
+ # Register the expert weights.
708
+ self.expert_weights.append(layer.get_expert_weights())
709
+ layer.set_eplb_state(
710
+ moe_layer_idx=layer_idx,
711
+ expert_load_view=expert_load_view,
712
+ logical_to_physical_map=logical_to_physical_map,
713
+ logical_replica_count=logical_replica_count,
714
+ )
715
+
716
+ def update_physical_experts_metadata(
717
+ self,
718
+ num_physical_experts: int,
719
+ num_local_physical_experts: int,
720
+ ) -> None:
721
+ assert self.num_local_physical_experts == num_local_physical_experts
722
+ self.num_physical_experts = num_physical_experts
723
+ self.num_local_physical_experts = num_local_physical_experts
724
+ self.num_redundant_experts = num_physical_experts - self.num_logical_experts
725
+ for layer in self.model.layers:
726
+ if isinstance(layer.mlp, Ernie4_5_MoeMoE):
727
+ moe = layer.mlp
728
+ moe.n_local_physical_experts = num_local_physical_experts
729
+ moe.n_physical_experts = num_physical_experts
730
+ moe.n_redundant_experts = self.num_redundant_experts
731
+ moe.experts.update_expert_map()
732
+
733
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
734
return self.model.get_input_embeddings(input_ids)
735