A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 13:15 UTC
[Performance] Add --enable-ep-weight-filter CLI option (#37351)
Signed-off-by: esmeetu <jasonailu87@gmail.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Roy Wang committed
Mar 18, 2026 at 09:36 UTC
761e0aa7a01ca764fdbe0eef563f0e8855630fe4
3 files changed
+18
-1
vllm/config/parallel.py
+7
@@ -138,6 +138,13 @@ class ParallelConfig:
138
"""Whether the deployed model is MoE (if known)."""
139
enable_expert_parallel: bool = False
140
"""Use expert parallelism instead of tensor parallelism for MoE layers."""
141
+ enable_ep_weight_filter: bool = False
142
+ """Skip non-local expert weights during model loading when expert
143
+ parallelism is active. Each rank only reads its own expert shard from
144
+ disk, which can drastically reduce storage I/O for MoE models with
145
+ per-expert weight tensors (e.g. DeepSeek, Mixtral, Kimi-K2.5). Has no
146
+ effect on 3D fused-expert checkpoints (e.g. GPT-OSS) or non-MoE
147
+ models."""
148
enable_eplb: bool = False
149
"""Enable expert parallelism load balancing for MoE layers."""
150
eplb_config: EPLBConfig = Field(default_factory=EPLBConfig)
vllm/engine/arg_utils.py
+6
@@ -419,6 +419,7 @@ class EngineArgs:
419
data_parallel_external_lb: bool = False
420
data_parallel_backend: DataParallelBackend = ParallelConfig.data_parallel_backend
421
enable_expert_parallel: bool = ParallelConfig.enable_expert_parallel
422
+ enable_ep_weight_filter: bool = ParallelConfig.enable_ep_weight_filter
423
moe_backend: MoEBackend = KernelConfig.moe_backend
424
all2all_backend: All2AllBackend = ParallelConfig.all2all_backend
425
enable_elastic_ep: bool = ParallelConfig.enable_elastic_ep
@@ -902,6 +903,10 @@ class EngineArgs:
903
"-ep",
904
**parallel_kwargs["enable_expert_parallel"],
905
)
906
+ parallel_group.add_argument(
907
+ "--enable-ep-weight-filter",
908
+ **parallel_kwargs["enable_ep_weight_filter"],
909
+ )
910
parallel_group.add_argument(
911
"--all2all-backend", **parallel_kwargs["all2all_backend"]
912
)
@@ -1731,6 +1736,7 @@ class EngineArgs:
1736
data_parallel_hybrid_lb=self.data_parallel_hybrid_lb,
1737
is_moe_model=model_config.is_moe,
1738
enable_expert_parallel=self.enable_expert_parallel,
1739
+ enable_ep_weight_filter=self.enable_ep_weight_filter,
1740
all2all_backend=self.all2all_backend,
1741
enable_elastic_ep=self.enable_elastic_ep,
1742
enable_dbo=self.enable_dbo,
vllm/model_executor/model_loader/default_loader.py
+5
-1
@@ -313,7 +313,11 @@ class DefaultModelLoader(BaseModelLoader):
313
vllm_config = get_current_vllm_config()
314
parallel_config = vllm_config.parallel_config
315
316
- if not (model_config.is_moe and parallel_config.enable_expert_parallel):
316
+ if not (
317
+ model_config.is_moe
318
+ and parallel_config.enable_expert_parallel
319
+ and parallel_config.enable_ep_weight_filter
320
+ ):
321
return
322
323
num_experts = model_config.get_num_experts()