A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 13:15 UTC
[Bugfix] Fix EP weight filter breaking EPLB and NVFP4 accuracy (#37322)
Signed-off-by: Elvir Crncevic <elvircrn@gmail.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
Elvir Crnčević committed
Mar 18, 2026 at 11:30 UTC
17c47fb8691f2efd7948659952c44ef167462534
2 files changed
+12
vllm/model_executor/model_loader/default_loader.py
+7
@@ -320,6 +320,13 @@ class DefaultModelLoader(BaseModelLoader):
320
):
321
return
322
323
+ # When EPLB is enabled, redundant physical expert slots may map to
324
+ # logical experts that belong to other ranks in the default partition.
325
+ # The weight loader needs to see ALL logical expert weights so it can
326
+ # populate these redundant slots. Skip the filter entirely.
327
+ if parallel_config.enable_eplb:
328
+ return
329
+
330
num_experts = model_config.get_num_experts()
331
if num_experts <= 0:
332
return
vllm/model_executor/model_loader/ep_weight_filter.py
+5
@@ -73,4 +73,9 @@ def should_skip_weight(
73
if eid is None:
74
# Not an expert weight (dense / shared-expert / embedding) → keep.
75
return False
76
+ # Only skip heavy weight tensors, never scale/metadata tensors.
77
+ # Scale tensors are tiny and some backends need them from ALL experts
78
+ # (e.g. FlashInfer NVFP4 computes a global max of activation scales).
79
+ if not weight_name.endswith(".weight"):
80
+ return False
81
return eid not in local_expert_ids