A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 04:45 UTC
[Bugfix][EPLB] Prevent user-provided EPLB config from being overwritten with defaults (#29911)
Signed-off-by: Sage Moore <sage@neuralmagic.com>
Sage Moore committed
Dec 2, 2025 at 14:20 UTC
e6f114ac25967b073954f7f3dc733672d173124c
2 files changed
+9
-21
tests/distributed/test_eplb_spec_decode.py
+9
-7
@@ -22,7 +22,14 @@ def get_model_args(
22
"num_speculative_tokens": 1,
23
"max_model_len": model_max_len,
24
}
25
-
25
+ eplb_config = {
26
+ "num_redundant_experts": tp_size,
27
+ "window_size": 128,
28
+ "step_interval": 1024,
29
+ "log_balancedness": False,
30
+ }
31
+ if use_async:
32
+ eplb_config["use_async"] = True
33
model_args = {
34
"pretrained": model_name,
35
"dtype": "auto",
@@ -31,15 +38,10 @@ def get_model_args(
38
"gpu_memory_utilization": 0.7,
39
"speculative_config": speculative_config,
40
"enable_expert_parallel": True,
34
- "num_redundant_experts": tp_size,
35
- "eplb_window_size": 128,
36
- "eplb_step_interval": 1024,
37
- "eplb_log_balancedness": False,
41
+ "eplb_config": eplb_config,
42
"enable_eplb": True,
43
"max_model_len": model_max_len,
44
}
41
- if use_async:
42
- model_args["eplb_config"] = {"use_async": True}
45
return model_args
46
47
vllm/engine/arg_utils.py
-14
@@ -421,10 +421,6 @@ class EngineArgs:
421
)
422
_api_process_count: int = ParallelConfig._api_process_count
423
_api_process_rank: int = ParallelConfig._api_process_rank
424
- num_redundant_experts: int = EPLBConfig.num_redundant_experts
425
- eplb_window_size: int = EPLBConfig.window_size
426
- eplb_step_interval: int = EPLBConfig.step_interval
427
- eplb_log_balancedness: bool = EPLBConfig.log_balancedness
424
max_parallel_loading_workers: int | None = (
425
ParallelConfig.max_parallel_loading_workers
426
)
@@ -1582,16 +1578,6 @@ class EngineArgs:
1578
)
1579
self.disable_nccl_for_dp_synchronization = True
1580
1585
- # Forward the deprecated CLI args to the EPLB config.
1586
- if self.num_redundant_experts is not None:
1587
- self.eplb_config.num_redundant_experts = self.num_redundant_experts
1588
- if self.eplb_window_size is not None:
1589
- self.eplb_config.window_size = self.eplb_window_size
1590
- if self.eplb_step_interval is not None:
1591
- self.eplb_config.step_interval = self.eplb_step_interval
1592
- if self.eplb_log_balancedness is not None:
1593
- self.eplb_config.log_balancedness = self.eplb_log_balancedness
1594
-
1581
parallel_config = ParallelConfig(
1582
pipeline_parallel_size=self.pipeline_parallel_size,
1583
tensor_parallel_size=self.tensor_parallel_size,