A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 09:45 UTC
[EPLB] Make async EPLB default (#43219)
Signed-off-by: Markov Ilya <markovilya19@gmail.com> Co-authored-by: Markov Ilya <markovilya19@gmail.com> Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com>
Ilya Markov committed
May 29, 2026 at 20:07 UTC
4aaba00f92d4fc481aeff10db72465e26463a632
7 files changed
+15
-15
.buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh
+1
@@ -49,6 +49,7 @@ for BACK in "${BACKENDS[@]}"; do
49
--data-parallel-size 2 \
50
--enable-expert-parallel \
51
--enable-eplb \
52
+ --eplb-config '{"use_async": false}' \
53
--trust-remote-code \
54
--max-model-len 2048 \
55
--all2all-backend "$BACK" \
.buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh
+1
-1
@@ -48,7 +48,7 @@ for BACK in "${BACKENDS[@]}"; do
48
--enforce-eager \
49
--enable-eplb \
50
--all2all-backend "$BACK" \
51
- --eplb-config '{"window_size":10, "step_interval":100, "num_redundant_experts":0, "log_balancedness":true}' \
51
+ --eplb-config '{"window_size":10, "step_interval":100, "num_redundant_experts":0, "log_balancedness":true, "use_async":false}' \
52
--tensor-parallel-size "${TENSOR_PARALLEL_SIZE}" \
53
--data-parallel-size "${DATA_PARALLEL_SIZE}" \
54
--enable-expert-parallel \
.buildkite/test-amd.yaml
+3
-3
@@ -1484,7 +1484,7 @@ steps:
1484
commands:
1485
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx-fp8-and-mixed.txt
1486
1487
-- label: DeepSeek V2-Lite Accuracy (4xH100-4xMI300) # TBD
1487
+- label: DeepSeek V2-Lite Sync EPLB Accuracy (4xH100-4xMI300) # TBD
1488
timeout_in_minutes: 180
1489
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1490
agent_pool: mi300_4
@@ -1526,7 +1526,7 @@ steps:
1526
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
1527
- pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large.txt --tp-size=4
1528
1529
-- label: Qwen3-30B-A3B-FP8-block Accuracy (4xH100-4xMI300) # TBD
1529
+- label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (4xH100-4xMI300) # TBD
1530
timeout_in_minutes: 180
1531
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1532
agent_pool: mi300_4
@@ -2895,7 +2895,7 @@ steps:
2895
commands:
2896
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx-fp8-and-mixed.txt
2897
2898
-- label: Qwen3-30B-A3B-FP8-block Accuracy (B200-MI355) # TBD
2898
+- label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (B200-MI355) # TBD
2899
timeout_in_minutes: 180
2900
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
2901
agent_pool: mi355_2
.buildkite/test_areas/e2e_integration.yaml
+6
-6
@@ -2,8 +2,8 @@ group: E2E Integration
2
depends_on:
3
- image-build
4
steps:
5
-- label: DeepSeek V2-Lite Accuracy
6
- key: deepseek-v2-lite-accuracy
5
+- label: DeepSeek V2-Lite Sync EPLB Accuracy
6
+ key: deepseek-v2-lite-sync-eplb-accuracy
7
timeout_in_minutes: 60
8
device: h100
9
optional: true
@@ -12,8 +12,8 @@ steps:
12
commands:
13
- bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh 0.25 200 8010
14
15
-- label: Qwen3-30B-A3B-FP8-block Accuracy
16
- key: qwen3-30b-a3b-fp8-block-accuracy
15
+- label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy
16
+ key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy
17
timeout_in_minutes: 60
18
device: h100
19
optional: true
@@ -22,8 +22,8 @@ steps:
22
commands:
23
- bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020
24
25
-- label: Qwen3-30B-A3B-FP8-block Accuracy (B200)
26
- key: qwen3-30b-a3b-fp8-block-accuracy-b200
25
+- label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (B200)
26
+ key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-b200
27
timeout_in_minutes: 60
28
device: b200-k8s
29
optional: true
docs/serving/expert_parallel_deployment.md
+1
-1
@@ -151,7 +151,7 @@ Configure EPLB with the `--eplb-config` argument, which accepts a JSON string. T
151
| `step_interval` | Frequency of rebalancing (every N engine steps) | 3000 |
152
| `log_balancedness` | Log balancedness metrics (avg tokens per expert ÷ max tokens per expert) | `false` |
153
| `num_redundant_experts` | Additional global experts per EP rank beyond equal distribution | `0` |
154
-| `use_async` | Use non-blocking EPLB for reduced latency overhead | `false` |
154
+| `use_async` | Use non-blocking EPLB for reduced latency overhead | `true` |
155
| `policy` | The policy type for expert parallel load balancing | `"default"` |
156
| `communicator` | Backend for expert weight transfers: `"torch_nccl"`, `"torch_gloo"`, `"pynccl"`, `"nixl"`, or `null` (auto) | `null` |
157
tests/distributed/test_eplb_spec_decode.py
+2
-3
@@ -15,7 +15,7 @@ def get_model_args(
15
spec_method: str,
16
tp_size: int,
17
model_max_len: int,
18
- use_async: bool = False,
18
+ use_async: bool = True,
19
) -> dict:
20
speculative_config = {
21
"method": spec_method,
@@ -28,9 +28,8 @@ def get_model_args(
28
"window_size": 128,
29
"step_interval": 1024,
30
"log_balancedness": False,
31
+ "use_async": use_async,
32
}
32
- if use_async:
33
- eplb_config["use_async"] = True
33
model_args = {
34
"pretrained": model_name,
35
"dtype": "auto",
vllm/config/parallel.py
+1
-1
@@ -78,7 +78,7 @@ class EPLBConfig:
78
"""
79
Interval for logging the balancedness.
80
"""
81
- use_async: bool = False
81
+ use_async: bool = True
82
"""
83
Whether to use non-blocking EPLB.
84
"""