A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 03:30 UTC
[CI] Add Async Eplb nightly CI tests (#29385)
Signed-off-by: David Chen <530634352@qq.com> Signed-off-by: WeiQing Chen <40507679+david6666666@users.noreply.github.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
WeiQing Chen committed
Dec 3, 2025 at 17:51 UTC
7fe9c1a2232275ee4cc7d65af3bc5b648543f367
5 files changed
+167
-4
.buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_async_eplb.sh
new
+73
@@ -0,0 +1,73 @@
1
+#!/usr/bin/env bash
2
+set -euxo pipefail
3
+
4
+# args: [THRESHOLD] [NUM_QUESTIONS] [START_PORT]
5
+THRESHOLD=${1:-0.25}
6
+NUM_Q=${2:-1319}
7
+PORT=${3:-8030}
8
+OUT_DIR=${OUT_DIR:-/tmp/vllm-scheduled}
9
+mkdir -p "${OUT_DIR}"
10
+
11
+wait_for_server() {
12
+ local port=$1
13
+ timeout 600 bash -c '
14
+ until curl -sf "http://127.0.0.1:'"$port"'/health" > /dev/null; do
15
+ sleep 1
16
+ done'
17
+}
18
+
19
+MODEL="deepseek-ai/DeepSeek-V2-lite"
20
+
21
+# Set BACKENDS based on platform
22
+if command -v rocm-smi &> /dev/null || [[ -d /opt/rocm ]] || [[ -n "${ROCM_PATH:-}" ]]; then
23
+ # ROCm platform
24
+ BACKENDS=("allgather_reducescatter")
25
+ # Disable MOE padding for ROCm since it is causing eplb to fail
26
+ export VLLM_ROCM_MOE_PADDING=0
27
+else
28
+ # Non-ROCm platform (CUDA/other)
29
+ BACKENDS=("deepep_high_throughput" "deepep_low_latency")
30
+fi
31
+
32
+cleanup() {
33
+ if [[ -n "${SERVER_PID:-}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then
34
+ kill "${SERVER_PID}" 2>/dev/null || true
35
+ for _ in {1..20}; do
36
+ kill -0 "${SERVER_PID}" 2>/dev/null || break
37
+ sleep 0.5
38
+ done
39
+ kill -9 "${SERVER_PID}" 2>/dev/null || true
40
+ fi
41
+}
42
+trap cleanup EXIT
43
+
44
+for BACK in "${BACKENDS[@]}"; do
45
+ VLLM_DEEP_GEMM_WARMUP=skip \
46
+ VLLM_ALL2ALL_BACKEND=$BACK \
47
+ vllm serve "$MODEL" \
48
+ --enforce-eager \
49
+ --tensor-parallel-size 2 \
50
+ --data-parallel-size 2 \
51
+ --enable-expert-parallel \
52
+ --enable-eplb \
53
+ --eplb-config '{"window_size":200,"step_interval":600,"use_async":true}' \
54
+ --trust-remote-code \
55
+ --max-model-len 2048 \
56
+ --port $PORT &
57
+ SERVER_PID=$!
58
+ wait_for_server $PORT
59
+
60
+ TAG=$(echo "$MODEL" | tr '/: \\n' '_____')
61
+ OUT="${OUT_DIR}/${TAG}_${BACK}_async_eplb.json"
62
+ python3 tests/evals/gsm8k/gsm8k_eval.py --host http://127.0.0.1 --port $PORT --num-questions ${NUM_Q} --save-results ${OUT}
63
+ python3 - <<PY
64
+import json; acc=json.load(open('${OUT}'))['accuracy']
65
+print(f"${MODEL} ${BACK}: accuracy {acc:.3f}")
66
+assert acc >= ${THRESHOLD}, f"${MODEL} ${BACK} accuracy {acc}"
67
+PY
68
+
69
+ cleanup
70
+ SERVER_PID=
71
+ sleep 1
72
+ PORT=$((PORT+1))
73
+done
.buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh
+1
@@ -50,6 +50,7 @@ for BACK in "${BACKENDS[@]}"; do
50
--data-parallel-size 2 \
51
--enable-expert-parallel \
52
--enable-eplb \
53
+ --eplb-config '{"window_size":200,"step_interval":600}' \
54
--trust-remote-code \
55
--max-model-len 2048 \
56
--port $PORT &
.buildkite/scripts/scheduled_integration_test/qwen3_next_mtp_async_eplb.sh
new
+74
@@ -0,0 +1,74 @@
1
+#!/usr/bin/env bash
2
+set -euxo pipefail
3
+
4
+# args: [THRESHOLD] [NUM_QUESTIONS] [START_PORT]
5
+THRESHOLD=${1:-0.25}
6
+NUM_Q=${2:-1319}
7
+PORT=${3:-8040}
8
+OUT_DIR=${OUT_DIR:-/tmp/vllm-scheduled}
9
+mkdir -p "${OUT_DIR}"
10
+
11
+wait_for_server() {
12
+ local port=$1
13
+ timeout 600 bash -c '
14
+ until curl -sf "http://127.0.0.1:'"$port"'/health" > /dev/null; do
15
+ sleep 1
16
+ done'
17
+}
18
+
19
+MODEL="Qwen/Qwen3-Next-80B-A3B-Instruct"
20
+
21
+# Set BACKENDS based on platform
22
+if command -v rocm-smi &> /dev/null || [[ -d /opt/rocm ]] || [[ -n "${ROCM_PATH:-}" ]]; then
23
+ # ROCm platform
24
+ BACKENDS=("allgather_reducescatter")
25
+ # Disable MOE padding for ROCm since it is causing eplb to fail
26
+ export VLLM_ROCM_MOE_PADDING=0
27
+else
28
+ # Non-ROCm platform (CUDA/other)
29
+ BACKENDS=("deepep_high_throughput" "deepep_low_latency")
30
+fi
31
+
32
+cleanup() {
33
+ if [[ -n "${SERVER_PID:-}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then
34
+ kill "${SERVER_PID}" 2>/dev/null || true
35
+ for _ in {1..20}; do
36
+ kill -0 "${SERVER_PID}" 2>/dev/null || break
37
+ sleep 0.5
38
+ done
39
+ kill -9 "${SERVER_PID}" 2>/dev/null || true
40
+ fi
41
+}
42
+trap cleanup EXIT
43
+
44
+for BACK in "${BACKENDS[@]}"; do
45
+ VLLM_DEEP_GEMM_WARMUP=skip \
46
+ VLLM_ALL2ALL_BACKEND=$BACK \
47
+ vllm serve "$MODEL" \
48
+ --enforce-eager \
49
+ --tensor-parallel-size 4 \
50
+ --enable-expert-parallel \
51
+ --enable-eplb \
52
+ --eplb-config '{"window_size":200,"step_interval":600,"use_async":true}' \
53
+ --speculative-config '{"method":"qwen3_next_mtp","num_speculative_tokens":1}' \
54
+ --trust-remote-code \
55
+ --max-model-len 2048 \
56
+ --gpu-memory-utilization 0.9 \
57
+ --port $PORT &
58
+ SERVER_PID=$!
59
+ wait_for_server $PORT
60
+
61
+ TAG=$(echo "$MODEL" | tr '/: \\n' '_____')
62
+ OUT="${OUT_DIR}/${TAG}_${BACK}.json"
63
+ python3 tests/evals/gsm8k/gsm8k_eval.py --host http://127.0.0.1 --port $PORT --num-questions ${NUM_Q} --save-results ${OUT}
64
+ python3 - <<PY
65
+import json; acc=json.load(open('${OUT}'))['accuracy']
66
+print(f"${MODEL} ${BACK}: accuracy {acc:.3f}")
67
+assert acc >= ${THRESHOLD}, f"${MODEL} ${BACK} accuracy {acc}"
68
+PY
69
+
70
+ cleanup
71
+ SERVER_PID=
72
+ sleep 1
73
+ PORT=$((PORT+1))
74
+done
.buildkite/test-pipeline.yaml
+19
-1
@@ -1373,4 +1373,22 @@ steps:
1373
num_gpus: 2
1374
working_dir: "/vllm-workspace"
1375
commands:
1376
- - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020 2 1
\ No newline at end of file
1376
+ - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020 2 1
1377
+
1378
+- label: DeepSeek V2-Lite Async EPLB Accuracy
1379
+ timeout_in_minutes: 60
1380
+ gpu: h100
1381
+ optional: true
1382
+ num_gpus: 4
1383
+ working_dir: "/vllm-workspace"
1384
+ commands:
1385
+ - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_async_eplb.sh 0.25 1319 8030
1386
+
1387
+- label: Qwen3-Next-80B-A3B-Instruct MTP Async EPLB Accuracy
1388
+ timeout_in_minutes: 60
1389
+ gpu: h100
1390
+ optional: true
1391
+ num_gpus: 4
1392
+ working_dir: "/vllm-workspace"
1393
+ commands:
1394
+ - bash .buildkite/scripts/scheduled_integration_test/qwen3_next_mtp_async_eplb.sh 0.8 1319 8040
vllm/distributed/eplb/rebalance_execute.py
-3
@@ -322,9 +322,6 @@ async def transfer_layer(
322
num_local_physical_experts = next(iter(expert_weights[0])).shape[0]
323
assert new_global_expert_indices.shape == (num_moe_layers, num_physical_experts)
324
assert num_physical_experts == ep_size * num_local_physical_experts
325
- # A buffer to hold the expert weights in one layer during the exchange.
326
- # NOTE: Currently we assume the same weights across different layers
327
- # have the same shape.
325
326
is_unchanged, is_received_locally, experts_recv_loc = move_to_buffer(
327
num_local_experts=num_local_physical_experts,