A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 16:15 UTC
Revert "[CI] Add Async Eplb nightly CI tests (#29385)" (#30431)
Sage Moore committed
Dec 10, 2025 at 16:48 UTC
b4054c8ab469a9c3c3c77a1c2f22f54a69b87145
5 files changed
+4
-167
.buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_async_eplb.sh
deleted
-73
@@ -1,73 +0,0 @@
1
-#!/usr/bin/env bash
2
-set -euxo pipefail
3
-
4
-# args: [THRESHOLD] [NUM_QUESTIONS] [START_PORT]
5
-THRESHOLD=${1:-0.25}
6
-NUM_Q=${2:-1319}
7
-PORT=${3:-8030}
8
-OUT_DIR=${OUT_DIR:-/tmp/vllm-scheduled}
9
-mkdir -p "${OUT_DIR}"
10
-
11
-wait_for_server() {
12
- local port=$1
13
- timeout 600 bash -c '
14
- until curl -sf "http://127.0.0.1:'"$port"'/health" > /dev/null; do
15
- sleep 1
16
- done'
17
-}
18
-
19
-MODEL="deepseek-ai/DeepSeek-V2-lite"
20
-
21
-# Set BACKENDS based on platform
22
-if command -v rocm-smi &> /dev/null || [[ -d /opt/rocm ]] || [[ -n "${ROCM_PATH:-}" ]]; then
23
- # ROCm platform
24
- BACKENDS=("allgather_reducescatter")
25
- # Disable MOE padding for ROCm since it is causing eplb to fail
26
- export VLLM_ROCM_MOE_PADDING=0
27
-else
28
- # Non-ROCm platform (CUDA/other)
29
- BACKENDS=("deepep_high_throughput" "deepep_low_latency")
30
-fi
31
-
32
-cleanup() {
33
- if [[ -n "${SERVER_PID:-}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then
34
- kill "${SERVER_PID}" 2>/dev/null || true
35
- for _ in {1..20}; do
36
- kill -0 "${SERVER_PID}" 2>/dev/null || break
37
- sleep 0.5
38
- done
39
- kill -9 "${SERVER_PID}" 2>/dev/null || true
40
- fi
41
-}
42
-trap cleanup EXIT
43
-
44
-for BACK in "${BACKENDS[@]}"; do
45
- VLLM_DEEP_GEMM_WARMUP=skip \
46
- VLLM_ALL2ALL_BACKEND=$BACK \
47
- vllm serve "$MODEL" \
48
- --enforce-eager \
49
- --tensor-parallel-size 2 \
50
- --data-parallel-size 2 \
51
- --enable-expert-parallel \
52
- --enable-eplb \
53
- --eplb-config '{"window_size":200,"step_interval":600,"use_async":true}' \
54
- --trust-remote-code \
55
- --max-model-len 2048 \
56
- --port $PORT &
57
- SERVER_PID=$!
58
- wait_for_server $PORT
59
-
60
- TAG=$(echo "$MODEL" | tr '/: \\n' '_____')
61
- OUT="${OUT_DIR}/${TAG}_${BACK}_async_eplb.json"
62
- python3 tests/evals/gsm8k/gsm8k_eval.py --host http://127.0.0.1 --port $PORT --num-questions ${NUM_Q} --save-results ${OUT}
63
- python3 - <<PY
64
-import json; acc=json.load(open('${OUT}'))['accuracy']
65
-print(f"${MODEL} ${BACK}: accuracy {acc:.3f}")
66
-assert acc >= ${THRESHOLD}, f"${MODEL} ${BACK} accuracy {acc}"
67
-PY
68
-
69
- cleanup
70
- SERVER_PID=
71
- sleep 1
72
- PORT=$((PORT+1))
73
-done
.buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh
-1
@@ -50,7 +50,6 @@ for BACK in "${BACKENDS[@]}"; do
50
--data-parallel-size 2 \
51
--enable-expert-parallel \
52
--enable-eplb \
53
- --eplb-config '{"window_size":200,"step_interval":600}' \
53
--trust-remote-code \
54
--max-model-len 2048 \
55
--port $PORT &
.buildkite/scripts/scheduled_integration_test/qwen3_next_mtp_async_eplb.sh
deleted
-74
@@ -1,74 +0,0 @@
1
-#!/usr/bin/env bash
2
-set -euxo pipefail
3
-
4
-# args: [THRESHOLD] [NUM_QUESTIONS] [START_PORT]
5
-THRESHOLD=${1:-0.25}
6
-NUM_Q=${2:-1319}
7
-PORT=${3:-8040}
8
-OUT_DIR=${OUT_DIR:-/tmp/vllm-scheduled}
9
-mkdir -p "${OUT_DIR}"
10
-
11
-wait_for_server() {
12
- local port=$1
13
- timeout 600 bash -c '
14
- until curl -sf "http://127.0.0.1:'"$port"'/health" > /dev/null; do
15
- sleep 1
16
- done'
17
-}
18
-
19
-MODEL="Qwen/Qwen3-Next-80B-A3B-Instruct"
20
-
21
-# Set BACKENDS based on platform
22
-if command -v rocm-smi &> /dev/null || [[ -d /opt/rocm ]] || [[ -n "${ROCM_PATH:-}" ]]; then
23
- # ROCm platform
24
- BACKENDS=("allgather_reducescatter")
25
- # Disable MOE padding for ROCm since it is causing eplb to fail
26
- export VLLM_ROCM_MOE_PADDING=0
27
-else
28
- # Non-ROCm platform (CUDA/other)
29
- BACKENDS=("deepep_high_throughput" "deepep_low_latency")
30
-fi
31
-
32
-cleanup() {
33
- if [[ -n "${SERVER_PID:-}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then
34
- kill "${SERVER_PID}" 2>/dev/null || true
35
- for _ in {1..20}; do
36
- kill -0 "${SERVER_PID}" 2>/dev/null || break
37
- sleep 0.5
38
- done
39
- kill -9 "${SERVER_PID}" 2>/dev/null || true
40
- fi
41
-}
42
-trap cleanup EXIT
43
-
44
-for BACK in "${BACKENDS[@]}"; do
45
- VLLM_DEEP_GEMM_WARMUP=skip \
46
- VLLM_ALL2ALL_BACKEND=$BACK \
47
- vllm serve "$MODEL" \
48
- --enforce-eager \
49
- --tensor-parallel-size 4 \
50
- --enable-expert-parallel \
51
- --enable-eplb \
52
- --eplb-config '{"window_size":200,"step_interval":600,"use_async":true}' \
53
- --speculative-config '{"method":"qwen3_next_mtp","num_speculative_tokens":1}' \
54
- --trust-remote-code \
55
- --max-model-len 2048 \
56
- --gpu-memory-utilization 0.9 \
57
- --port $PORT &
58
- SERVER_PID=$!
59
- wait_for_server $PORT
60
-
61
- TAG=$(echo "$MODEL" | tr '/: \\n' '_____')
62
- OUT="${OUT_DIR}/${TAG}_${BACK}.json"
63
- python3 tests/evals/gsm8k/gsm8k_eval.py --host http://127.0.0.1 --port $PORT --num-questions ${NUM_Q} --save-results ${OUT}
64
- python3 - <<PY
65
-import json; acc=json.load(open('${OUT}'))['accuracy']
66
-print(f"${MODEL} ${BACK}: accuracy {acc:.3f}")
67
-assert acc >= ${THRESHOLD}, f"${MODEL} ${BACK} accuracy {acc}"
68
-PY
69
-
70
- cleanup
71
- SERVER_PID=
72
- sleep 1
73
- PORT=$((PORT+1))
74
-done
.buildkite/test-pipeline.yaml
+1
-19
@@ -1379,22 +1379,4 @@ steps:
1379
num_gpus: 2
1380
working_dir: "/vllm-workspace"
1381
commands:
1382
- - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020 2 1
1383
-
1384
-- label: DeepSeek V2-Lite Async EPLB Accuracy
1385
- timeout_in_minutes: 60
1386
- gpu: h100
1387
- optional: true
1388
- num_gpus: 4
1389
- working_dir: "/vllm-workspace"
1390
- commands:
1391
- - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_async_eplb.sh 0.25 1319 8030
1392
-
1393
-- label: Qwen3-Next-80B-A3B-Instruct MTP Async EPLB Accuracy
1394
- timeout_in_minutes: 60
1395
- gpu: h100
1396
- optional: true
1397
- num_gpus: 4
1398
- working_dir: "/vllm-workspace"
1399
- commands:
1400
- - bash .buildkite/scripts/scheduled_integration_test/qwen3_next_mtp_async_eplb.sh 0.8 1319 8040
1382
+ - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020 2 1
\ No newline at end of file
vllm/distributed/eplb/rebalance_execute.py
+3
@@ -322,6 +322,9 @@ async def transfer_layer(
322
num_local_physical_experts = next(iter(expert_weights[0])).shape[0]
323
assert new_global_expert_indices.shape == (num_moe_layers, num_physical_experts)
324
assert num_physical_experts == ep_size * num_local_physical_experts
325
+ # A buffer to hold the expert weights in one layer during the exchange.
326
+ # NOTE: Currently we assume the same weights across different layers
327
+ # have the same shape.
328
329
is_unchanged, is_received_locally, experts_recv_loc = move_to_buffer(
330
num_local_experts=num_local_physical_experts,