A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 22:30 UTC
[CI][EPLB] Add Async EPLB end-to-end integration test to CI (#40168)
Signed-off-by: Sage Moore <sage@neuralmagic.com>
Sage Moore committed
Apr 20, 2026 at 07:22 UTC
def8f52200151c801867dde9ce27f829bce00105
2 files changed
+64
.buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_dp4_async_eplb.sh
new
+55
@@ -0,0 +1,55 @@
1
+#!/usr/bin/env bash
2
+set -euxo pipefail
3
+
4
+# args: [THRESHOLD] [NUM_QUESTIONS] [START_PORT]
5
+THRESHOLD=${1:-0.8}
6
+NUM_Q=${2:-1319}
7
+PORT=${3:-8050}
8
+OUT_DIR=${OUT_DIR:-/tmp/vllm-scheduled}
9
+mkdir -p "${OUT_DIR}"
10
+
11
+wait_for_server() {
12
+ local port=$1
13
+ timeout 600 bash -c '
14
+ until curl -sf "http://127.0.0.1:'"$port"'/health" > /dev/null; do
15
+ sleep 1
16
+ done'
17
+}
18
+
19
+MODEL="Qwen/Qwen3-30B-A3B-FP8"
20
+BACK="allgather_reducescatter"
21
+
22
+cleanup() {
23
+ if [[ -n "${SERVER_PID:-}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then
24
+ kill "${SERVER_PID}" 2>/dev/null || true
25
+ for _ in {1..20}; do
26
+ kill -0 "${SERVER_PID}" 2>/dev/null || break
27
+ sleep 0.5
28
+ done
29
+ kill -9 "${SERVER_PID}" 2>/dev/null || true
30
+ fi
31
+}
32
+trap cleanup EXIT
33
+
34
+VLLM_DEEP_GEMM_WARMUP=skip \
35
+vllm serve "$MODEL" \
36
+--enforce-eager \
37
+--data-parallel-size 4 \
38
+--enable-expert-parallel \
39
+--enable-eplb \
40
+--all2all-backend "$BACK" \
41
+--eplb-config '{"window_size":20, "step_interval":100, "use_async":true}' \
42
+--trust-remote-code \
43
+--max-model-len 2048 \
44
+--port "$PORT" &
45
+SERVER_PID=$!
46
+wait_for_server "$PORT"
47
+
48
+TAG=$(echo "$MODEL" | tr '/: \\n' '_____')
49
+OUT="${OUT_DIR}/${TAG}_${BACK}.json"
50
+python3 tests/evals/gsm8k/gsm8k_eval.py --host http://127.0.0.1 --port "$PORT" --num-questions "${NUM_Q}" --save-results "${OUT}"
51
+python3 - <<PY
52
+import json; acc=json.load(open('${OUT}'))['accuracy']
53
+print(f"${MODEL} ${BACK}: accuracy {acc:.3f}")
54
+assert acc >= ${THRESHOLD}, f"${MODEL} ${BACK} accuracy {acc}"
55
+PY
.buildkite/test_areas/e2e_integration.yaml
+9
@@ -29,6 +29,15 @@ steps:
29
commands:
30
- bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020 2 1
31
32
+- label: Qwen3-30B-A3B-FP8 DP4 Async EPLB Accuracy
33
+ timeout_in_minutes: 60
34
+ device: h100
35
+ optional: true
36
+ num_devices: 4
37
+ working_dir: "/vllm-workspace"
38
+ commands:
39
+ - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_dp4_async_eplb.sh 0.8 200 8050
40
+
41
- label: DeepSeek V2-Lite Prefetch Offload Accuracy (H100)
42
timeout_in_minutes: 60
43
device: h100