| 1 | #!/usr/bin/env bash |
| 2 | set -euxo pipefail |
| 3 | |
| 4 | # args: [THRESHOLD] [NUM_QUESTIONS] [START_PORT] |
| 5 | THRESHOLD=${1:-0.25} |
| 6 | NUM_Q=${2:-1319} |
| 7 | PORT=${3:-8040} |
| 8 | OUT_DIR=${OUT_DIR:-/tmp/vllm-scheduled} |
| 9 | mkdir -p "${OUT_DIR}" |
| 10 | |
| 11 | wait_for_server() { |
| 12 | local port=$1 |
| 13 | timeout "${VLLM_ENGINE_READY_TIMEOUT_S:-600}" bash -c ' |
| 14 | until curl -sf "http://127.0.0.1:'"$port"'/health" > /dev/null; do |
| 15 | sleep 1 |
| 16 | done' |
| 17 | } |
| 18 | |
| 19 | MODEL="Qwen/Qwen3-Next-80B-A3B-Instruct" |
| 20 | |
| 21 | # Set BACKENDS and platform-specific args based on platform |
| 22 | if command -v rocm-smi &> /dev/null || [[ -d /opt/rocm ]] || [[ -n "${ROCM_PATH:-}" ]]; then |
| 23 | # ROCm platform |
| 24 | BACKENDS=("allgather_reducescatter") |
| 25 | # Disable MOE padding for ROCm since it is causing eplb to fail |
| 26 | export VLLM_ROCM_MOE_PADDING=0 |
| 27 | PLATFORM_ARGS=("--no-async-scheduling" "--attention-backend=TRITON_ATTN") |
| 28 | echo "Disabled async scheduling for ROCm platform due to issues with spec decode." |
| 29 | else |
| 30 | # Non-ROCm platform (CUDA/other) |
| 31 | BACKENDS=("deepep_high_throughput" "deepep_low_latency") |
| 32 | PLATFORM_ARGS=() |
| 33 | fi |
| 34 | |
| 35 | cleanup() { |
| 36 | if [[ -n "${SERVER_PID:-}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then |
| 37 | kill "${SERVER_PID}" 2>/dev/null || true |
| 38 | for _ in {1..20}; do |
| 39 | kill -0 "${SERVER_PID}" 2>/dev/null || break |
| 40 | sleep 0.5 |
| 41 | done |
| 42 | kill -9 "${SERVER_PID}" 2>/dev/null || true |
| 43 | fi |
| 44 | } |
| 45 | trap cleanup EXIT |
| 46 | |
| 47 | for BACK in "${BACKENDS[@]}"; do |
| 48 | VLLM_DEEP_GEMM_WARMUP=skip \ |
| 49 | vllm serve "$MODEL" \ |
| 50 | --enforce-eager \ |
| 51 | --tensor-parallel-size 4 \ |
| 52 | --enable-expert-parallel \ |
| 53 | --enable-eplb \ |
| 54 | --all2all-backend "$BACK" \ |
| 55 | --eplb-config '{"window_size":200,"step_interval":600,"use_async":true}' \ |
| 56 | --speculative-config '{"method":"qwen3_next_mtp","num_speculative_tokens":1}' \ |
| 57 | --trust-remote-code \ |
| 58 | --max-model-len 2048 \ |
| 59 | --gpu-memory-utilization 0.9 \ |
| 60 | "${PLATFORM_ARGS[@]}" \ |
| 61 | --port "$PORT" & |
| 62 | SERVER_PID=$! |
| 63 | wait_for_server "$PORT" |
| 64 | |
| 65 | TAG=$(echo "$MODEL" | tr '/: \\n' '_____') |
| 66 | OUT="${OUT_DIR}/${TAG}_${BACK}.json" |
| 67 | python3 tests/evals/gsm8k/gsm8k_eval.py --host http://127.0.0.1 --port "$PORT" --num-questions "${NUM_Q}" --save-results "${OUT}" |
| 68 | python3 - <<PY |
| 69 | import json; acc=json.load(open('${OUT}'))['accuracy'] |
| 70 | print(f"${MODEL} ${BACK}: accuracy {acc:.3f}") |
| 71 | assert acc >= ${THRESHOLD}, f"${MODEL} ${BACK} accuracy {acc}" |
| 72 | PY |
| 73 | |
| 74 | cleanup |
| 75 | SERVER_PID= |
| 76 | sleep 1 |
| 77 | PORT=$((PORT+1)) |
| 78 | done |