[XPU][Test] Support MultiConnector accuracy testing on XPU (#51160)
Signed-off-by: zhenwei-intel <zhenwei.liu@intel.com>
liuzhenwei committed
Aug 6, 2026 at 13:49 UTC
2a0323b4e0f08b29d11a6f3bfa154b3bd7bb7975
2 files changed
+20
-5
tests/v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh
+19
-4
@@ -18,6 +18,9 @@
18
# Environment variables:
19
# MODEL_NAMES - model to test (default: Qwen/Qwen3-0.6B)
20
# GPU_MEMORY_UTILIZATION - GPU memory fraction (default: 0.6)
21
+# MAX_MODEL_LEN - maximum model context length
22
+# KV_BUFFER_DEVICE - NIXL KV buffer device (default: cuda; use xpu for XPU)
23
+# NUM_CONCURRENT - lm-eval request concurrency (default: 100)
24
# ATTENTION_BACKEND - optional attention backend for vllm serve
25
# VLLM_SERVE_EXTRA_ARGS - comma-separated extra args for vllm serve
26
# SKIP_CROSS_LAYERS - set to 1 to skip the cross-layer layout test
@@ -35,6 +38,8 @@ fi
38
39
GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.6}
40
BLOCK_SIZE=${BLOCK_SIZE:-128}
41
+MAX_MODEL_LEN=${MAX_MODEL_LEN:-8192}
42
+KV_BUFFER_DEVICE=${KV_BUFFER_DEVICE:-cuda}
43
ATTENTION_BACKEND=${ATTENTION_BACKEND:-}
44
VLLM_SERVE_EXTRA_ARGS=${VLLM_SERVE_EXTRA_ARGS:-}
45
@@ -42,6 +47,12 @@ SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
47
GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../.." && pwd -P)}"
48
SMI_BIN=$(which nvidia-smi || which rocm-smi || echo "")
49
50
+if [[ "$KV_BUFFER_DEVICE" == "xpu" ]]; then
51
+ DEVICE_VISIBILITY_ENV=ZE_AFFINITY_MASK
52
+else
53
+ DEVICE_VISIBILITY_ENV=CUDA_VISIBLE_DEVICES
54
+fi
55
+
56
# ── KV transfer configs ─────────────────────────────────────────────────
57
58
# Normal layout: OffloadingConnector prefers cross-layer but NixlConnector
@@ -51,7 +62,8 @@ KV_CONFIG_NORMAL='{
62
"kv_role":"kv_both",
63
"kv_connector_extra_config":{
64
"connectors":[
54
- {"kv_connector":"NixlConnector","kv_role":"kv_both"},
65
+ {"kv_connector":"NixlConnector","kv_role":"kv_both",
66
+ "kv_buffer_device":"'$KV_BUFFER_DEVICE'"},
67
{"kv_connector":"OffloadingConnector","kv_role":"kv_both",
68
"kv_connector_extra_config":{"cpu_bytes_to_use":1000000000}}
69
]
@@ -67,6 +79,7 @@ KV_CONFIG_CROSS_LAYERS='{
79
"kv_connector_extra_config":{
80
"connectors":[
81
{"kv_connector":"NixlConnector","kv_role":"kv_both",
82
+ "kv_buffer_device":"'$KV_BUFFER_DEVICE'",
83
"kv_connector_extra_config":{"enable_cross_layers_blocks":"True"}},
84
{"kv_connector":"OffloadingConnector","kv_role":"kv_both",
85
"kv_connector_extra_config":{"cpu_bytes_to_use":1000000000}}
@@ -99,7 +112,7 @@ get_num_gpus() {
112
elif [[ "$SMI_BIN" == *"rocm"* ]]; then
113
$SMI_BIN -l | grep -c GPU
114
else
102
- echo "1"
115
+ echo "2"
116
fi
117
}
118
@@ -124,7 +137,7 @@ run_tests_for_model() {
137
138
# ── Start prefill instance ──
139
echo "Starting prefill instance on GPU $PREFILL_GPU, port $PREFILL_PORT"
127
- BASE_CMD="CUDA_VISIBLE_DEVICES=$PREFILL_GPU \
140
+ BASE_CMD="$DEVICE_VISIBILITY_ENV=$PREFILL_GPU \
141
VLLM_KV_CACHE_LAYOUT='HND' \
142
UCX_NET_DEVICES=all \
143
VLLM_NIXL_SIDE_CHANNEL_PORT=$PREFILL_SIDE_CHANNEL_PORT \
@@ -132,6 +145,7 @@ run_tests_for_model() {
145
--port $PREFILL_PORT \
146
--enforce-eager \
147
--block-size ${BLOCK_SIZE} \
148
+ --max-model-len ${MAX_MODEL_LEN} \
149
--gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
150
--tensor-parallel-size 1 \
151
--kv-transfer-config '$kv_config'"
@@ -149,7 +163,7 @@ run_tests_for_model() {
163
164
# ── Start decode instance ──
165
echo "Starting decode instance on GPU $DECODE_GPU, port $DECODE_PORT"
152
- BASE_CMD="CUDA_VISIBLE_DEVICES=$DECODE_GPU \
166
+ BASE_CMD="$DEVICE_VISIBILITY_ENV=$DECODE_GPU \
167
VLLM_KV_CACHE_LAYOUT='HND' \
168
UCX_NET_DEVICES=all \
169
VLLM_NIXL_SIDE_CHANNEL_PORT=$DECODE_SIDE_CHANNEL_PORT \
@@ -157,6 +171,7 @@ run_tests_for_model() {
171
--port $DECODE_PORT \
172
--enforce-eager \
173
--block-size ${BLOCK_SIZE} \
174
+ --max-model-len ${MAX_MODEL_LEN} \
175
--gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
176
--tensor-parallel-size 1 \
177
--kv-transfer-config '$kv_config'"
tests/v1/kv_connector/nixl_integration/test_accuracy.py
+1
-1
@@ -6,7 +6,7 @@ import lm_eval
6
import openai
7
8
BASE_URL = "http://localhost:8192/v1"
9
-NUM_CONCURRENT = 100
9
+NUM_CONCURRENT = int(os.getenv("NUM_CONCURRENT", "100"))
10
TASK = "gsm8k"
11
FILTER = "exact_match,strict-match"
12
# TODO(#43186): Widened from 0.03 to absorb chunk_scan/SSU numeric jitter