[XPU] Support ZE_AFFINITY_MASK passthrough in xpu_disagg_acc_test (#47105)
Signed-off-by: zhenwei-intel <zhenwei.liu@intel.com>
liuzhenwei committed
Jul 1, 2026 at 01:06 UTC
7a341fa109e569c8d265545adfca756f42451244
3 files changed
+57
-24
.buildkite/intel_jobs/misc_intel.yaml
+25
@@ -103,6 +103,31 @@ steps:
103
pytest -v -s v1/kv_offload &&
104
pytest -v -s v1/kv_connector/unit/test_offloading_connector.py'
105
106
+- label: NixlConnector PD accuracy (2 GPUs)
107
+ timeout_in_minutes: 60
108
+ num_devices: 2
109
+ device: intel_gpu
110
+ agent_tags:
111
+ label: production
112
+ gpu: 2+
113
+ mem: 24+
114
+ no_plugin: true
115
+ working_dir: "."
116
+ env:
117
+ REGISTRY: "public.ecr.aws/q9t5s3a7"
118
+ REPO: "vllm-ci-test-repo"
119
+ VLLM_TEST_DEVICE: "xpu"
120
+ source_file_dependencies:
121
+ - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
122
+ - vllm/v1/worker/kv_connector_model_runner_mixin.py
123
+ - tests/v1/kv_connector/nixl_integration/
124
+ - vllm/platforms/xpu.py
125
+ commands:
126
+ - >-
127
+ bash .buildkite/scripts/hardware_ci/run-intel-test.sh
128
+ 'cd tests &&
129
+ bash v1/kv_connector/nixl_integration/run_xpu_disagg_accuracy_test.sh'
130
+
131
- label: Regression
132
key: regression
133
timeout_in_minutes: 30
.buildkite/intel_jobs/test-intel.yaml
-1
@@ -68,7 +68,6 @@ steps:
68
- >-
69
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
70
'cd tests &&
71
- bash v1/kv_connector/nixl_integration/run_xpu_disagg_accuracy_test.sh &&
71
pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py &&
72
pytest -v -s v1/engine --ignore=v1/engine/test_output_processor.py &&
73
pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py -k "not test_topk_only and not test_topp_only and not test_topk_and_topp" &&
tests/v1/kv_connector/nixl_integration/run_xpu_disagg_accuracy_test.sh
+32
-23
@@ -21,27 +21,28 @@ DECODER_TP_SIZE=${DECODER_TP_SIZE:-1}
21
KV_BUFFER_DEVICE=${KV_BUFFER_DEVICE:-"xpu"}
22
GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.8}
23
24
-generate_affinity_mask() {
25
- local count=$1
26
- local start=${2:-0}
27
- local mask=""
28
- local i
29
-
30
- for ((i=0; i<count; i++)); do
31
- local device=$((start + i))
32
- if [[ -z "${mask}" ]]; then
33
- mask="${device}"
34
- else
35
- mask="${mask},${device}"
36
- fi
24
+# Parse available devices from ZE_AFFINITY_MASK (e.g. "2,3") or default to "0,1,...".
25
+# compute_gpu_id indexes into this list to assign devices per-instance.
26
+if [[ -n "${ZE_AFFINITY_MASK:-}" ]]; then
27
+ IFS=',' read -r -a AVAILABLE_DEVICES <<< "${ZE_AFFINITY_MASK}"
28
+else
29
+ # Default: devices 0 .. (PREFILLER_TP_SIZE + DECODER_TP_SIZE - 1)
30
+ AVAILABLE_DEVICES=()
31
+ for ((i=0; i<PREFILLER_TP_SIZE + DECODER_TP_SIZE; i++)); do
32
+ AVAILABLE_DEVICES+=("$i")
33
done
38
-
39
- echo "${mask}"
34
+fi
35
+
36
+compute_gpu_id() {
37
+ local start=$1
38
+ local tp_size=$2
39
+ local gpu_id="${AVAILABLE_DEVICES[$start]}"
40
+ for (( j=1; j<tp_size; j++ )); do
41
+ gpu_id="${gpu_id},${AVAILABLE_DEVICES[$((start + j))]}"
42
+ done
43
+ echo "${gpu_id}"
44
}
45
42
-PREFILLER_ZE_AFFINITY_MASK=${PREFILLER_ZE_AFFINITY_MASK:-$(generate_affinity_mask "${PREFILLER_TP_SIZE}" 0)}
43
-DECODER_ZE_AFFINITY_MASK=${DECODER_ZE_AFFINITY_MASK:-$(generate_affinity_mask "${DECODER_TP_SIZE}" "${PREFILLER_TP_SIZE}")}
44
-
46
47
# execution env
48
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
@@ -69,8 +70,11 @@ wait_for_server() {
70
}
71
72
launch_baseline() {
73
+ local BASELINE_GPU_ID
74
+ BASELINE_GPU_ID=$(compute_gpu_id 0 1)
75
+
76
BASELINE_BASE_CMD="
73
- ZE_AFFINITY_MASK=0 \
77
+ ZE_AFFINITY_MASK=$BASELINE_GPU_ID \
78
VLLM_WORKER_MULTIPROC_METHOD=spawn \
79
VLLM_ENABLE_V1_MULTIPROCESSING=1 vllm serve $MODEL_NAME \
80
--host ${BASELINE_HOST} \
@@ -89,9 +93,14 @@ launch_baseline() {
93
}
94
95
launch_pd() {
96
+ local PREFILL_GPU_ID
97
+ PREFILL_GPU_ID=$(compute_gpu_id 0 "${PREFILLER_TP_SIZE}")
98
+ local DECODE_GPU_ID
99
+ DECODE_GPU_ID=$(compute_gpu_id "${PREFILLER_TP_SIZE}" "${DECODER_TP_SIZE}")
100
+
101
PREFILL_BASE_CMD="
93
- ZE_AFFINITY_MASK=${PREFILLER_ZE_AFFINITY_MASK} \
94
- VLLM_MULTIPROC_EXECUTE_MODEL_TIMEOUT_S=200 \
102
+ ZE_AFFINITY_MASK=$PREFILL_GPU_ID \
103
+ VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=200 \
104
VLLM_NIXL_SIDE_CHANNEL_HOST=${PREFILL_HOST} \
105
VLLM_NIXL_SIDE_CHANNEL_PORT=${PREFILL_NIXL_SIDE_PORT} \
106
VLLM_WORKER_MULTIPROC_METHOD=spawn \
@@ -109,8 +118,8 @@ launch_pd() {
118
119
120
DECODE_BASE_CMD="
112
- ZE_AFFINITY_MASK=${DECODER_ZE_AFFINITY_MASK} \
113
- VLLM_MULTIPROC_EXECUTE_MODEL_TIMEOUT_S=200 \
121
+ ZE_AFFINITY_MASK=$DECODE_GPU_ID \
122
+ VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=200 \
123
VLLM_WORKER_MULTIPROC_METHOD=spawn \
124
VLLM_ENABLE_V1_MULTIPROCESSING=1 vllm serve $MODEL_NAME \
125
--host ${DECODE_HOST} \