A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[CI] Nixl integration tests (#27010)
Signed-off-by: NickLucche <nlucches@redhat.com>
Nicolò Lucchesi committed
Oct 17, 2025 at 16:13 UTC
2ba60ec7febaa439dd40040018116f55834b4170
5 files changed
+72
-9
.buildkite/test-pipeline.yaml
+11
@@ -1084,6 +1084,17 @@ steps:
1084
- tests/weight_loading
1085
commands:
1086
- bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-large.txt
1087
+
1088
+- label: NixlConnector P/D accuracy tests (Distributed) # 30min
1089
+ timeout_in_minutes: 30
1090
+ working_dir: "/vllm-workspace/tests"
1091
+ num_gpus: 4
1092
+ source_file_dependencies:
1093
+ - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
1094
+ - tests/v1/kv_connector/nixl_integration/
1095
+ commands:
1096
+ - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt
1097
+ - bash v1/kv_connector/nixl_integration/tp_config_sweep_accuracy_test.sh
1098
1099
1100
##### multi gpus test #####
tests/v1/kv_connector/nixl_integration/run_accuracy_test.sh
+13
-7
@@ -34,15 +34,21 @@ else
34
fi
35
36
# Models to run
37
-MODELS=(
38
- "Qwen/Qwen3-0.6B"
39
-)
37
+MODEL_NAMES=${MODEL_NAMES:-}
38
+if [[ -n "$MODEL_NAMES" ]]; then
39
+ MODELS=("$MODEL_NAMES")
40
+else
41
+ MODELS=(
42
+ "Qwen/Qwen3-0.6B"
43
+ )
44
+fi
45
46
# Number of prefill and decode instances to create
47
NUM_PREFILL_INSTANCES=${NUM_PREFILL_INSTANCES:-1} # Default to 1
48
NUM_DECODE_INSTANCES=${NUM_DECODE_INSTANCES:-1} # Default to 1
49
PREFILLER_TP_SIZE=${PREFILLER_TP_SIZE:-1}
50
DECODER_TP_SIZE=${DECODER_TP_SIZE:-1}
51
+GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.2}
52
53
# Find the git repository root directory
54
GIT_ROOT=$(git rev-parse --show-toplevel)
@@ -130,7 +136,7 @@ run_tests_for_model() {
136
vllm serve $model_name \
137
--port $PORT \
138
--enforce-eager \
133
- --gpu-memory-utilization 0.2 \
139
+ --gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
140
--tensor-parallel-size $PREFILLER_TP_SIZE \
141
--kv-transfer-config '$KV_CONFIG'"
142
@@ -171,7 +177,7 @@ run_tests_for_model() {
177
vllm serve $model_name \
178
--port $PORT \
179
--enforce-eager \
174
- --gpu-memory-utilization 0.2 \
180
+ --gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
181
--tensor-parallel-size $DECODER_TP_SIZE \
182
--kv-transfer-config '$KV_CONFIG'"
183
@@ -200,7 +206,7 @@ run_tests_for_model() {
206
done
207
208
# Build the command for the proxy server with all the hosts and ports
203
- PROXY_CMD="python ${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/toy_proxy_server.py --port 8192"
209
+ PROXY_CMD="python3 ${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/toy_proxy_server.py --port 8192"
210
211
# Add all prefill hosts and ports
212
PROXY_CMD+=" --prefiller-hosts ${PREFILL_HOSTS[@]}"
@@ -219,7 +225,7 @@ run_tests_for_model() {
225
226
# Run lm eval for this model
227
echo "Running tests for $model_name"
222
- TEST_MODEL=$model_name python -m pytest -s -x ${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/test_accuracy.py
228
+ TEST_MODEL=$model_name python3 -m pytest -s -x ${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/test_accuracy.py
229
230
# Clean up before running next model
231
cleanup_instances
tests/v1/kv_connector/nixl_integration/test_accuracy.py
+6
-1
@@ -12,7 +12,12 @@ FILTER = "exact_match,strict-match"
12
RTOL = 0.03
13
14
# Model-specific expected values
15
-EXPECTED_VALUES = {"Qwen/Qwen3-0.6B": 0.41, "deepseek-ai/deepseek-vl2-small": 0.59}
15
+EXPECTED_VALUES = {
16
+ "Qwen/Qwen3-0.6B": 0.41,
17
+ "deepseek-ai/deepseek-vl2-small": 0.59,
18
+ "deepseek-ai/deepseek-vl2-tiny": 0.19,
19
+ "deepseek-ai/DeepSeek-V2-Lite-Chat": 0.65,
20
+}
21
22
SIMPLE_PROMPT = (
23
"The best part about working on vLLM is that I got to meet so many people across "
tests/v1/kv_connector/nixl_integration/toy_proxy_server.py
+2
-1
@@ -76,7 +76,8 @@ def parse_args():
76
parser = argparse.ArgumentParser()
77
78
parser.add_argument("--port", type=int, default=8000)
79
- parser.add_argument("--host", type=str, default="localhost")
79
+ # Always use 127.0.0.1 as localhost binds to IPv6 which is blocked on CI
80
+ parser.add_argument("--host", type=str, default="127.0.0.1")
81
82
# For prefiller instances
83
parser.add_argument(
tests/v1/kv_connector/nixl_integration/tp_config_sweep_accuracy_test.sh
new
+40
@@ -0,0 +1,40 @@
1
+#!/usr/bin/env bash
2
+set -euo pipefail
3
+
4
+# Utility to run integration tests sequentially with varying TP configurations.
5
+SCRIPT="v1/kv_connector/nixl_integration/run_accuracy_test.sh"
6
+
7
+# Define test configurations
8
+configs=(
9
+ "GPU_MEMORY_UTILIZATION=0.6 PREFILLER_TP_SIZE=2 DECODER_TP_SIZE=2"
10
+ "GPU_MEMORY_UTILIZATION=0.6 PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=2"
11
+ "GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=deepseek-ai/deepseek-vl2-tiny" # MLA case
12
+ "GPU_MEMORY_UTILIZATION=0.8 PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=2 MODEL_NAMES=deepseek-ai/deepseek-vl2-tiny"
13
+)
14
+
15
+run_tests() {
16
+ local label=$1
17
+ local extra_env=$2
18
+
19
+ echo "=== Running tests (${label}) ==="
20
+ for cfg in "${configs[@]}"; do
21
+ echo "-> Running with ${cfg} ${extra_env:+and ${extra_env}}"
22
+ # Use 'env' to safely set variables without eval
23
+ if ! env ${extra_env} ${cfg} bash "${SCRIPT}"; then
24
+ echo "❌ Test failed for config: ${cfg} ${extra_env:+(${extra_env})}"
25
+ exit 1
26
+ fi
27
+ done
28
+ echo "✅ All ${label} tests passed!"
29
+}
30
+
31
+# Run tests
32
+run_tests "default backend" ""
33
+
34
+# Check if FLASHINFER is set (non-empty)
35
+if [[ -n "${FLASHINFER:-}" ]]; then
36
+ echo "FLASHINFER is set, rerunning with VLLM_ATTENTION_BACKEND=FLASHINFER"
37
+ run_tests "FLASHINFER backend" "VLLM_ATTENTION_BACKEND=FLASHINFER"
38
+else
39
+ echo "FLASHINFER not set, skipping FLASHINFER runs."
40
+fi