[CI] Deflake the multi-API-server metrics test and Mooncake PD ports (#59507)
Signed-off-by: khluu <khluu000@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Kevin H. Luu committed
Sep 30, 2026 at 17:22 UTC
8bc31bd21075c5ca8a070e6da84e96e2eaf816ad
3 files changed
+34
-14
tests/entrypoints/serve/dev/rlhf/test_weight_operation_metrics.py
+10
-4
@@ -21,8 +21,10 @@ from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo
21
22
DURATION = "vllm:rl_weight_update_operation_duration_seconds"
23
IN_FLIGHT = "vllm:rl_weight_update_operations_in_flight"
24
-# Enough requests that the kernel spreads them over both API servers.
25
-SYNCS = 8
24
+# The API servers accept from one shared listening socket, so the kernel may
25
+# hand one process most connections. Sync until both have recorded, within a cap.
26
+MIN_SYNCS = 8
27
+MAX_SYNCS = 64
28
29
30
def scrape(url: str) -> dict[tuple[str, str], float]:
@@ -75,11 +77,15 @@ def test_weight_sync_metrics_aggregate_across_api_servers(tmp_path):
77
client=HTTPVLLMWeightSyncClient(url),
78
source=ModuleSource(trainer),
79
)
78
- for _ in range(SYNCS):
80
+ syncs = 0
81
+ while syncs < MIN_SYNCS or (
82
+ len(pids_that_recorded(tmp_path)) < 2 and syncs < MAX_SYNCS
83
+ ):
84
engine.send_weights()
85
+ syncs += 1
86
metrics = scrape(url)
87
82
- calls = {"init": 1, "start": SYNCS, "update": SYNCS, "finish": SYNCS}
88
+ calls = {"init": 1, "start": syncs, "update": syncs, "finish": syncs}
89
for operation, count in calls.items():
90
assert metrics[(f"{DURATION}_count", operation)] == count, operation
91
assert metrics[(IN_FLIGHT, operation)] == 0, operation
tests/v1/kv_connector/mooncake_integration/run_accuracy_test.sh
+23
-9
@@ -78,6 +78,20 @@ wait_for_server() {
78
done" && return 0 || return 1
79
}
80
81
+# b200-k8s pods use host networking, so a fixed port can already be taken by
82
+# another job on the node. Bind all the ports at once so they are distinct.
83
+free_ports() {
84
+ python3 - "$1" <<'EOF'
85
+import socket
86
+import sys
87
+
88
+socks = [socket.socket() for _ in range(int(sys.argv[1]))]
89
+for s in socks:
90
+ s.bind(("", 0))
91
+print(" ".join(str(s.getsockname()[1]) for s in socks))
92
+EOF
93
+}
94
+
95
# Function to clean up previous instances
96
cleanup_instances() {
97
echo "Cleaning up any running vLLM instances..."
@@ -107,6 +121,9 @@ run_tests_for_model() {
121
122
# Proxy args: repeated --prefill URL BOOTSTRAP_PORT and --decode URL.
123
PROXY_ARGS=()
124
+ # Per prefiller: server and bootstrap. Per decoder: server. Then the proxy.
125
+ read -r -a PORTS <<< "$(free_ports $((2 * NUM_PREFILL_INSTANCES + NUM_DECODE_INSTANCES + 1)))"
126
+ PROXY_PORT=${PORTS[${#PORTS[@]} - 1]}
127
128
# Start prefill instances
129
for i in $(seq 0 $((NUM_PREFILL_INSTANCES-1))); do
@@ -119,10 +136,8 @@ run_tests_for_model() {
136
GPU_ID="${GPU_ID},${NEXT_GPU}"
137
done
138
122
- # Calculate port number (base port + instance number)
123
- PORT=$((8100 + i))
124
- # Bootstrap server port for this prefiller. Avoid clash across instances.
125
- BOOTSTRAP_PORT=$((8998 + i))
139
+ PORT=${PORTS[$((2 * i))]}
140
+ BOOTSTRAP_PORT=${PORTS[$((2 * i + 1))]}
141
142
echo "Starting prefill instance $i on GPU $GPU_ID, port $PORT, bootstrap $BOOTSTRAP_PORT"
143
@@ -162,8 +177,7 @@ run_tests_for_model() {
177
NEXT_GPU=$(((GPU_ID + j) % $(get_num_gpus)))
178
GPU_ID="${GPU_ID},${NEXT_GPU}"
179
done
165
- # Calculate port number (base port + instance number)
166
- PORT=$((8200 + i))
180
+ PORT=${PORTS[$((2 * NUM_PREFILL_INSTANCES + i))]}
181
182
echo "Starting decode instance $i on GPU $GPU_ID, port $PORT"
183
@@ -205,16 +219,16 @@ run_tests_for_model() {
219
done
220
221
# Start the proxy server
208
- echo "Starting proxy server on port 8192"
222
+ echo "Starting proxy server on port $PROXY_PORT"
223
python3 "${GIT_ROOT}/examples/disaggregated/mooncake_connector/mooncake_connector_proxy.py" \
210
- --port 8192 "${PROXY_ARGS[@]}" &
224
+ --port "$PROXY_PORT" "${PROXY_ARGS[@]}" &
225
226
# Wait for the proxy to start
227
sleep 5
228
229
# Run lm eval for this model
230
echo "Running tests for $model_name"
217
- TEST_MODEL=$model_name python3 -m pytest -s -x \
231
+ TEST_MODEL=$model_name PROXY_PORT=$PROXY_PORT python3 -m pytest -s -x \
232
"${GIT_ROOT}/tests/v1/kv_connector/mooncake_integration/test_accuracy.py"
233
234
# Clean up before running next model
tests/v1/kv_connector/mooncake_integration/test_accuracy.py
+1
-1
@@ -5,7 +5,7 @@ import os
5
import lm_eval
6
import openai
7
8
-BASE_URL = "http://localhost:8192/v1"
8
+BASE_URL = f"http://localhost:{os.environ.get('PROXY_PORT', 8192)}/v1"
9
NUM_CONCURRENT = 100
10
TASK = "gsm8k"
11
FILTER = "exact_match,strict-match"