[CI] Fix MultiConnector accuracy test lifecycle (#53023)
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: Itay Etelis <92247226+Etelis@users.noreply.github.com> Co-authored-by: OpenAI Codex <codex@openai.com>
Andreas Karatzas committed
Aug 21, 2026 at 15:19 UTC
3e47a9a8240d2b91498d8c97a925b58e6c29ff9f
1 file changed
+98
-15
tests/v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh
+98
-15
@@ -90,22 +90,102 @@ KV_CONFIG_CROSS_LAYERS=$(echo "$KV_CONFIG_CROSS_LAYERS" | tr -d '[:space:]')
90
91
# ── Helpers ──────────────────────────────────────────────────────────────
92
93
-trap 'kill $(jobs -pr) 2>/dev/null' SIGINT SIGTERM EXIT
93
+PROCESS_GROUPS=()
94
+LAST_PROCESS_PID=
95
+
96
+start_background_process() {
97
+ local command=$1
98
+ setsid bash -c "$command" &
99
+ LAST_PROCESS_PID=$!
100
+ PROCESS_GROUPS+=("$LAST_PROCESS_PID")
101
+}
102
+
103
+owned_process_is_running() {
104
+ local process_pid=$1
105
+ kill -0 "$process_pid" 2>/dev/null || \
106
+ kill -0 -- "-$process_pid" 2>/dev/null
107
+}
108
109
wait_for_server() {
110
local port=$1
97
- timeout 1200 bash -c "
98
- until curl -s localhost:${port}/v1/completions > /dev/null; do
99
- sleep 1
100
- done" && return 0 || return 1
111
+ local process_pid=$2
112
+ local process_name=$3
113
+ local endpoint=${4:-/health}
114
+ local deadline=$((SECONDS + 1200))
115
+
116
+ while ((SECONDS < deadline)); do
117
+ if ! kill -0 "$process_pid" 2>/dev/null; then
118
+ local status=0
119
+ wait "$process_pid" || status=$?
120
+ echo "$process_name exited with status $status before becoming ready"
121
+ return 1
122
+ fi
123
+ if curl -fs "http://localhost:${port}${endpoint}" >/dev/null 2>&1; then
124
+ return 0
125
+ fi
126
+ sleep 1
127
+ done
128
+
129
+ echo "$process_name did not become ready on port $port"
130
+ return 1
131
+}
132
+
133
+wait_for_gpu_memory_release() {
134
+ if [[ "$SMI_BIN" == *"rocm"* ]]; then
135
+ PYTHONPATH="${GIT_ROOT}" python3 -c \
136
+ "from tests.utils import wait_for_memory_to_settle; wait_for_memory_to_settle()"
137
+ fi
138
}
139
140
cleanup_instances() {
104
- echo "Cleaning up any running vLLM instances..."
105
- pkill -f "vllm serve" || true
106
- sleep 2
141
+ if ((${#PROCESS_GROUPS[@]} == 0)); then
142
+ return
143
+ fi
144
+
145
+ echo "Cleaning up any running vLLM and proxy instances..."
146
+ local process_groups=("${PROCESS_GROUPS[@]}")
147
+ local process_group
148
+ for process_group in "${process_groups[@]}"; do
149
+ kill -TERM "$process_group" 2>/dev/null || true
150
+ kill -TERM -- "-$process_group" 2>/dev/null || true
151
+ done
152
+
153
+ local deadline=$((SECONDS + 30))
154
+ local processes_running=1
155
+ while ((processes_running && SECONDS < deadline)); do
156
+ processes_running=0
157
+ for process_group in "${process_groups[@]}"; do
158
+ if owned_process_is_running "$process_group"; then
159
+ processes_running=1
160
+ break
161
+ fi
162
+ done
163
+ if ((processes_running)); then
164
+ sleep 1
165
+ fi
166
+ done
167
+
168
+ for process_group in "${process_groups[@]}"; do
169
+ if owned_process_is_running "$process_group"; then
170
+ echo "Force-stopping process group $process_group after cleanup timeout"
171
+ kill -KILL "$process_group" 2>/dev/null || true
172
+ kill -KILL -- "-$process_group" 2>/dev/null || true
173
+ fi
174
+ wait "$process_group" 2>/dev/null || true
175
+ done
176
+
177
+ PROCESS_GROUPS=()
178
}
179
180
+cleanup_on_exit() {
181
+ local rc=$?
182
+ cleanup_instances || true
183
+ exit "$rc"
184
+}
185
+
186
+trap cleanup_on_exit EXIT
187
+trap 'exit 130' SIGINT SIGTERM
188
+
189
get_num_gpus() {
190
if [[ "$SMI_BIN" == *"nvidia"* ]]; then
191
$SMI_BIN --query-gpu=name --format=csv,noheader | wc -l
@@ -159,7 +239,8 @@ run_tests_for_model() {
239
if [[ -n "$ATTENTION_BACKEND" ]]; then
240
BASE_CMD="${BASE_CMD} --attention-backend $ATTENTION_BACKEND"
241
fi
162
- eval "$BASE_CMD &"
242
+ start_background_process "$BASE_CMD"
243
+ local prefill_pid=$LAST_PROCESS_PID
244
245
# ── Start decode instance ──
246
echo "Starting decode instance on GPU $DECODE_GPU, port $DECODE_PORT"
@@ -185,13 +266,14 @@ run_tests_for_model() {
266
if [[ -n "$ATTENTION_BACKEND" ]]; then
267
BASE_CMD="${BASE_CMD} --attention-backend $ATTENTION_BACKEND"
268
fi
188
- eval "$BASE_CMD &"
269
+ start_background_process "$BASE_CMD"
270
+ local decode_pid=$LAST_PROCESS_PID
271
272
# ── Wait for servers ──
273
echo "Waiting for prefill instance on port $PREFILL_PORT to start..."
192
- wait_for_server "$PREFILL_PORT"
274
+ wait_for_server "$PREFILL_PORT" "$prefill_pid" "Prefill server"
275
echo "Waiting for decode instance on port $DECODE_PORT to start..."
194
- wait_for_server "$DECODE_PORT"
276
+ wait_for_server "$DECODE_PORT" "$decode_pid" "Decode server"
277
278
# ── Start proxy ──
279
PROXY_CMD="python3 ${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/toy_proxy_server.py --port 8192"
@@ -201,8 +283,9 @@ run_tests_for_model() {
283
PROXY_CMD+=" --decoder-ports $DECODE_PORT"
284
285
echo "Starting proxy server with command: $PROXY_CMD"
204
- $PROXY_CMD &
205
- sleep 5
286
+ start_background_process "$PROXY_CMD"
287
+ local proxy_pid=$LAST_PROCESS_PID
288
+ wait_for_server 8192 "$proxy_pid" "Proxy server" /healthcheck
289
290
# ── Run accuracy test ──
291
echo "Running accuracy tests for $model_name ($label)"
@@ -211,7 +294,7 @@ run_tests_for_model() {
294
295
# ── Cleanup ──
296
cleanup_instances
214
- sleep 3
297
+ wait_for_gpu_memory_release
298
}
299
300
# ── Main ─────────────────────────────────────────────────────────────────