A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 22:45 UTC
[ROCm][CI] Fix realtime test timeouts caused by aiter JIT compilation delays (#35052)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
Feb 22, 2026 at 04:07 UTC
dd8c3a7fb2b2448d04bb00934f2cacf43bb14c3b
1 file changed
+63
-2
tests/entrypoints/openai/test_realtime_validation.py
+63
-2
@@ -4,6 +4,7 @@
4
import asyncio
5
import base64
6
import json
7
+import warnings
8
9
import librosa
10
import numpy as np
@@ -85,7 +86,41 @@ async def test_multi_chunk_streaming(
86
87
await send_event(ws, {"type": "session.update", "model": model_name})
88
88
- # Send commit to start transcription
89
+ # Wait for the server to acknowledge the session update.
90
+ try:
91
+ while True:
92
+ event = await receive_event(ws, timeout=5.0)
93
+ if event["type"] == "session.updated":
94
+ break
95
+ except TimeoutError:
96
+ warnings.warn(
97
+ f"session.updated not received within {5.0}s after "
98
+ "session.update. The server may not implement this event.",
99
+ stacklevel=2,
100
+ )
101
+
102
+ # (ROCm) Warm-up: send a non-final commit (required to start
103
+ # transcription) with a small audio chunk to trigger aiter
104
+ # compilation on first use.
105
+ await send_event(ws, {"type": "input_audio_buffer.commit"})
106
+ await send_event(
107
+ ws,
108
+ {
109
+ "type": "input_audio_buffer.append",
110
+ "audio": mary_had_lamb_audio_chunks[0],
111
+ },
112
+ )
113
+ await send_event(ws, {"type": "input_audio_buffer.commit", "final": True})
114
+
115
+ # (ROCm) Drain all warm-up responses with generous timeout for
116
+ # JIT compilation
117
+ warmup_done = False
118
+ while not warmup_done:
119
+ event = await receive_event(ws, timeout=360.0)
120
+ if event["type"] in ("transcription.done", "error"):
121
+ warmup_done = True
122
+
123
+ # Now send the real test audio
124
await send_event(ws, {"type": "input_audio_buffer.commit"})
125
126
# Send multiple audio chunks
@@ -153,6 +188,18 @@ async def test_empty_commit_does_not_crash_engine(
188
189
await send_event(ws, {"type": "session.update", "model": model_name})
190
191
+ try:
192
+ while True:
193
+ event = await receive_event(ws, timeout=5.0)
194
+ if event["type"] == "session.updated":
195
+ break
196
+ except TimeoutError:
197
+ warnings.warn(
198
+ f"session.updated not received within {5.0}s after "
199
+ "session.update. The server may not implement this event.",
200
+ stacklevel=2,
201
+ )
202
+
203
# Start generation without sending any audio
204
await send_event(ws, {"type": "input_audio_buffer.commit"})
205
@@ -161,7 +208,8 @@ async def test_empty_commit_does_not_crash_engine(
208
209
# We should get *some* response (error or empty transcription),
210
# but the engine must NOT crash.
164
- event = await receive_event(ws, timeout=30.0)
211
+ # (ROCm) Use generous timeout for first request (aiter JIT compilation)
212
+ event = await receive_event(ws, timeout=360.0)
213
assert event["type"] in (
214
"error",
215
"transcription.done",
@@ -176,6 +224,19 @@ async def test_empty_commit_does_not_crash_engine(
224
225
await send_event(ws, {"type": "session.update", "model": model_name})
226
227
+ try:
228
+ while True:
229
+ event = await receive_event(ws, timeout=5.0)
230
+ if event["type"] == "session.updated":
231
+ break
232
+ except TimeoutError:
233
+ warnings.warn(
234
+ f"session.updated not received within {5.0}s after "
235
+ "session.update. The server may not implement this event.",
236
+ stacklevel=2,
237
+ )
238
+
239
+ # Start transcription
240
await send_event(ws, {"type": "input_audio_buffer.commit"})
241
242
for chunk in mary_had_lamb_audio_chunks: