A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 00:45 UTC
[CI] Fix realtime WebSocket timeout deadlock and unhandled model validation errors (#37483)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
Mar 25, 2026 at 05:24 UTC
9ac2fcafbb193692fdebe03b3da58625e26382f0
2 files changed
+83
-3
tests/entrypoints/openai/realtime/test_realtime_validation.py
+74
-2
@@ -15,6 +15,13 @@ from tests.entrypoints.openai.conftest import add_attention_backend
15
from tests.utils import ROCM_ENV_OVERRIDES, ROCM_EXTRA_ARGS, RemoteOpenAIServer
16
from vllm.assets.audio import AudioAsset
17
18
+# Increase engine iteration timeout for ROCm where first-use JIT compilation
19
+# can exceed the default 60s, causing a silent deadlock in feed_tokens.
20
+REALTIME_ENV_OVERRIDES = {
21
+ **ROCM_ENV_OVERRIDES,
22
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": "600",
23
+}
24
+
25
MISTRAL_FORMAT_ARGS = [
26
"--tokenizer_mode",
27
"mistral",
@@ -77,7 +84,7 @@ async def test_multi_chunk_streaming(
84
add_attention_backend(server_args, rocm_aiter_fa_attention)
85
86
with RemoteOpenAIServer(
80
- model_name, server_args, env_dict=ROCM_ENV_OVERRIDES
87
+ model_name, server_args, env_dict=REALTIME_ENV_OVERRIDES
88
) as remote_server:
89
ws_url = _get_websocket_url(remote_server)
90
async with websockets.connect(ws_url) as ws:
@@ -180,7 +187,7 @@ async def test_empty_commit_does_not_crash_engine(
187
add_attention_backend(server_args, rocm_aiter_fa_attention)
188
189
with RemoteOpenAIServer(
183
- model_name, server_args, env_dict=ROCM_ENV_OVERRIDES
190
+ model_name, server_args, env_dict=REALTIME_ENV_OVERRIDES
191
) as remote_server:
192
ws_url = _get_websocket_url(remote_server)
193
@@ -257,3 +264,68 @@ async def test_empty_commit_does_not_crash_engine(
264
elif event["type"] == "error":
265
pytest.fail(f"Engine error after empty commit: {event}")
266
assert done_received
267
+
268
+
269
+@pytest.mark.asyncio
270
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
271
+async def test_session_update_invalid_model_returns_error(
272
+ model_name, rocm_aiter_fa_attention
273
+):
274
+ """Test that session.update with an invalid model returns an error."""
275
+ server_args = ["--enforce-eager", "--max-model-len", "2048"]
276
+
277
+ if model_name.startswith("mistralai"):
278
+ server_args += MISTRAL_FORMAT_ARGS
279
+
280
+ add_attention_backend(server_args, rocm_aiter_fa_attention)
281
+
282
+ with RemoteOpenAIServer(
283
+ model_name, server_args, env_dict=REALTIME_ENV_OVERRIDES
284
+ ) as remote_server:
285
+ ws_url = _get_websocket_url(remote_server)
286
+ async with websockets.connect(ws_url) as ws:
287
+ event = await receive_event(ws, timeout=30.0)
288
+ assert event["type"] == "session.created"
289
+
290
+ # Send session.update with a model that doesn't exist
291
+ await send_event(
292
+ ws,
293
+ {"type": "session.update", "model": "nonexistent-model"},
294
+ )
295
+
296
+ event = await receive_event(ws, timeout=10.0)
297
+ assert event["type"] == "error"
298
+ assert "nonexistent-model" in event["error"]
299
+
300
+
301
+@pytest.mark.asyncio
302
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
303
+async def test_commit_without_session_update_returns_error(
304
+ model_name, rocm_aiter_fa_attention
305
+):
306
+ """Test that committing before validating the model returns an error
307
+ and does not fall through to processing."""
308
+ server_args = ["--enforce-eager", "--max-model-len", "2048"]
309
+
310
+ if model_name.startswith("mistralai"):
311
+ server_args += MISTRAL_FORMAT_ARGS
312
+
313
+ add_attention_backend(server_args, rocm_aiter_fa_attention)
314
+
315
+ with RemoteOpenAIServer(
316
+ model_name, server_args, env_dict=REALTIME_ENV_OVERRIDES
317
+ ) as remote_server:
318
+ ws_url = _get_websocket_url(remote_server)
319
+ async with websockets.connect(ws_url) as ws:
320
+ event = await receive_event(ws, timeout=30.0)
321
+ assert event["type"] == "session.created"
322
+
323
+ # Send commit without sending session.update first
324
+ await send_event(
325
+ ws,
326
+ {"type": "input_audio_buffer.commit", "final": True},
327
+ )
328
+
329
+ event = await receive_event(ws, timeout=10.0)
330
+ assert event["type"] == "error"
331
+ assert "model_not_validated" in event.get("code", "")
vllm/entrypoints/openai/realtime/connection.py
+9
-1
@@ -102,7 +102,14 @@ class RealtimeConnection:
102
event_type = event.get("type")
103
if event_type == "session.update":
104
logger.debug("Session updated: %s", event)
105
- self._check_model(event["model"])
105
+ model = event.get("model")
106
+ if model is None:
107
+ await self.send_error("Missing required field: model", "invalid_event")
108
+ return
109
+ err = self._check_model(model)
110
+ if err is not None:
111
+ await self.send_error(err.error.message, "model_not_found")
112
+ return
113
self._is_model_validated = True
114
elif event_type == "input_audio_buffer.append":
115
append_event = InputAudioBufferAppend(**event)
@@ -140,6 +147,7 @@ class RealtimeConnection:
147
err_msg,
148
"model_not_validated",
149
)
150
+ return
151
152
commit_event = InputAudioBufferCommit(**event)
153
# final signals that the audio is finished