[Bugfix] Return HTTP 500 for non-streaming generate errors (#49195)
Signed-off-by: WEI CHENG CHIU <waynehacking8@gmail.com>
Wei-Cheng (Wayne) Chiu committed
Aug 22, 2026 at 00:07 UTC
b41dc4ec722a70b141696e75ff04c7016eb5ed15
2 files changed
+27
-1
tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py
+25
-1
@@ -9,7 +9,7 @@ from unittest.mock import AsyncMock, MagicMock
9
import pytest
10
11
from vllm.config.multimodal import MultiModalConfig
12
-from vllm.entrypoints.openai.engine.protocol import StreamOptions
12
+from vllm.entrypoints.openai.engine.protocol import GenerationError, StreamOptions
13
from vllm.entrypoints.openai.models.protocol import BaseModelPath
14
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
15
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
@@ -229,6 +229,30 @@ async def test_serve_tokens_threads_session_id_header_to_engine():
229
assert engine.generate.call_args.kwargs["session_id"] == "header-session"
230
231
232
+@pytest.mark.asyncio
233
+async def test_non_stream_error():
234
+ """finish_reason='error' raises for the HTTP exception handler."""
235
+ engine = _mock_engine()
236
+
237
+ async def mock_generate(*args, **kwargs):
238
+ yield _make_request_output(
239
+ "req-1", token_ids=[], finish_reason="error", finished=True
240
+ )
241
+
242
+ engine.generate = MagicMock(side_effect=mock_generate)
243
+ serving = _build_serving_tokens(engine)
244
+
245
+ request = GenerateRequest(
246
+ token_ids=[1, 2, 3],
247
+ sampling_params=SamplingParams(max_tokens=10),
248
+ model=MODEL_NAME,
249
+ stream=False,
250
+ )
251
+
252
+ with pytest.raises(GenerationError):
253
+ await serving.serve_tokens(request)
254
+
255
+
256
@pytest.mark.asyncio
257
async def test_stream_basic():
258
"""Streaming returns SSE chunks with correct token_ids and ends with [DONE]."""
vllm/entrypoints/scale_out/token_in_token_out/serving.py
+2
@@ -288,6 +288,8 @@ class ServingTokens(GenerateBaseServing):
288
choices: list[GenerateResponseChoice] = []
289
num_generated_tokens = 0
290
for output in final_res.outputs:
291
+ self._raise_if_error(output.finish_reason, request_id)
292
+
293
token_ids = output.token_ids
294
out_logprobs = output.logprobs
295