A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Frontend] Update OpenAI error response to upstream format (#22099)
Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>
Moritz Sanft committed
Aug 7, 2025 at 08:06 UTC
370661856bcfc4cdc9a88580cb70d66b7ac9fc7c
10 files changed
+73
-67
tests/entrypoints/openai/test_classification.py
+2
-3
@@ -121,8 +121,7 @@ def test_invalid_truncate_prompt_tokens_error(server: RemoteOpenAIServer,
121
122
error = classification_response.json()
123
assert classification_response.status_code == 400
124
- assert error["object"] == "error"
125
- assert "truncate_prompt_tokens" in error["message"]
124
+ assert "truncate_prompt_tokens" in error["error"]["message"]
125
126
127
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@@ -137,7 +136,7 @@ def test_empty_input_error(server: RemoteOpenAIServer, model_name: str):
136
137
error = classification_response.json()
138
assert classification_response.status_code == 400
140
- assert error["object"] == "error"
139
+ assert "error" in error
140
141
142
@pytest.mark.parametrize("model_name", [MODEL_NAME])
tests/entrypoints/openai/test_lora_resolvers.py
+4
-4
@@ -160,8 +160,8 @@ async def test_serving_completion_resolver_not_found(mock_serving_setup,
160
mock_engine.generate.assert_not_called()
161
162
assert isinstance(response, ErrorResponse)
163
- assert response.code == HTTPStatus.NOT_FOUND.value
164
- assert non_existent_model in response.message
163
+ assert response.error.code == HTTPStatus.NOT_FOUND.value
164
+ assert non_existent_model in response.error.message
165
166
167
@pytest.mark.asyncio
@@ -190,8 +190,8 @@ async def test_serving_completion_resolver_add_lora_fails(
190
191
# Assert the correct error response
192
assert isinstance(response, ErrorResponse)
193
- assert response.code == HTTPStatus.BAD_REQUEST.value
194
- assert invalid_model in response.message
193
+ assert response.error.code == HTTPStatus.BAD_REQUEST.value
194
+ assert invalid_model in response.error.message
195
196
197
@pytest.mark.asyncio
tests/entrypoints/openai/test_serving_models.py
+8
-8
@@ -66,8 +66,8 @@ async def test_load_lora_adapter_missing_fields():
66
request = LoadLoRAAdapterRequest(lora_name="", lora_path="")
67
response = await serving_models.load_lora_adapter(request)
68
assert isinstance(response, ErrorResponse)
69
- assert response.type == "InvalidUserInput"
70
- assert response.code == HTTPStatus.BAD_REQUEST
69
+ assert response.error.type == "InvalidUserInput"
70
+ assert response.error.code == HTTPStatus.BAD_REQUEST
71
72
73
@pytest.mark.asyncio
@@ -84,8 +84,8 @@ async def test_load_lora_adapter_duplicate():
84
lora_path="/path/to/adapter1")
85
response = await serving_models.load_lora_adapter(request)
86
assert isinstance(response, ErrorResponse)
87
- assert response.type == "InvalidUserInput"
88
- assert response.code == HTTPStatus.BAD_REQUEST
87
+ assert response.error.type == "InvalidUserInput"
88
+ assert response.error.code == HTTPStatus.BAD_REQUEST
89
assert len(serving_models.lora_requests) == 1
90
91
@@ -110,8 +110,8 @@ async def test_unload_lora_adapter_missing_fields():
110
request = UnloadLoRAAdapterRequest(lora_name="", lora_int_id=None)
111
response = await serving_models.unload_lora_adapter(request)
112
assert isinstance(response, ErrorResponse)
113
- assert response.type == "InvalidUserInput"
114
- assert response.code == HTTPStatus.BAD_REQUEST
113
+ assert response.error.type == "InvalidUserInput"
114
+ assert response.error.code == HTTPStatus.BAD_REQUEST
115
116
117
@pytest.mark.asyncio
@@ -120,5 +120,5 @@ async def test_unload_lora_adapter_not_found():
120
request = UnloadLoRAAdapterRequest(lora_name="nonexistent_adapter")
121
response = await serving_models.unload_lora_adapter(request)
122
assert isinstance(response, ErrorResponse)
123
- assert response.type == "NotFoundError"
124
- assert response.code == HTTPStatus.NOT_FOUND
123
+ assert response.error.type == "NotFoundError"
124
+ assert response.error.code == HTTPStatus.NOT_FOUND
tests/entrypoints/openai/test_transcription_validation.py
+11
-6
@@ -116,8 +116,10 @@ async def test_non_asr_model(winning_call):
116
file=winning_call,
117
language="en",
118
temperature=0.0)
119
- assert res.code == 400 and not res.text
120
- assert res.message == "The model does not support Transcriptions API"
119
+ err = res.error
120
+ assert err["code"] == 400 and not res.text
121
+ assert err[
122
+ "message"] == "The model does not support Transcriptions API"
123
124
125
@pytest.mark.asyncio
@@ -133,12 +135,15 @@ async def test_completion_endpoints():
135
"role": "system",
136
"content": "You are a helpful assistant."
137
}])
136
- assert res.code == 400
137
- assert res.message == "The model does not support Chat Completions API"
138
+ err = res.error
139
+ assert err["code"] == 400
140
+ assert err[
141
+ "message"] == "The model does not support Chat Completions API"
142
143
res = await client.completions.create(model=model_name, prompt="Hello")
140
- assert res.code == 400
141
- assert res.message == "The model does not support Completions API"
144
+ err = res.error
145
+ assert err["code"] == 400
146
+ assert err["message"] == "The model does not support Completions API"
147
148
149
@pytest.mark.asyncio
tests/entrypoints/openai/test_translation_validation.py
+3
-2
@@ -73,8 +73,9 @@ async def test_non_asr_model(foscolo):
73
res = await client.audio.translations.create(model=model_name,
74
file=foscolo,
75
temperature=0.0)
76
- assert res.code == 400 and not res.text
77
- assert res.message == "The model does not support Translations API"
76
+ err = res.error
77
+ assert err["code"] == 400 and not res.text
78
+ assert err["message"] == "The model does not support Translations API"
79
80
81
@pytest.mark.asyncio
vllm/entrypoints/openai/api_server.py
+26
-24
@@ -62,7 +62,8 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
62
DetokenizeRequest,
63
DetokenizeResponse,
64
EmbeddingRequest,
65
- EmbeddingResponse, ErrorResponse,
65
+ EmbeddingResponse, ErrorInfo,
66
+ ErrorResponse,
67
LoadLoRAAdapterRequest,
68
PoolingRequest, PoolingResponse,
69
RerankRequest, RerankResponse,
@@ -506,7 +507,7 @@ async def tokenize(request: TokenizeRequest, raw_request: Request):
507
508
if isinstance(generator, ErrorResponse):
509
return JSONResponse(content=generator.model_dump(),
509
- status_code=generator.code)
510
+ status_code=generator.error.code)
511
elif isinstance(generator, TokenizeResponse):
512
return JSONResponse(content=generator.model_dump())
513
@@ -540,7 +541,7 @@ async def detokenize(request: DetokenizeRequest, raw_request: Request):
541
542
if isinstance(generator, ErrorResponse):
543
return JSONResponse(content=generator.model_dump(),
543
- status_code=generator.code)
544
+ status_code=generator.error.code)
545
elif isinstance(generator, DetokenizeResponse):
546
return JSONResponse(content=generator.model_dump())
547
@@ -556,7 +557,7 @@ def maybe_register_tokenizer_info_endpoint(args):
557
"""Get comprehensive tokenizer information."""
558
result = await tokenization(raw_request).get_tokenizer_info()
559
return JSONResponse(content=result.model_dump(),
559
- status_code=result.code if isinstance(
560
+ status_code=result.error.code if isinstance(
561
result, ErrorResponse) else 200)
562
563
@@ -603,7 +604,7 @@ async def create_responses(request: ResponsesRequest, raw_request: Request):
604
605
if isinstance(generator, ErrorResponse):
606
return JSONResponse(content=generator.model_dump(),
606
- status_code=generator.code)
607
+ status_code=generator.error.code)
608
elif isinstance(generator, ResponsesResponse):
609
return JSONResponse(content=generator.model_dump())
610
return StreamingResponse(content=generator, media_type="text/event-stream")
@@ -620,7 +621,7 @@ async def retrieve_responses(response_id: str, raw_request: Request):
621
622
if isinstance(response, ErrorResponse):
623
return JSONResponse(content=response.model_dump(),
623
- status_code=response.code)
624
+ status_code=response.error.code)
625
return JSONResponse(content=response.model_dump())
626
627
@@ -635,7 +636,7 @@ async def cancel_responses(response_id: str, raw_request: Request):
636
637
if isinstance(response, ErrorResponse):
638
return JSONResponse(content=response.model_dump(),
638
- status_code=response.code)
639
+ status_code=response.error.code)
640
return JSONResponse(content=response.model_dump())
641
642
@@ -670,7 +671,7 @@ async def create_chat_completion(request: ChatCompletionRequest,
671
672
if isinstance(generator, ErrorResponse):
673
return JSONResponse(content=generator.model_dump(),
673
- status_code=generator.code)
674
+ status_code=generator.error.code)
675
676
elif isinstance(generator, ChatCompletionResponse):
677
return JSONResponse(content=generator.model_dump())
@@ -715,7 +716,7 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
716
717
if isinstance(generator, ErrorResponse):
718
return JSONResponse(content=generator.model_dump(),
718
- status_code=generator.code)
719
+ status_code=generator.error.code)
720
elif isinstance(generator, CompletionResponse):
721
return JSONResponse(content=generator.model_dump())
722
@@ -744,7 +745,7 @@ async def create_embedding(request: EmbeddingRequest, raw_request: Request):
745
746
if isinstance(generator, ErrorResponse):
747
return JSONResponse(content=generator.model_dump(),
747
- status_code=generator.code)
748
+ status_code=generator.error.code)
749
elif isinstance(generator, EmbeddingResponse):
750
return JSONResponse(content=generator.model_dump())
751
@@ -772,7 +773,7 @@ async def create_pooling(request: PoolingRequest, raw_request: Request):
773
generator = await handler.create_pooling(request, raw_request)
774
if isinstance(generator, ErrorResponse):
775
return JSONResponse(content=generator.model_dump(),
775
- status_code=generator.code)
776
+ status_code=generator.error.code)
777
elif isinstance(generator, PoolingResponse):
778
return JSONResponse(content=generator.model_dump())
779
@@ -792,7 +793,7 @@ async def create_classify(request: ClassificationRequest,
793
generator = await handler.create_classify(request, raw_request)
794
if isinstance(generator, ErrorResponse):
795
return JSONResponse(content=generator.model_dump(),
795
- status_code=generator.code)
796
+ status_code=generator.error.code)
797
798
elif isinstance(generator, ClassificationResponse):
799
return JSONResponse(content=generator.model_dump())
@@ -821,7 +822,7 @@ async def create_score(request: ScoreRequest, raw_request: Request):
822
generator = await handler.create_score(request, raw_request)
823
if isinstance(generator, ErrorResponse):
824
return JSONResponse(content=generator.model_dump(),
824
- status_code=generator.code)
825
+ status_code=generator.error.code)
826
elif isinstance(generator, ScoreResponse):
827
return JSONResponse(content=generator.model_dump())
828
@@ -881,7 +882,7 @@ async def create_transcriptions(raw_request: Request,
882
883
if isinstance(generator, ErrorResponse):
884
return JSONResponse(content=generator.model_dump(),
884
- status_code=generator.code)
885
+ status_code=generator.error.code)
886
887
elif isinstance(generator, TranscriptionResponse):
888
return JSONResponse(content=generator.model_dump())
@@ -922,7 +923,7 @@ async def create_translations(request: Annotated[TranslationRequest,
923
924
if isinstance(generator, ErrorResponse):
925
return JSONResponse(content=generator.model_dump(),
925
- status_code=generator.code)
926
+ status_code=generator.error.code)
927
928
elif isinstance(generator, TranslationResponse):
929
return JSONResponse(content=generator.model_dump())
@@ -950,7 +951,7 @@ async def do_rerank(request: RerankRequest, raw_request: Request):
951
generator = await handler.do_rerank(request, raw_request)
952
if isinstance(generator, ErrorResponse):
953
return JSONResponse(content=generator.model_dump(),
953
- status_code=generator.code)
954
+ status_code=generator.error.code)
955
elif isinstance(generator, RerankResponse):
956
return JSONResponse(content=generator.model_dump())
957
@@ -1175,7 +1176,7 @@ async def invocations(raw_request: Request):
1176
msg = ("Cannot find suitable handler for request. "
1177
f"Expected one of: {type_names}")
1178
res = base(raw_request).create_error_response(message=msg)
1178
- return JSONResponse(content=res.model_dump(), status_code=res.code)
1179
+ return JSONResponse(content=res.model_dump(), status_code=res.error.code)
1180
1181
1182
if envs.VLLM_TORCH_PROFILER_DIR:
@@ -1211,7 +1212,7 @@ if envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
1212
response = await handler.load_lora_adapter(request)
1213
if isinstance(response, ErrorResponse):
1214
return JSONResponse(content=response.model_dump(),
1214
- status_code=response.code)
1215
+ status_code=response.error.code)
1216
1217
return Response(status_code=200, content=response)
1218
@@ -1223,7 +1224,7 @@ if envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
1224
response = await handler.unload_lora_adapter(request)
1225
if isinstance(response, ErrorResponse):
1226
return JSONResponse(content=response.model_dump(),
1226
- status_code=response.code)
1227
+ status_code=response.error.code)
1228
1229
return Response(status_code=200, content=response)
1230
@@ -1502,9 +1503,10 @@ def build_app(args: Namespace) -> FastAPI:
1503
1504
@app.exception_handler(HTTPException)
1505
async def http_exception_handler(_: Request, exc: HTTPException):
1505
- err = ErrorResponse(message=exc.detail,
1506
+ err = ErrorResponse(
1507
+ error=ErrorInfo(message=exc.detail,
1508
type=HTTPStatus(exc.status_code).phrase,
1507
- code=exc.status_code)
1509
+ code=exc.status_code))
1510
return JSONResponse(err.model_dump(), status_code=exc.status_code)
1511
1512
@app.exception_handler(RequestValidationError)
@@ -1518,9 +1520,9 @@ def build_app(args: Namespace) -> FastAPI:
1520
else:
1521
message = exc_str
1522
1521
- err = ErrorResponse(message=message,
1522
- type=HTTPStatus.BAD_REQUEST.phrase,
1523
- code=HTTPStatus.BAD_REQUEST)
1523
+ err = ErrorResponse(error=ErrorInfo(message=message,
1524
+ type=HTTPStatus.BAD_REQUEST.phrase,
1525
+ code=HTTPStatus.BAD_REQUEST))
1526
return JSONResponse(err.model_dump(),
1527
status_code=HTTPStatus.BAD_REQUEST)
1528
vllm/entrypoints/openai/protocol.py
+5
-2
@@ -78,14 +78,17 @@ class OpenAIBaseModel(BaseModel):
78
return result
79
80
81
-class ErrorResponse(OpenAIBaseModel):
82
- object: str = "error"
81
+class ErrorInfo(OpenAIBaseModel):
82
message: str
83
type: str
84
param: Optional[str] = None
85
code: int
86
87
88
+class ErrorResponse(OpenAIBaseModel):
89
+ error: ErrorInfo
90
+
91
+
92
class ModelPermission(OpenAIBaseModel):
93
id: str = Field(default_factory=lambda: f"modelperm-{random_uuid()}")
94
object: str = "model_permission"
vllm/entrypoints/openai/run_batch.py
+1
-1
@@ -302,7 +302,7 @@ async def run_request(serving_engine_func: Callable,
302
id=f"vllm-{random_uuid()}",
303
custom_id=request.custom_id,
304
response=BatchResponseData(
305
- status_code=response.code,
305
+ status_code=response.error.code,
306
request_id=f"vllm-batch-{random_uuid()}"),
307
error=response,
308
)
vllm/entrypoints/openai/serving_engine.py
+9
-12
@@ -47,10 +47,10 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
47
EmbeddingChatRequest,
48
EmbeddingCompletionRequest,
49
EmbeddingRequest,
50
- EmbeddingResponse, ErrorResponse,
51
- PoolingResponse, RerankRequest,
52
- ResponsesRequest, ScoreRequest,
53
- ScoreResponse,
50
+ EmbeddingResponse, ErrorInfo,
51
+ ErrorResponse, PoolingResponse,
52
+ RerankRequest, ResponsesRequest,
53
+ ScoreRequest, ScoreResponse,
54
TokenizeChatRequest,
55
TokenizeCompletionRequest,
56
TokenizeResponse,
@@ -412,21 +412,18 @@ class OpenAIServing:
412
message: str,
413
err_type: str = "BadRequestError",
414
status_code: HTTPStatus = HTTPStatus.BAD_REQUEST) -> ErrorResponse:
415
- return ErrorResponse(message=message,
416
- type=err_type,
417
- code=status_code.value)
415
+ return ErrorResponse(error=ErrorInfo(
416
+ message=message, type=err_type, code=status_code.value))
417
418
def create_streaming_error_response(
419
self,
420
message: str,
421
err_type: str = "BadRequestError",
422
status_code: HTTPStatus = HTTPStatus.BAD_REQUEST) -> str:
424
- json_str = json.dumps({
425
- "error":
423
+ json_str = json.dumps(
424
self.create_error_response(message=message,
425
err_type=err_type,
428
- status_code=status_code).model_dump()
429
- })
426
+ status_code=status_code).model_dump())
427
return json_str
428
429
async def _check_model(
@@ -445,7 +442,7 @@ class OpenAIServing:
442
if isinstance(load_result, LoRARequest):
443
return None
444
if isinstance(load_result, ErrorResponse) and \
448
- load_result.code == HTTPStatus.BAD_REQUEST.value:
445
+ load_result.error.code == HTTPStatus.BAD_REQUEST.value:
446
error_response = load_result
447
448
return error_response or self.create_error_response(
vllm/entrypoints/openai/serving_models.py
+4
-5
@@ -9,7 +9,7 @@ from typing import Optional, Union
9
10
from vllm.config import ModelConfig
11
from vllm.engine.protocol import EngineClient
12
-from vllm.entrypoints.openai.protocol import (ErrorResponse,
12
+from vllm.entrypoints.openai.protocol import (ErrorInfo, ErrorResponse,
13
LoadLoRAAdapterRequest,
14
ModelCard, ModelList,
15
ModelPermission,
@@ -82,7 +82,7 @@ class OpenAIServingModels:
82
load_result = await self.load_lora_adapter(
83
request=load_request, base_model_name=lora.base_model_name)
84
if isinstance(load_result, ErrorResponse):
85
- raise ValueError(load_result.message)
85
+ raise ValueError(load_result.error.message)
86
87
def is_base_model(self, model_name) -> bool:
88
return any(model.name == model_name for model in self.base_model_paths)
@@ -284,6 +284,5 @@ def create_error_response(
284
message: str,
285
err_type: str = "BadRequestError",
286
status_code: HTTPStatus = HTTPStatus.BAD_REQUEST) -> ErrorResponse:
287
- return ErrorResponse(message=message,
288
- type=err_type,
289
- code=status_code.value)
287
+ return ErrorResponse(error=ErrorInfo(
288
+ message=message, type=err_type, code=status_code.value))