A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 05:15 UTC
[openai api] log exception in exception handler (1/N) (#31164)
Signed-off-by: Andy Xie <andy.xning@gmail.com>
Ning Xie committed
Mar 6, 2026 at 00:00 UTC
176c799f4c512daf0904556940fc9a2c938af5ce
37 files changed
+912
-1191
tests/entrypoints/openai/responses/test_errors.py
-27
@@ -6,7 +6,6 @@ from unittest.mock import MagicMock
6
7
import pytest
8
9
-from vllm.entrypoints.openai.engine.protocol import ErrorResponse
9
from vllm.entrypoints.openai.engine.serving import GenerationError, OpenAIServing
10
11
@@ -38,32 +37,6 @@ async def test_raise_if_error_raises_generation_error():
37
serving._raise_if_error(None, "test-request-id") # should not raise
38
39
41
-@pytest.mark.asyncio
42
-async def test_convert_generation_error_to_response():
43
- """test _convert_generation_error_to_response creates proper ErrorResponse"""
44
- mock_engine = MagicMock()
45
- mock_engine.model_config = MagicMock()
46
- mock_engine.model_config.max_model_len = 100
47
- mock_models = MagicMock()
48
-
49
- serving = OpenAIServing(
50
- engine_client=mock_engine,
51
- models=mock_models,
52
- request_logger=None,
53
- )
54
-
55
- # create a GenerationError
56
- gen_error = GenerationError("Internal server error")
57
-
58
- # convert to ErrorResponse
59
- error_response = serving._convert_generation_error_to_response(gen_error)
60
-
61
- assert isinstance(error_response, ErrorResponse)
62
- assert error_response.error.type == "InternalServerError"
63
- assert error_response.error.message == "Internal server error"
64
- assert error_response.error.code == HTTPStatus.INTERNAL_SERVER_ERROR
65
-
66
-
40
@pytest.mark.asyncio
41
async def test_convert_generation_error_to_streaming_response():
42
"""test _convert_generation_error_to_streaming_response output"""
tests/entrypoints/openai/responses/test_harmony.py
+2
-2
@@ -13,7 +13,7 @@ from typing import Any
13
import pytest
14
import pytest_asyncio
15
import requests
16
-from openai import BadRequestError, NotFoundError, OpenAI
16
+from openai import InternalServerError, NotFoundError, OpenAI
17
from openai_harmony import Message
18
19
from ....utils import RemoteOpenAIServer
@@ -698,7 +698,7 @@ async def test_function_calling_multi_turn(client: OpenAI, model_name: str):
698
async def test_function_calling_required(client: OpenAI, model_name: str):
699
tools = [GET_WEATHER_SCHEMA]
700
701
- with pytest.raises(BadRequestError):
701
+ with pytest.raises(InternalServerError):
702
await client.responses.create(
703
model=model_name,
704
input="What's the weather like in Paris today?",
tests/entrypoints/openai/test_chat_error.py
+3
-8
@@ -2,7 +2,6 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
from dataclasses import dataclass, field
5
-from http import HTTPStatus
5
from typing import Any
6
from unittest.mock import AsyncMock, MagicMock, patch
7
@@ -11,7 +10,7 @@ import pytest
10
from vllm.config.multimodal import MultiModalConfig
11
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
12
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
14
-from vllm.entrypoints.openai.engine.protocol import ErrorResponse
13
+from vllm.entrypoints.openai.engine.protocol import GenerationError
14
from vllm.entrypoints.openai.models.protocol import BaseModelPath
15
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
16
from vllm.outputs import CompletionOutput, RequestOutput
@@ -145,12 +144,8 @@ async def test_chat_error_non_stream():
144
stream=False,
145
)
146
148
- response = await serving_chat.create_chat_completion(request)
149
-
150
- assert isinstance(response, ErrorResponse)
151
- assert response.error.type == "InternalServerError"
152
- assert response.error.message == "Internal server error"
153
- assert response.error.code == HTTPStatus.INTERNAL_SERVER_ERROR
147
+ with pytest.raises(GenerationError):
148
+ await serving_chat.create_chat_completion(request)
149
150
151
@pytest.mark.asyncio
tests/entrypoints/openai/test_completion_error.py
+3
-8
@@ -2,7 +2,6 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
from dataclasses import dataclass, field
5
-from http import HTTPStatus
5
from typing import Any
6
from unittest.mock import MagicMock
7
@@ -11,7 +10,7 @@ import pytest
10
from vllm.config.multimodal import MultiModalConfig
11
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
12
from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
14
-from vllm.entrypoints.openai.engine.protocol import ErrorResponse
13
+from vllm.entrypoints.openai.engine.protocol import GenerationError
14
from vllm.entrypoints.openai.models.protocol import BaseModelPath
15
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
16
from vllm.outputs import CompletionOutput, RequestOutput
@@ -131,12 +130,8 @@ async def test_completion_error_non_stream():
130
stream=False,
131
)
132
134
- response = await serving_completion.create_completion(request)
135
-
136
- assert isinstance(response, ErrorResponse)
137
- assert response.error.type == "InternalServerError"
138
- assert response.error.message == "Internal server error"
139
- assert response.error.code == HTTPStatus.INTERNAL_SERVER_ERROR
133
+ with pytest.raises(GenerationError):
134
+ await serving_completion.create_completion(request)
135
136
137
@pytest.mark.asyncio
tests/entrypoints/openai/test_openai_schema.py
+28
-2
@@ -1,12 +1,18 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import json
4
+from http import HTTPStatus
5
from typing import Final
6
7
import pytest
8
import schemathesis
9
+from httpx import URL
10
from hypothesis import settings
11
from schemathesis import GenerationConfig
12
+from schemathesis.checks import not_a_server_error
13
+from schemathesis.internal.checks import CheckContext
14
+from schemathesis.models import Case
15
+from schemathesis.transports.responses import GenericResponse
16
17
from ...utils import RemoteOpenAIServer
18
@@ -127,10 +133,25 @@ def before_generate_case(context: schemathesis.hooks.HookContext, strategy):
133
return strategy.filter(no_invalid_types)
134
135
136
+def customized_not_a_server_error(
137
+ ctx: CheckContext, response: GenericResponse, case: Case
138
+) -> bool | None:
139
+ try:
140
+ return not_a_server_error(ctx, response, case)
141
+ except Exception:
142
+ if (
143
+ URL(response.request.url).path
144
+ in ["/v1/chat/completions/render", "/v1/chat/completions"]
145
+ and response.status_code == HTTPStatus.NOT_IMPLEMENTED.value
146
+ ):
147
+ return True
148
+ raise
149
+
150
+
151
@schema.parametrize()
152
@schema.override(headers={"Content-Type": "application/json"})
153
@settings(deadline=LONG_TIMEOUT_SECONDS * 1000, max_examples=50)
133
-def test_openapi_stateless(case: schemathesis.Case):
154
+def test_openapi_stateless(case: Case):
155
key = (
156
case.operation.method.upper(),
157
case.operation.path,
@@ -155,4 +176,9 @@ def test_openapi_stateless(case: schemathesis.Case):
176
}.get(key, DEFAULT_TIMEOUT_SECONDS)
177
178
# No need to verify SSL certificate for localhost
158
- case.call_and_validate(verify=False, timeout=timeout)
179
+ case.call_and_validate(
180
+ verify=False,
181
+ timeout=timeout,
182
+ additional_checks=(customized_not_a_server_error,),
183
+ excluded_checks=(not_a_server_error,),
184
+ )
tests/entrypoints/openai/test_serving_chat.py
+5
-6
@@ -23,6 +23,7 @@ from vllm.entrypoints.openai.engine.protocol import (
23
)
24
from vllm.entrypoints.openai.models.serving import BaseModelPath, OpenAIServingModels
25
from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
26
+from vllm.exceptions import VLLMValidationError
27
from vllm.inputs import TokensPrompt
28
from vllm.outputs import CompletionOutput, RequestOutput
29
from vllm.renderers.hf import HfRenderer
@@ -818,9 +819,8 @@ async def test_serving_chat_mistral_token_ids_prompt_is_validated():
819
max_tokens=10,
820
)
821
821
- resp = await serving_chat.create_chat_completion(req)
822
- assert isinstance(resp, ErrorResponse)
823
- assert "context length is only" in resp.error.message
822
+ with pytest.raises(VLLMValidationError):
823
+ await serving_chat.create_chat_completion(req)
824
825
826
@pytest.mark.asyncio
@@ -860,9 +860,8 @@ async def test_serving_chat_mistral_token_ids_prompt_too_long_is_rejected():
860
max_tokens=1,
861
)
862
863
- resp = await serving_chat.create_chat_completion(req)
864
- assert isinstance(resp, ErrorResponse)
865
- assert "context length is only" in resp.error.message
863
+ with pytest.raises(VLLMValidationError):
864
+ await serving_chat.create_chat_completion(req)
865
866
867
@pytest.mark.asyncio
tests/v1/engine/test_async_llm.py
+3
-8
@@ -17,9 +17,6 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
17
ChatCompletionResponse,
18
)
19
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
20
-from vllm.entrypoints.openai.engine.protocol import (
21
- ErrorResponse,
22
-)
20
from vllm.entrypoints.openai.models.protocol import BaseModelPath
21
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
22
from vllm.inputs import PromptType
@@ -542,11 +539,9 @@ async def test_header_dp_rank_argument():
539
# Test 2: Out-of-range DP rank (1)
540
mock_raw_request.headers = {"X-data-parallel-rank": "1"}
541
545
- # should return ErrorResponse for out-of-range rank
546
- response2 = await serving_chat.create_chat_completion(req, mock_raw_request)
547
- assert isinstance(response2, ErrorResponse), (
548
- "Expected an ErrorResponse for out-of-range DP rank"
549
- )
542
+ # should raise ValueError for out-of-range rank
543
+ with pytest.raises(ValueError):
544
+ await serving_chat.create_chat_completion(req, mock_raw_request)
545
546
547
@pytest.mark.asyncio
vllm/entrypoints/launcher.py
+2
-41
@@ -4,11 +4,10 @@
4
import asyncio
5
import signal
6
import socket
7
-from http import HTTPStatus
7
from typing import Any
8
9
import uvicorn
11
-from fastapi import FastAPI, Request, Response
10
+from fastapi import FastAPI
11
12
from vllm import envs
13
from vllm.engine.protocol import EngineClient
@@ -19,7 +18,6 @@ from vllm.entrypoints.constants import (
18
from vllm.entrypoints.ssl import SSLCertRefresher
19
from vllm.logger import init_logger
20
from vllm.utils.network_utils import find_process_using_port
22
-from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
21
22
logger = init_logger(__name__)
23
@@ -75,7 +73,7 @@ async def serve_http(
73
config.h11_max_header_count = h11_max_header_count
74
config.load()
75
server = uvicorn.Server(config)
78
- _add_shutdown_handlers(app, server)
76
+ app.state.server = server
77
78
loop = asyncio.get_running_loop()
79
@@ -148,40 +146,3 @@ def terminate_if_errored(server: uvicorn.Server, engine: EngineClient):
146
engine_errored = engine.errored and not engine.is_running
147
if not envs.VLLM_KEEP_ALIVE_ON_ENGINE_DEATH and engine_errored:
148
server.should_exit = True
151
-
152
-
153
-def _add_shutdown_handlers(app: FastAPI, server: uvicorn.Server) -> None:
154
- """
155
- VLLM V1 AsyncLLM catches exceptions and returns
156
- only two types: EngineGenerateError and EngineDeadError.
157
-
158
- EngineGenerateError is raised by the per request generate()
159
- method. This error could be request specific (and therefore
160
- recoverable - e.g. if there is an error in input processing).
161
-
162
- EngineDeadError is raised by the background output_handler
163
- method. This error is global and therefore not recoverable.
164
-
165
- We register these @app.exception_handlers to return nice
166
- responses to the end user if they occur and shut down if needed.
167
- See https://fastapi.tiangolo.com/tutorial/handling-errors/
168
- for more details on how exception handlers work.
169
-
170
- If an exception is encountered in a StreamingResponse
171
- generator, the exception is not raised, since we already sent
172
- a 200 status. Rather, we send an error message as the next chunk.
173
- Since the exception is not raised, this means that the server
174
- will not automatically shut down. Instead, we use the watchdog
175
- background task for check for errored state.
176
- """
177
-
178
- @app.exception_handler(RuntimeError)
179
- @app.exception_handler(EngineDeadError)
180
- @app.exception_handler(EngineGenerateError)
181
- async def runtime_exception_handler(request: Request, __):
182
- terminate_if_errored(
183
- server=server,
184
- engine=request.app.state.engine_client,
185
- )
186
-
187
- return Response(status_code=HTTPStatus.INTERNAL_SERVER_ERROR)
vllm/entrypoints/openai/api_server.py
+6
-1
@@ -31,6 +31,8 @@ from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_se
31
from vllm.entrypoints.openai.models.protocol import BaseModelPath
32
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
33
from vllm.entrypoints.openai.server_utils import (
34
+ engine_error_handler,
35
+ exception_handler,
36
get_uvicorn_log_config,
37
http_exception_handler,
38
lifespan,
@@ -57,6 +59,7 @@ from vllm.usage.usage_lib import UsageContext
59
from vllm.utils.argparse_utils import FlexibleArgumentParser
60
from vllm.utils.network_utils import is_valid_ipv6_address
61
from vllm.utils.system_utils import decorate_logs, set_ulimit
62
+from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
63
from vllm.version import __version__ as VLLM_VERSION
64
65
prometheus_multiproc_dir: tempfile.TemporaryDirectory
@@ -250,6 +253,9 @@ def build_app(
253
254
app.exception_handler(HTTPException)(http_exception_handler)
255
app.exception_handler(RequestValidationError)(validation_exception_handler)
256
+ app.exception_handler(EngineGenerateError)(engine_error_handler)
257
+ app.exception_handler(EngineDeadError)(engine_error_handler)
258
+ app.exception_handler(Exception)(exception_handler)
259
260
# Ensure --api-key option from CLI takes precedence over VLLM_API_KEY
261
if tokens := [key for key in (args.api_key or [envs.VLLM_API_KEY]) if key]:
@@ -355,7 +361,6 @@ async def init_app_state(
361
chat_template=resolved_chat_template,
362
chat_template_content_format=args.chat_template_content_format,
363
trust_request_chat_template=args.trust_request_chat_template,
358
- log_error_stack=args.log_error_stack,
364
)
365
366
if any(task in supported_tasks for task in ("generate", "render")):
vllm/entrypoints/openai/chat_completion/api_router.py
+4
-8
@@ -39,6 +39,7 @@ def chat(request: Request) -> OpenAIServingChat | None:
39
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
40
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
41
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
42
+ HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
43
},
44
)
45
@with_cancellation
@@ -54,10 +55,7 @@ async def create_chat_completion(request: ChatCompletionRequest, raw_request: Re
55
message="The model does not support Chat Completions API"
56
)
57
57
- try:
58
- generator = await handler.create_chat_completion(request, raw_request)
59
- except Exception as e:
60
- generator = handler.create_error_response(e)
58
+ generator = await handler.create_chat_completion(request, raw_request)
59
60
if isinstance(generator, ErrorResponse):
61
return JSONResponse(
@@ -81,6 +79,7 @@ async def create_chat_completion(request: ChatCompletionRequest, raw_request: Re
79
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
80
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
81
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
82
+ HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
83
},
84
)
85
async def render_chat_completion(request: ChatCompletionRequest, raw_request: Request):
@@ -93,10 +92,7 @@ async def render_chat_completion(request: ChatCompletionRequest, raw_request: Re
92
message="The model does not support Chat Completions API"
93
)
94
96
- try:
97
- result = await handler.render_chat_request(request)
98
- except Exception as e:
99
- result = handler.create_error_response(e)
95
+ result = await handler.render_chat_request(request)
96
97
if isinstance(result, ErrorResponse):
98
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
vllm/entrypoints/openai/chat_completion/serving.py
+147
-177
@@ -8,7 +8,6 @@ from collections.abc import AsyncGenerator, AsyncIterator
8
from collections.abc import Sequence as GenericSequence
9
from typing import Any, Final
10
11
-import jinja2
11
import partial_json_parser
12
import regex as re
13
from fastapi import Request
@@ -105,7 +104,6 @@ class OpenAIServingChat(OpenAIServing):
104
enable_force_include_usage: bool = False,
105
enable_log_outputs: bool = False,
106
enable_log_deltas: bool = True,
108
- log_error_stack: bool = False,
107
default_chat_template_kwargs: dict[str, Any] | None = None,
108
) -> None:
109
super().__init__(
@@ -113,7 +111,6 @@ class OpenAIServingChat(OpenAIServing):
111
models=models,
112
request_logger=request_logger,
113
return_tokens_as_token_ids=return_tokens_as_token_ids,
116
- log_error_stack=log_error_stack,
114
)
115
116
self.response_role = response_role
@@ -235,81 +232,76 @@ class OpenAIServingChat(OpenAIServing):
232
if self.engine_client.errored:
233
raise self.engine_client.dead_error
234
238
- try:
239
- tokenizer = self.renderer.tokenizer
240
-
241
- tool_parser = self.tool_parser
242
-
243
- if is_mistral_tokenizer(tokenizer):
244
- # because of issues with pydantic we need to potentially
245
- # re-serialize the tool_calls field of the request
246
- # for more info: see comment in `maybe_serialize_tool_calls`
247
- _mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type]
248
- _mt.truncate_tool_call_ids(request) # type: ignore[arg-type]
249
- _mt.validate_request_params(request)
250
-
251
- # Check if tool parsing is unavailable (common condition)
252
- tool_parsing_unavailable = (
253
- tool_parser is None
254
- and not is_mistral_tokenizer(tokenizer)
255
- and not self.use_harmony
256
- )
235
+ tokenizer = self.renderer.tokenizer
236
258
- # Validate tool_choice when tool parsing is required but unavailable
259
- if tool_parsing_unavailable and request.tool_choice not in (
260
- None,
261
- "none",
262
- ):
263
- if request.tool_choice == "auto" and not self.enable_auto_tools:
264
- # for hf tokenizers, "auto" tools requires
265
- # --enable-auto-tool-choice and --tool-call-parser
266
- return self.create_error_response(
267
- '"auto" tool choice requires '
268
- "--enable-auto-tool-choice and --tool-call-parser to be set"
269
- )
270
- elif request.tool_choice != "auto":
271
- # "required" or named tool requires tool parser
272
- return self.create_error_response(
273
- f'tool_choice="{request.tool_choice}" requires '
274
- "--tool-call-parser to be set"
275
- )
237
+ tool_parser = self.tool_parser
238
+
239
+ if is_mistral_tokenizer(tokenizer):
240
+ # because of issues with pydantic we need to potentially
241
+ # re-serialize the tool_calls field of the request
242
+ # for more info: see comment in `maybe_serialize_tool_calls`
243
+ _mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type]
244
+ _mt.truncate_tool_call_ids(request) # type: ignore[arg-type]
245
+ _mt.validate_request_params(request)
246
+
247
+ # Check if tool parsing is unavailable (common condition)
248
+ tool_parsing_unavailable = (
249
+ tool_parser is None
250
+ and not is_mistral_tokenizer(tokenizer)
251
+ and not self.use_harmony
252
+ )
253
277
- if request.tools is None or (
278
- request.tool_choice == "none"
279
- and self.exclude_tools_when_tool_choice_none
280
- ):
281
- tool_dicts = None
282
- else:
283
- tool_dicts = [tool.model_dump() for tool in request.tools]
284
-
285
- if not self.use_harmony:
286
- # Common case.
287
- error_check_ret = self._validate_chat_template(
288
- request_chat_template=request.chat_template,
289
- chat_template_kwargs=request.chat_template_kwargs,
290
- trust_request_chat_template=self.trust_request_chat_template,
254
+ # Validate tool_choice when tool parsing is required but unavailable
255
+ if tool_parsing_unavailable and request.tool_choice not in (
256
+ None,
257
+ "none",
258
+ ):
259
+ if request.tool_choice == "auto" and not self.enable_auto_tools:
260
+ # for hf tokenizers, "auto" tools requires
261
+ # --enable-auto-tool-choice and --tool-call-parser
262
+ return self.create_error_response(
263
+ '"auto" tool choice requires '
264
+ "--enable-auto-tool-choice and --tool-call-parser to be set"
265
)
292
- if error_check_ret is not None:
293
- return error_check_ret
294
-
295
- conversation, engine_prompts = await self._preprocess_chat(
296
- request,
297
- request.messages,
298
- default_template=self.chat_template,
299
- default_template_content_format=self.chat_template_content_format,
300
- default_template_kwargs=self.default_chat_template_kwargs,
301
- tool_dicts=tool_dicts,
302
- tool_parser=tool_parser,
266
+ elif request.tool_choice != "auto":
267
+ # "required" or named tool requires tool parser
268
+ return self.create_error_response(
269
+ f'tool_choice="{request.tool_choice}" requires '
270
+ "--tool-call-parser to be set"
271
)
304
- else:
305
- # For GPT-OSS.
306
- should_include_tools = tool_dicts is not None
307
- conversation, engine_prompts = self._make_request_with_harmony(
308
- request, should_include_tools
309
- )
310
- except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e:
311
- logger.exception("Error in preprocessing prompt inputs")
312
- return self.create_error_response(e)
272
+
273
+ if request.tools is None or (
274
+ request.tool_choice == "none" and self.exclude_tools_when_tool_choice_none
275
+ ):
276
+ tool_dicts = None
277
+ else:
278
+ tool_dicts = [tool.model_dump() for tool in request.tools]
279
+
280
+ if not self.use_harmony:
281
+ # Common case.
282
+ error_check_ret = self._validate_chat_template(
283
+ request_chat_template=request.chat_template,
284
+ chat_template_kwargs=request.chat_template_kwargs,
285
+ trust_request_chat_template=self.trust_request_chat_template,
286
+ )
287
+ if error_check_ret is not None:
288
+ return error_check_ret
289
+
290
+ conversation, engine_prompts = await self._preprocess_chat(
291
+ request,
292
+ request.messages,
293
+ default_template=self.chat_template,
294
+ default_template_content_format=self.chat_template_content_format,
295
+ default_template_kwargs=self.default_chat_template_kwargs,
296
+ tool_dicts=tool_dicts,
297
+ tool_parser=tool_parser,
298
+ )
299
+ else:
300
+ # For GPT-OSS.
301
+ should_include_tools = tool_dicts is not None
302
+ conversation, engine_prompts = self._make_request_with_harmony(
303
+ request, should_include_tools
304
+ )
305
306
return conversation, engine_prompts
307
@@ -329,20 +321,16 @@ class OpenAIServingChat(OpenAIServing):
321
tokenizer = self.renderer.tokenizer
322
assert tokenizer is not None
323
reasoning_parser: ReasoningParser | None = None
332
- try:
333
- if self.reasoning_parser_cls:
334
- # Pass the same chat template kwargs as used in tokenization
335
- chat_template_kwargs = self._prepare_extra_chat_template_kwargs(
336
- request.chat_template_kwargs,
337
- self.default_chat_template_kwargs,
338
- )
339
- reasoning_parser = self.reasoning_parser_cls(
340
- tokenizer,
341
- chat_template_kwargs=chat_template_kwargs, # type: ignore[call-arg]
342
- )
343
- except RuntimeError as e:
344
- logger.exception("Error in reasoning parser creation.")
345
- return self.create_error_response(str(e))
324
+ if self.reasoning_parser_cls:
325
+ # Pass the same chat template kwargs as used in tokenization
326
+ chat_template_kwargs = self._prepare_extra_chat_template_kwargs(
327
+ request.chat_template_kwargs,
328
+ self.default_chat_template_kwargs,
329
+ )
330
+ reasoning_parser = self.reasoning_parser_cls(
331
+ tokenizer,
332
+ chat_template_kwargs=chat_template_kwargs, # type: ignore[call-arg]
333
+ )
334
result = await self.render_chat_request(request)
335
if isinstance(result, ErrorResponse):
336
return result
@@ -357,15 +345,9 @@ class OpenAIServingChat(OpenAIServing):
345
if raw_request:
346
raw_request.state.request_metadata = request_metadata
347
360
- try:
361
- lora_request = self._maybe_get_adapters(
362
- request, supports_default_mm_loras=True
363
- )
348
+ lora_request = self._maybe_get_adapters(request, supports_default_mm_loras=True)
349
365
- model_name = self.models.model_name(lora_request)
366
- except (ValueError, TypeError, RuntimeError) as e:
367
- logger.exception("Error preparing request components")
368
- return self.create_error_response(e)
350
+ model_name = self.models.model_name(lora_request)
351
352
# Extract data_parallel_rank from header (router can inject it)
353
data_parallel_rank = self._get_data_parallel_rank(raw_request)
@@ -373,81 +355,76 @@ class OpenAIServingChat(OpenAIServing):
355
# Schedule the request and get the result generator.
356
max_model_len = self.model_config.max_model_len
357
generators: list[AsyncGenerator[RequestOutput, None]] = []
376
- try:
377
- for i, engine_prompt in enumerate(engine_prompts):
378
- prompt_token_ids = self._extract_prompt_components(
379
- engine_prompt
380
- ).token_ids
381
-
382
- # If we are creating sub requests for multiple prompts, ensure that they
383
- # have unique request ids.
384
- sub_request_id = (
385
- request_id if len(engine_prompts) == 1 else f"{request_id}_{i}"
386
- )
358
+ for i, engine_prompt in enumerate(engine_prompts):
359
+ prompt_token_ids = self._extract_prompt_components(engine_prompt).token_ids
360
+
361
+ # If we are creating sub requests for multiple prompts, ensure that they
362
+ # have unique request ids.
363
+ sub_request_id = (
364
+ request_id if len(engine_prompts) == 1 else f"{request_id}_{i}"
365
+ )
366
+
367
+ max_tokens = get_max_tokens(
368
+ max_model_len,
369
+ request.max_completion_tokens
370
+ if request.max_completion_tokens is not None
371
+ else request.max_tokens,
372
+ self._extract_prompt_len(engine_prompt),
373
+ self.default_sampling_params,
374
+ self.override_max_tokens,
375
+ )
376
388
- max_tokens = get_max_tokens(
389
- max_model_len,
390
- request.max_completion_tokens
391
- if request.max_completion_tokens is not None
392
- else request.max_tokens,
393
- self._extract_prompt_len(engine_prompt),
377
+ sampling_params: SamplingParams | BeamSearchParams
378
+ if request.use_beam_search:
379
+ sampling_params = request.to_beam_search_params(
380
+ max_tokens, self.default_sampling_params
381
+ )
382
+ else:
383
+ sampling_params = request.to_sampling_params(
384
+ max_tokens,
385
self.default_sampling_params,
395
- self.override_max_tokens,
386
)
387
398
- sampling_params: SamplingParams | BeamSearchParams
399
- if request.use_beam_search:
400
- sampling_params = request.to_beam_search_params(
401
- max_tokens, self.default_sampling_params
402
- )
403
- else:
404
- sampling_params = request.to_sampling_params(
405
- max_tokens,
406
- self.default_sampling_params,
407
- )
388
+ self._log_inputs(
389
+ sub_request_id,
390
+ engine_prompt,
391
+ params=sampling_params,
392
+ lora_request=lora_request,
393
+ )
394
409
- self._log_inputs(
410
- sub_request_id,
411
- engine_prompt,
395
+ trace_headers = (
396
+ None
397
+ if raw_request is None
398
+ else await self._get_trace_headers(raw_request.headers)
399
+ )
400
+
401
+ if isinstance(sampling_params, BeamSearchParams):
402
+ generator = self.beam_search(
403
+ prompt=engine_prompt,
404
+ request_id=sub_request_id,
405
params=sampling_params,
406
lora_request=lora_request,
407
+ trace_headers=trace_headers,
408
)
415
-
416
- trace_headers = (
417
- None
418
- if raw_request is None
419
- else await self._get_trace_headers(raw_request.headers)
409
+ else:
410
+ reasoning_ended = (
411
+ reasoning_parser.is_reasoning_end(prompt_token_ids or [])
412
+ if reasoning_parser
413
+ else None
414
)
415
422
- if isinstance(sampling_params, BeamSearchParams):
423
- generator = self.beam_search(
424
- prompt=engine_prompt,
425
- request_id=sub_request_id,
426
- params=sampling_params,
427
- lora_request=lora_request,
428
- trace_headers=trace_headers,
429
- )
430
- else:
431
- reasoning_ended = (
432
- reasoning_parser.is_reasoning_end(prompt_token_ids or [])
433
- if reasoning_parser
434
- else None
435
- )
436
-
437
- generator = self.engine_client.generate(
438
- engine_prompt,
439
- sampling_params,
440
- sub_request_id,
441
- lora_request=lora_request,
442
- trace_headers=trace_headers,
443
- priority=request.priority,
444
- data_parallel_rank=data_parallel_rank,
445
- reasoning_ended=reasoning_ended,
446
- )
416
+ generator = self.engine_client.generate(
417
+ engine_prompt,
418
+ sampling_params,
419
+ sub_request_id,
420
+ lora_request=lora_request,
421
+ trace_headers=trace_headers,
422
+ priority=request.priority,
423
+ data_parallel_rank=data_parallel_rank,
424
+ reasoning_ended=reasoning_ended,
425
+ )
426
448
- generators.append(generator)
449
- except ValueError as e:
450
- return self.create_error_response(e)
427
+ generators.append(generator)
428
429
assert len(generators) == 1
430
(result_generator,) = generators
@@ -464,21 +441,16 @@ class OpenAIServingChat(OpenAIServing):
441
reasoning_parser,
442
)
443
467
- try:
468
- return await self.chat_completion_full_generator(
469
- request,
470
- result_generator,
471
- request_id,
472
- model_name,
473
- conversation,
474
- tokenizer,
475
- request_metadata,
476
- reasoning_parser,
477
- )
478
- except GenerationError as e:
479
- return self._convert_generation_error_to_response(e)
480
- except ValueError as e:
481
- return self.create_error_response(e)
444
+ return await self.chat_completion_full_generator(
445
+ request,
446
+ result_generator,
447
+ request_id,
448
+ model_name,
449
+ conversation,
450
+ tokenizer,
451
+ request_metadata,
452
+ reasoning_parser,
453
+ )
454
455
def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
456
if request.add_generation_prompt:
@@ -1414,8 +1386,6 @@ class OpenAIServingChat(OpenAIServing):
1386
final_res = res
1387
except asyncio.CancelledError:
1388
return self.create_error_response("Client disconnected")
1417
- except ValueError as e:
1418
- return self.create_error_response(e)
1389
1390
assert final_res is not None
1391
vllm/entrypoints/openai/completion/api_router.py
+2
-8
@@ -54,10 +54,7 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
54
message="The model does not support Completions API"
55
)
56
57
- try:
58
- generator = await handler.create_completion(request, raw_request)
59
- except Exception as e:
60
- generator = handler.create_error_response(e)
57
+ generator = await handler.create_completion(request, raw_request)
58
59
if isinstance(generator, ErrorResponse):
60
return JSONResponse(
@@ -91,10 +88,7 @@ async def render_completion(request: CompletionRequest, raw_request: Request):
88
message="The model does not support Completions API"
89
)
90
94
- try:
95
- result = await handler.render_completion_request(request)
96
- except Exception as e:
97
- result = handler.create_error_response(e)
91
+ result = await handler.render_completion_request(request)
92
93
if isinstance(result, ErrorResponse):
94
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
vllm/entrypoints/openai/completion/serving.py
+51
-69
@@ -7,7 +7,6 @@ from collections.abc import AsyncGenerator, AsyncIterator
7
from collections.abc import Sequence as GenericSequence
8
from typing import cast
9
10
-import jinja2
10
from fastapi import Request
11
12
from vllm.engine.protocol import EngineClient
@@ -56,14 +55,12 @@ class OpenAIServingCompletion(OpenAIServing):
55
return_tokens_as_token_ids: bool = False,
56
enable_prompt_tokens_details: bool = False,
57
enable_force_include_usage: bool = False,
59
- log_error_stack: bool = False,
58
):
59
super().__init__(
60
engine_client=engine_client,
61
models=models,
62
request_logger=request_logger,
63
return_tokens_as_token_ids=return_tokens_as_token_ids,
66
- log_error_stack=log_error_stack,
64
)
65
66
self.enable_prompt_tokens_details = enable_prompt_tokens_details
@@ -110,15 +107,11 @@ class OpenAIServingCompletion(OpenAIServing):
107
"prompt_logprobs is not compatible with prompt embeds."
108
)
109
113
- try:
114
- engine_prompts = await self._preprocess_completion(
115
- request,
116
- prompt_input=request.prompt,
117
- prompt_embeds=request.prompt_embeds,
118
- )
119
- except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e:
120
- logger.exception("Error in preprocessing prompt inputs")
121
- return self.create_error_response(e)
110
+ engine_prompts = await self._preprocess_completion(
111
+ request,
112
+ prompt_input=request.prompt,
113
+ prompt_embeds=request.prompt_embeds,
114
+ )
115
116
return engine_prompts
117
@@ -149,11 +142,7 @@ class OpenAIServingCompletion(OpenAIServing):
142
if raw_request:
143
raw_request.state.request_metadata = request_metadata
144
152
- try:
153
- lora_request = self._maybe_get_adapters(request)
154
- except (ValueError, TypeError, RuntimeError) as e:
155
- logger.exception("Error preparing request components")
156
- return self.create_error_response(e)
145
+ lora_request = self._maybe_get_adapters(request)
146
147
# Extract data_parallel_rank from header (router can inject it)
148
data_parallel_rank = self._get_data_parallel_rank(raw_request)
@@ -161,64 +150,61 @@ class OpenAIServingCompletion(OpenAIServing):
150
# Schedule the request and get the result generator.
151
max_model_len = self.model_config.max_model_len
152
generators: list[AsyncGenerator[RequestOutput, None]] = []
164
- try:
165
- for i, engine_prompt in enumerate(engine_prompts):
166
- max_tokens = get_max_tokens(
167
- max_model_len,
168
- request.max_tokens,
169
- self._extract_prompt_len(engine_prompt),
153
+ for i, engine_prompt in enumerate(engine_prompts):
154
+ max_tokens = get_max_tokens(
155
+ max_model_len,
156
+ request.max_tokens,
157
+ self._extract_prompt_len(engine_prompt),
158
+ self.default_sampling_params,
159
+ self.override_max_tokens,
160
+ )
161
+
162
+ sampling_params: SamplingParams | BeamSearchParams
163
+ if request.use_beam_search:
164
+ sampling_params = request.to_beam_search_params(
165
+ max_tokens, self.default_sampling_params
166
+ )
167
+ else:
168
+ sampling_params = request.to_sampling_params(
169
+ max_tokens,
170
self.default_sampling_params,
171
- self.override_max_tokens,
171
)
172
174
- sampling_params: SamplingParams | BeamSearchParams
175
- if request.use_beam_search:
176
- sampling_params = request.to_beam_search_params(
177
- max_tokens, self.default_sampling_params
178
- )
179
- else:
180
- sampling_params = request.to_sampling_params(
181
- max_tokens,
182
- self.default_sampling_params,
183
- )
173
+ request_id_item = f"{request_id}-{i}"
174
+
175
+ self._log_inputs(
176
+ request_id_item,
177
+ engine_prompt,
178
+ params=sampling_params,
179
+ lora_request=lora_request,
180
+ )
181
185
- request_id_item = f"{request_id}-{i}"
182
+ trace_headers = (
183
+ None
184
+ if raw_request is None
185
+ else await self._get_trace_headers(raw_request.headers)
186
+ )
187
187
- self._log_inputs(
188
- request_id_item,
189
- engine_prompt,
188
+ if isinstance(sampling_params, BeamSearchParams):
189
+ generator = self.beam_search(
190
+ prompt=engine_prompt,
191
+ request_id=request_id,
192
params=sampling_params,
193
lora_request=lora_request,
194
+ trace_headers=trace_headers,
195
)
193
-
194
- trace_headers = (
195
- None
196
- if raw_request is None
197
- else await self._get_trace_headers(raw_request.headers)
196
+ else:
197
+ generator = self.engine_client.generate(
198
+ engine_prompt,
199
+ sampling_params,
200
+ request_id_item,
201
+ lora_request=lora_request,
202
+ trace_headers=trace_headers,
203
+ priority=request.priority,
204
+ data_parallel_rank=data_parallel_rank,
205
)
206
200
- if isinstance(sampling_params, BeamSearchParams):
201
- generator = self.beam_search(
202
- prompt=engine_prompt,
203
- request_id=request_id,
204
- params=sampling_params,
205
- lora_request=lora_request,
206
- trace_headers=trace_headers,
207
- )
208
- else:
209
- generator = self.engine_client.generate(
210
- engine_prompt,
211
- sampling_params,
212
- request_id_item,
213
- lora_request=lora_request,
214
- trace_headers=trace_headers,
215
- priority=request.priority,
216
- data_parallel_rank=data_parallel_rank,
217
- )
218
-
219
- generators.append(generator)
220
- except ValueError as e:
221
- return self.create_error_response(e)
207
+ generators.append(generator)
208
209
result_generator = merge_async_iterators(*generators)
210
@@ -273,10 +259,6 @@ class OpenAIServingCompletion(OpenAIServing):
259
)
260
except asyncio.CancelledError:
261
return self.create_error_response("Client disconnected")
276
- except GenerationError as e:
277
- return self._convert_generation_error_to_response(e)
278
- except ValueError as e:
279
- return self.create_error_response(e)
262
263
# When user requests streaming but we don't stream, we still need to
264
# return a streaming response with a single event.
vllm/entrypoints/openai/engine/protocol.py
+9
@@ -4,6 +4,7 @@
4
# Adapted from
5
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
6
import time
7
+from http import HTTPStatus
8
from typing import Any, ClassVar, Literal, TypeAlias
9
10
import regex as re
@@ -262,6 +263,14 @@ class DeltaMessage(OpenAIBaseModel):
263
tool_calls: list[DeltaToolCall] = Field(default_factory=list)
264
265
266
+class GenerationError(Exception):
267
+ """raised when finish_reason indicates internal server error (500)"""
268
+
269
+ def __init__(self, message: str = "Internal server error"):
270
+ super().__init__(message)
271
+ self.status_code = HTTPStatus.INTERNAL_SERVER_ERROR
272
+
273
+
274
####### Tokens IN <> Tokens OUT #######
275
class GenerateRequest(BaseModel):
276
request_id: str = Field(
vllm/entrypoints/openai/engine/serving.py
+48
-126
@@ -2,9 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import asyncio
4
import json
5
-import sys
5
import time
7
-import traceback
6
from collections.abc import AsyncGenerator, Callable, Mapping, Sequence
7
from dataclasses import dataclass, field
8
from http import HTTPStatus
@@ -38,10 +36,10 @@ from vllm.entrypoints.openai.completion.protocol import (
36
CompletionResponse,
37
)
38
from vllm.entrypoints.openai.engine.protocol import (
41
- ErrorInfo,
39
ErrorResponse,
40
FunctionCall,
41
FunctionDefinition,
42
+ GenerationError,
43
)
44
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
45
from vllm.entrypoints.openai.responses.context import (
@@ -89,7 +87,7 @@ from vllm.entrypoints.serve.tokenize.protocol import (
87
TokenizeCompletionRequest,
88
TokenizeResponse,
89
)
92
-from vllm.entrypoints.utils import get_max_tokens, sanitize_message
90
+from vllm.entrypoints.utils import create_error_response, get_max_tokens
91
from vllm.exceptions import VLLMValidationError
92
from vllm.inputs.data import (
93
ProcessorInputs,
@@ -125,15 +123,6 @@ from vllm.utils.async_utils import (
123
)
124
from vllm.utils.mistral import is_mistral_tokenizer
125
128
-
129
-class GenerationError(Exception):
130
- """raised when finish_reason indicates internal server error (500)"""
131
-
132
- def __init__(self, message: str = "Internal server error"):
133
- super().__init__(message)
134
- self.status_code = HTTPStatus.INTERNAL_SERVER_ERROR
135
-
136
-
126
logger = init_logger(__name__)
127
128
@@ -225,7 +214,6 @@ class OpenAIServing:
214
*,
215
request_logger: RequestLogger | None,
216
return_tokens_as_token_ids: bool = False,
228
- log_error_stack: bool = False,
217
):
218
super().__init__()
219
@@ -236,8 +224,6 @@ class OpenAIServing:
224
self.request_logger = request_logger
225
self.return_tokens_as_token_ids = return_tokens_as_token_ids
226
239
- self.log_error_stack = log_error_stack
240
-
227
self.model_config = engine_client.model_config
228
self.renderer = engine_client.renderer
229
self.io_processor = engine_client.io_processor
@@ -526,133 +512,79 @@ class OpenAIServing:
512
"""Schedule the request and get the result generator."""
513
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
514
529
- try:
530
- trace_headers = (
531
- None
532
- if ctx.raw_request is None
533
- else await self._get_trace_headers(ctx.raw_request.headers)
534
- )
515
+ trace_headers = (
516
+ None
517
+ if ctx.raw_request is None
518
+ else await self._get_trace_headers(ctx.raw_request.headers)
519
+ )
520
536
- pooling_params = self._create_pooling_params(ctx)
537
- if isinstance(pooling_params, ErrorResponse):
538
- return pooling_params
521
+ pooling_params = self._create_pooling_params(ctx)
522
+ if isinstance(pooling_params, ErrorResponse):
523
+ return pooling_params
524
540
- if ctx.engine_prompts is None:
541
- return self.create_error_response("Engine prompts not available")
525
+ if ctx.engine_prompts is None:
526
+ return self.create_error_response("Engine prompts not available")
527
543
- for i, engine_prompt in enumerate(ctx.engine_prompts):
544
- request_id_item = f"{ctx.request_id}-{i}"
528
+ for i, engine_prompt in enumerate(ctx.engine_prompts):
529
+ request_id_item = f"{ctx.request_id}-{i}"
530
546
- self._log_inputs(
547
- request_id_item,
548
- engine_prompt,
549
- params=pooling_params,
550
- lora_request=ctx.lora_request,
551
- )
552
-
553
- generator = self.engine_client.encode(
554
- engine_prompt,
555
- pooling_params,
556
- request_id_item,
557
- lora_request=ctx.lora_request,
558
- trace_headers=trace_headers,
559
- priority=getattr(ctx.request, "priority", 0),
560
- )
531
+ self._log_inputs(
532
+ request_id_item,
533
+ engine_prompt,
534
+ params=pooling_params,
535
+ lora_request=ctx.lora_request,
536
+ )
537
562
- generators.append(generator)
538
+ generator = self.engine_client.encode(
539
+ engine_prompt,
540
+ pooling_params,
541
+ request_id_item,
542
+ lora_request=ctx.lora_request,
543
+ trace_headers=trace_headers,
544
+ priority=getattr(ctx.request, "priority", 0),
545
+ )
546
564
- ctx.result_generator = merge_async_iterators(*generators)
547
+ generators.append(generator)
548
566
- return None
549
+ ctx.result_generator = merge_async_iterators(*generators)
550
568
- except Exception as e:
569
- return self.create_error_response(e)
551
+ return None
552
553
async def _collect_batch(
554
self,
555
ctx: ServeContext,
556
) -> ErrorResponse | None:
557
"""Collect batch results from the result generator."""
576
- try:
577
- if ctx.engine_prompts is None:
578
- return self.create_error_response("Engine prompts not available")
558
+ if ctx.engine_prompts is None:
559
+ return self.create_error_response("Engine prompts not available")
560
580
- num_prompts = len(ctx.engine_prompts)
581
- final_res_batch: list[PoolingRequestOutput | None]
582
- final_res_batch = [None] * num_prompts
561
+ num_prompts = len(ctx.engine_prompts)
562
+ final_res_batch: list[PoolingRequestOutput | None]
563
+ final_res_batch = [None] * num_prompts
564
584
- if ctx.result_generator is None:
585
- return self.create_error_response("Result generator not available")
565
+ if ctx.result_generator is None:
566
+ return self.create_error_response("Result generator not available")
567
587
- async for i, res in ctx.result_generator:
588
- final_res_batch[i] = res
589
-
590
- if None in final_res_batch:
591
- return self.create_error_response(
592
- "Failed to generate results for all prompts"
593
- )
568
+ async for i, res in ctx.result_generator:
569
+ final_res_batch[i] = res
570
595
- ctx.final_res_batch = [res for res in final_res_batch if res is not None]
571
+ if None in final_res_batch:
572
+ return self.create_error_response(
573
+ "Failed to generate results for all prompts"
574
+ )
575
597
- return None
576
+ ctx.final_res_batch = [res for res in final_res_batch if res is not None]
577
599
- except Exception as e:
600
- return self.create_error_response(e)
578
+ return None
579
580
+ @staticmethod
581
def create_error_response(
603
- self,
582
message: str | Exception,
583
err_type: str = "BadRequestError",
584
status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
585
param: str | None = None,
586
) -> ErrorResponse:
609
- exc: Exception | None = None
610
-
611
- if isinstance(message, Exception):
612
- exc = message
613
-
614
- from vllm.exceptions import VLLMValidationError
615
-
616
- if isinstance(exc, VLLMValidationError):
617
- err_type = "BadRequestError"
618
- status_code = HTTPStatus.BAD_REQUEST
619
- param = exc.parameter
620
- elif isinstance(exc, (ValueError, TypeError, RuntimeError, OverflowError)):
621
- # Common validation errors from user input
622
- err_type = "BadRequestError"
623
- status_code = HTTPStatus.BAD_REQUEST
624
- param = None
625
- elif isinstance(exc, NotImplementedError):
626
- err_type = "NotImplementedError"
627
- status_code = HTTPStatus.NOT_IMPLEMENTED
628
- param = None
629
- elif exc.__class__.__name__ == "TemplateError":
630
- # jinja2.TemplateError (avoid importing jinja2)
631
- err_type = "BadRequestError"
632
- status_code = HTTPStatus.BAD_REQUEST
633
- param = None
634
- else:
635
- err_type = "InternalServerError"
636
- status_code = HTTPStatus.INTERNAL_SERVER_ERROR
637
- param = None
638
-
639
- message = str(exc)
640
-
641
- if self.log_error_stack:
642
- exc_type, _, _ = sys.exc_info()
643
- if exc_type is not None:
644
- traceback.print_exc()
645
- else:
646
- traceback.print_stack()
647
-
648
- return ErrorResponse(
649
- error=ErrorInfo(
650
- message=sanitize_message(message),
651
- type=err_type,
652
- code=status_code.value,
653
- param=param,
654
- )
655
- )
587
+ return create_error_response(message, err_type, status_code, param)
588
589
def create_streaming_error_response(
590
self,
@@ -680,16 +612,6 @@ class OpenAIServing:
612
)
613
raise GenerationError("Internal server error")
614
683
- def _convert_generation_error_to_response(
684
- self, e: GenerationError
685
- ) -> ErrorResponse:
686
- """Convert GenerationError to ErrorResponse."""
687
- return self.create_error_response(
688
- str(e),
689
- err_type="InternalServerError",
690
- status_code=e.status_code,
691
- )
692
-
615
def _convert_generation_error_to_streaming_response(
616
self, e: GenerationError
617
) -> str:
vllm/entrypoints/openai/generate/api_router.py
-4
@@ -87,7 +87,6 @@ async def init_generate_state(
87
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
88
enable_force_include_usage=args.enable_force_include_usage,
89
enable_log_outputs=args.enable_log_outputs,
90
- log_error_stack=args.log_error_stack,
90
)
91
if "generate" in supported_tasks
92
else None
@@ -111,7 +110,6 @@ async def init_generate_state(
110
enable_force_include_usage=args.enable_force_include_usage,
111
enable_log_outputs=args.enable_log_outputs,
112
enable_log_deltas=args.enable_log_deltas,
114
- log_error_stack=args.log_error_stack,
113
)
114
if any(task in supported_tasks for task in ("generate", "render"))
115
else None
@@ -127,7 +125,6 @@ async def init_generate_state(
125
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
126
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
127
enable_force_include_usage=args.enable_force_include_usage,
130
- log_error_stack=args.log_error_stack,
128
)
129
if any(task in supported_tasks for task in ("generate", "render"))
130
else None
@@ -156,7 +153,6 @@ async def init_generate_state(
153
state.openai_serving_models,
154
request_logger=request_logger,
155
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
159
- log_error_stack=args.log_error_stack,
156
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
157
enable_log_outputs=args.enable_log_outputs,
158
force_no_detokenize=args.tokens_only,
vllm/entrypoints/openai/realtime/api_router.py
-1
@@ -68,7 +68,6 @@ def init_realtime_state(
68
engine_client,
69
state.openai_serving_models,
70
request_logger=request_logger,
71
- log_error_stack=args.log_error_stack,
71
)
72
if "realtime" in supported_tasks
73
else None
vllm/entrypoints/openai/realtime/serving.py
-2
@@ -33,13 +33,11 @@ class OpenAIServingRealtime(OpenAIServing):
33
models: OpenAIServingModels,
34
*,
35
request_logger: RequestLogger | None,
36
- log_error_stack: bool = False,
36
):
37
super().__init__(
38
engine_client=engine_client,
39
models=models,
40
request_logger=request_logger,
42
- log_error_stack=log_error_stack,
41
)
42
43
self.task_type: Literal["realtime"] = "realtime"
vllm/entrypoints/openai/responses/api_router.py
+8
-16
@@ -63,10 +63,8 @@ async def create_responses(request: ResponsesRequest, raw_request: Request):
63
return base_server.create_error_response(
64
message="The model does not support Responses API"
65
)
66
- try:
67
- generator = await handler.create_responses(request, raw_request)
68
- except Exception as e:
69
- generator = handler.create_error_response(e)
66
+
67
+ generator = await handler.create_responses(request, raw_request)
68
69
if isinstance(generator, ErrorResponse):
70
return JSONResponse(
@@ -95,14 +93,11 @@ async def retrieve_responses(
93
message="The model does not support Responses API"
94
)
95
98
- try:
99
- response = await handler.retrieve_responses(
100
- response_id,
101
- starting_after=starting_after,
102
- stream=stream,
103
- )
104
- except Exception as e:
105
- response = handler.create_error_response(e)
96
+ response = await handler.retrieve_responses(
97
+ response_id,
98
+ starting_after=starting_after,
99
+ stream=stream,
100
+ )
101
102
if isinstance(response, ErrorResponse):
103
return JSONResponse(
@@ -125,10 +120,7 @@ async def cancel_responses(response_id: str, raw_request: Request):
120
message="The model does not support Responses API"
121
)
122
128
- try:
129
- response = await handler.cancel_responses(response_id)
130
- except Exception as e:
131
- response = handler.create_error_response(e)
123
+ response = await handler.cancel_responses(response_id)
124
125
if isinstance(response, ErrorResponse):
126
return JSONResponse(
vllm/entrypoints/openai/responses/serving.py
+92
-129
@@ -11,7 +11,6 @@ from copy import copy
11
from http import HTTPStatus
12
from typing import Final
13
14
-import jinja2
14
from fastapi import Request
15
from openai.types.responses import (
16
ResponseContentPartAddedEvent,
@@ -174,14 +173,12 @@ class OpenAIServingResponses(OpenAIServing):
173
enable_prompt_tokens_details: bool = False,
174
enable_force_include_usage: bool = False,
175
enable_log_outputs: bool = False,
177
- log_error_stack: bool = False,
176
) -> None:
177
super().__init__(
178
engine_client=engine_client,
179
models=models,
180
request_logger=request_logger,
181
return_tokens_as_token_ids=return_tokens_as_token_ids,
184
- log_error_stack=log_error_stack,
182
)
183
184
self.chat_template = chat_template
@@ -365,28 +362,15 @@ class OpenAIServingResponses(OpenAIServing):
362
else:
363
prev_response = None
364
368
- try:
369
- lora_request = self._maybe_get_adapters(request)
370
- model_name = self.models.model_name(lora_request)
371
-
372
- if self.use_harmony:
373
- messages, engine_prompts = self._make_request_with_harmony(
374
- request, prev_response
375
- )
376
- else:
377
- messages, engine_prompts = await self._make_request(
378
- request, prev_response
379
- )
365
+ lora_request = self._maybe_get_adapters(request)
366
+ model_name = self.models.model_name(lora_request)
367
381
- except (
382
- ValueError,
383
- TypeError,
384
- RuntimeError,
385
- jinja2.TemplateError,
386
- NotImplementedError,
387
- ) as e:
388
- logger.exception("Error in preprocessing prompt inputs")
389
- return self.create_error_response(e)
368
+ if self.use_harmony:
369
+ messages, engine_prompts = self._make_request_with_harmony(
370
+ request, prev_response
371
+ )
372
+ else:
373
+ messages, engine_prompts = await self._make_request(request, prev_response)
374
375
request_metadata = RequestResponseMetadata(request_id=request.request_id)
376
if raw_request:
@@ -424,86 +408,83 @@ class OpenAIServingResponses(OpenAIServing):
408
else:
409
assert len(builtin_tool_list) == 0
410
available_tools = []
427
- try:
428
- tokenizer = self.renderer.get_tokenizer()
429
-
430
- for engine_prompt in engine_prompts:
431
- maybe_error = self._validate_generator_input(engine_prompt)
432
- if maybe_error is not None:
433
- return maybe_error
434
-
435
- default_max_tokens = get_max_tokens(
436
- max_model_len,
437
- request.max_output_tokens,
438
- self._extract_prompt_len(engine_prompt),
439
- self.default_sampling_params,
440
- self.override_max_tokens,
441
- )
411
+ tokenizer = self.renderer.get_tokenizer()
412
+
413
+ for engine_prompt in engine_prompts:
414
+ maybe_error = self._validate_generator_input(engine_prompt)
415
+ if maybe_error is not None:
416
+ return maybe_error
417
+
418
+ default_max_tokens = get_max_tokens(
419
+ max_model_len,
420
+ request.max_output_tokens,
421
+ self._extract_prompt_len(engine_prompt),
422
+ self.default_sampling_params,
423
+ self.override_max_tokens,
424
+ )
425
443
- sampling_params = request.to_sampling_params(
444
- default_max_tokens, self.default_sampling_params
445
- )
426
+ sampling_params = request.to_sampling_params(
427
+ default_max_tokens, self.default_sampling_params
428
+ )
429
447
- trace_headers = (
448
- None
449
- if raw_request is None
450
- else await self._get_trace_headers(raw_request.headers)
451
- )
430
+ trace_headers = (
431
+ None
432
+ if raw_request is None
433
+ else await self._get_trace_headers(raw_request.headers)
434
+ )
435
453
- context: ConversationContext
454
- if self.use_harmony:
455
- if request.stream:
456
- context = StreamingHarmonyContext(messages, available_tools)
457
- else:
458
- context = HarmonyContext(messages, available_tools)
436
+ context: ConversationContext
437
+ if self.use_harmony:
438
+ if request.stream:
439
+ context = StreamingHarmonyContext(messages, available_tools)
440
else:
460
- if envs.VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT:
461
- # This is a feature in development for parsing
462
- # tokens during generation instead of at the end
463
- context = ParsableContext(
464
- response_messages=messages,
465
- tokenizer=tokenizer,
466
- reasoning_parser_cls=self.parser.reasoning_parser_cls
467
- if self.parser
468
- else None,
469
- request=request,
470
- tool_parser_cls=self.parser.tool_parser_cls
471
- if self.parser
472
- else None,
473
- available_tools=available_tools,
474
- chat_template=self.chat_template,
475
- chat_template_content_format=self.chat_template_content_format,
476
- )
477
- else:
478
- context = SimpleContext()
479
-
480
- if self.parser and self.parser.reasoning_parser_cls is not None:
481
- reasoning_parser = self.parser.reasoning_parser_cls(tokenizer)
482
- if (
483
- isinstance(
484
- struct_out := sampling_params.structured_outputs,
485
- StructuredOutputsParams,
486
- )
487
- and struct_out.all_non_structural_tag_constraints_none()
488
- ):
489
- sampling_params.structured_outputs = replace(
490
- struct_out,
491
- structural_tag=reasoning_parser.prepare_structured_tag(
492
- struct_out.structural_tag, self.tool_server
493
- ),
494
- )
495
- generator = self._generate_with_builtin_tools(
496
- request_id=request.request_id,
497
- engine_prompt=engine_prompt,
498
- sampling_params=sampling_params,
499
- context=context,
500
- lora_request=lora_request,
501
- priority=request.priority,
502
- trace_headers=trace_headers,
503
- )
504
- generators.append(generator)
505
- except ValueError as e:
506
- return self.create_error_response(e)
441
+ context = HarmonyContext(messages, available_tools)
442
+ else:
443
+ if envs.VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT:
444
+ # This is a feature in development for parsing
445
+ # tokens during generation instead of at the end
446
+ context = ParsableContext(
447
+ response_messages=messages,
448
+ tokenizer=tokenizer,
449
+ reasoning_parser_cls=self.parser.reasoning_parser_cls
450
+ if self.parser
451
+ else None,
452
+ request=request,
453
+ tool_parser_cls=self.parser.tool_parser_cls
454
+ if self.parser
455
+ else None,
456
+ available_tools=available_tools,
457
+ chat_template=self.chat_template,
458
+ chat_template_content_format=self.chat_template_content_format,
459
+ )
460
+ else:
461
+ context = SimpleContext()
462
+
463
+ if self.parser and self.parser.reasoning_parser_cls is not None:
464
+ reasoning_parser = self.parser.reasoning_parser_cls(tokenizer)
465
+ if (
466
+ isinstance(
467
+ struct_out := sampling_params.structured_outputs,
468
+ StructuredOutputsParams,
469
+ )
470
+ and struct_out.all_non_structural_tag_constraints_none()
471
+ ):
472
+ sampling_params.structured_outputs = replace(
473
+ struct_out,
474
+ structural_tag=reasoning_parser.prepare_structured_tag(
475
+ struct_out.structural_tag, self.tool_server
476
+ ),
477
+ )
478
+ generator = self._generate_with_builtin_tools(
479
+ request_id=request.request_id,
480
+ engine_prompt=engine_prompt,
481
+ sampling_params=sampling_params,
482
+ context=context,
483
+ lora_request=lora_request,
484
+ priority=request.priority,
485
+ trace_headers=trace_headers,
486
+ )
487
+ generators.append(generator)
488
489
assert len(generators) == 1
490
(result_generator,) = generators
@@ -578,20 +559,15 @@ class OpenAIServingResponses(OpenAIServing):
559
request_metadata,
560
)
561
581
- try:
582
- return await self.responses_full_generator(
583
- request,
584
- sampling_params,
585
- result_generator,
586
- context,
587
- model_name,
588
- tokenizer,
589
- request_metadata,
590
- )
591
- except GenerationError as e:
592
- return self._convert_generation_error_to_response(e)
593
- except Exception as e:
594
- return self.create_error_response(e)
562
+ return await self.responses_full_generator(
563
+ request,
564
+ sampling_params,
565
+ result_generator,
566
+ context,
567
+ model_name,
568
+ tokenizer,
569
+ request_metadata,
570
+ )
571
572
async def _make_request(
573
self,
@@ -675,8 +651,6 @@ class OpenAIServingResponses(OpenAIServing):
651
pass
652
except asyncio.CancelledError:
653
return self.create_error_response("Client disconnected")
678
- except ValueError as e:
679
- return self.create_error_response(e)
654
655
# NOTE: Implementation of status is still WIP, but for now
656
# we guarantee that if the status is not "completed", it is accurate.
@@ -1129,16 +1103,11 @@ class OpenAIServingResponses(OpenAIServing):
1103
new_event_signal = asyncio.Event()
1104
self.event_store[request.request_id] = (event_deque, new_event_signal)
1105
response = None
1106
+ generator = self.responses_stream_generator(request, *args, **kwargs)
1107
try:
1133
- generator = self.responses_stream_generator(request, *args, **kwargs)
1108
async for event in generator:
1109
event_deque.append(event)
1110
new_event_signal.set() # Signal new event available
1137
- except GenerationError as e:
1138
- response = self._convert_generation_error_to_response(e)
1139
- except Exception as e:
1140
- logger.exception("Background request failed for %s", request.request_id)
1141
- response = self.create_error_response(e)
1111
finally:
1112
new_event_signal.set()
1113
@@ -1157,13 +1126,7 @@ class OpenAIServingResponses(OpenAIServing):
1126
*args,
1127
**kwargs,
1128
):
1160
- try:
1161
- response = await self.responses_full_generator(request, *args, **kwargs)
1162
- except GenerationError as e:
1163
- response = self._convert_generation_error_to_response(e)
1164
- except Exception as e:
1165
- logger.exception("Background request failed for %s", request.request_id)
1166
- response = self.create_error_response(e)
1129
+ response = await self.responses_full_generator(request, *args, **kwargs)
1130
1131
if isinstance(response, ErrorResponse):
1132
# If the request has failed, update the status to "failed".
vllm/entrypoints/openai/server_utils.py
+76
-4
@@ -11,7 +11,7 @@ from contextlib import asynccontextmanager
11
from http import HTTPStatus
12
13
import pydantic
14
-from fastapi import FastAPI, HTTPException, Request
14
+from fastapi import FastAPI, HTTPException, Request, Response
15
from fastapi.exceptions import RequestValidationError
16
from fastapi.responses import JSONResponse
17
from starlette.concurrency import iterate_in_threadpool
@@ -20,11 +20,13 @@ from starlette.types import ASGIApp, Message, Receive, Scope, Send
20
21
from vllm import envs
22
from vllm.engine.protocol import EngineClient
23
+from vllm.entrypoints.launcher import terminate_if_errored
24
from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse
24
-from vllm.entrypoints.utils import sanitize_message
25
+from vllm.entrypoints.utils import create_error_response, sanitize_message
26
from vllm.exceptions import VLLMValidationError
27
from vllm.logger import init_logger
28
from vllm.utils.gc_utils import freeze_gc_heap
29
+from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
30
31
logger = init_logger("vllm.entrypoints.openai.server_utils")
32
@@ -309,7 +311,69 @@ async def log_response(request: Request, call_next):
311
return response
312
313
312
-async def http_exception_handler(_: Request, exc: HTTPException):
314
+async def engine_error_handler(
315
+ req: Request, exc: EngineDeadError | EngineGenerateError
316
+):
317
+ """
318
+ VLLM V1 AsyncLLM catches exceptions and returns
319
+ only two types: EngineGenerateError and EngineDeadError.
320
+
321
+ EngineGenerateError is raised by the per request generate()
322
+ method. This error could be request specific (and therefore
323
+ recoverable - e.g. if there is an error in input processing).
324
+
325
+ EngineDeadError is raised by the background output_handler
326
+ method. This error is global and therefore not recoverable.
327
+
328
+ We register these @app.exception_handlers to return nice
329
+ responses to the end user if they occur and shut down if needed.
330
+ See https://fastapi.tiangolo.com/tutorial/handling-errors/
331
+ for more details on how exception handlers work.
332
+
333
+ If an exception is encountered in a StreamingResponse
334
+ generator, the exception is not raised, since we already sent
335
+ a 200 status. Rather, we send an error message as the next chunk.
336
+ Since the exception is not raised, this means that the server
337
+ will not automatically shut down. Instead, we use the watchdog
338
+ background task for check for errored state.
339
+ """
340
+
341
+ if req.app.state.args.log_error_stack:
342
+ logger.exception(
343
+ "Engine Exception caught. Request id: %s",
344
+ req.state.request_metadata.request_id
345
+ if hasattr(req.state, "request_metadata")
346
+ else None,
347
+ )
348
+
349
+ terminate_if_errored(
350
+ server=req.app.state.server,
351
+ engine=req.app.state.engine_client,
352
+ )
353
+ return Response(status_code=HTTPStatus.INTERNAL_SERVER_ERROR)
354
+
355
+
356
+async def exception_handler(req: Request, exc: Exception):
357
+ if req.app.state.args.log_error_stack:
358
+ logger.exception(
359
+ "Exception caught. Request id: %s",
360
+ req.state.request_metadata.request_id
361
+ if hasattr(req.state, "request_metadata")
362
+ else None,
363
+ )
364
+
365
+ err = create_error_response(exc)
366
+ return JSONResponse(err.model_dump(), status_code=err.error.code)
367
+
368
+
369
+async def http_exception_handler(req: Request, exc: HTTPException):
370
+ if req.app.state.args.log_error_stack:
371
+ logger.exception(
372
+ "HTTPException caught. Request id: %s",
373
+ req.state.request_metadata.request_id
374
+ if hasattr(req.state, "request_metadata")
375
+ else None,
376
+ )
377
err = ErrorResponse(
378
error=ErrorInfo(
379
message=sanitize_message(exc.detail),
@@ -320,7 +384,15 @@ async def http_exception_handler(_: Request, exc: HTTPException):
384
return JSONResponse(err.model_dump(), status_code=exc.status_code)
385
386
323
-async def validation_exception_handler(_: Request, exc: RequestValidationError):
387
+async def validation_exception_handler(req: Request, exc: RequestValidationError):
388
+ if req.app.state.args.log_error_stack:
389
+ logger.exception(
390
+ "RequestValidationError caught. Request id: %s",
391
+ req.state.request_metadata.request_id
392
+ if hasattr(req.state, "request_metadata")
393
+ else None,
394
+ )
395
+
396
param = None
397
errors = exc.errors()
398
for error in errors:
vllm/entrypoints/openai/speech_to_text/api_router.py
+5
-10
@@ -71,10 +71,9 @@ async def create_transcriptions(
71
)
72
73
audio_data = await request.file.read()
74
- try:
75
- generator = await handler.create_transcription(audio_data, request, raw_request)
76
- except Exception as e:
77
- return handler.create_error_response(e)
74
+
75
+ generator = await handler.create_transcription(audio_data, request, raw_request)
76
+
77
if isinstance(generator, ErrorResponse):
78
return JSONResponse(
79
content=generator.model_dump(), status_code=generator.error.code
@@ -108,10 +107,8 @@ async def create_translations(
107
)
108
109
audio_data = await request.file.read()
111
- try:
112
- generator = await handler.create_translation(audio_data, request, raw_request)
113
- except Exception as e:
114
- return handler.create_error_response(e)
110
+
111
+ generator = await handler.create_translation(audio_data, request, raw_request)
112
113
if isinstance(generator, ErrorResponse):
114
return JSONResponse(
@@ -140,7 +137,6 @@ def init_transcription_state(
137
engine_client,
138
state.openai_serving_models,
139
request_logger=request_logger,
143
- log_error_stack=args.log_error_stack,
140
enable_force_include_usage=args.enable_force_include_usage,
141
)
142
if "transcription" in supported_tasks
@@ -151,7 +147,6 @@ def init_transcription_state(
147
engine_client,
148
state.openai_serving_models,
149
request_logger=request_logger,
154
- log_error_stack=args.log_error_stack,
150
enable_force_include_usage=args.enable_force_include_usage,
151
)
152
if "transcription" in supported_tasks
vllm/entrypoints/openai/speech_to_text/serving.py
-4
@@ -40,7 +40,6 @@ class OpenAIServingTranscription(OpenAISpeechToText):
40
*,
41
request_logger: RequestLogger | None,
42
return_tokens_as_token_ids: bool = False,
43
- log_error_stack: bool = False,
43
enable_force_include_usage: bool = False,
44
):
45
super().__init__(
@@ -49,7 +48,6 @@ class OpenAIServingTranscription(OpenAISpeechToText):
48
request_logger=request_logger,
49
return_tokens_as_token_ids=return_tokens_as_token_ids,
50
task_type="transcribe",
52
- log_error_stack=log_error_stack,
51
enable_force_include_usage=enable_force_include_usage,
52
)
53
@@ -113,7 +111,6 @@ class OpenAIServingTranslation(OpenAISpeechToText):
111
*,
112
request_logger: RequestLogger | None,
113
return_tokens_as_token_ids: bool = False,
116
- log_error_stack: bool = False,
114
enable_force_include_usage: bool = False,
115
):
116
super().__init__(
@@ -122,7 +119,6 @@ class OpenAIServingTranslation(OpenAISpeechToText):
119
request_logger=request_logger,
120
return_tokens_as_token_ids=return_tokens_as_token_ids,
121
task_type="translate",
125
- log_error_stack=log_error_stack,
122
enable_force_include_usage=enable_force_include_usage,
123
)
124
vllm/entrypoints/openai/speech_to_text/speech_to_text.py
+45
-57
@@ -97,7 +97,6 @@ class OpenAISpeechToText(OpenAIServing):
97
request_logger: RequestLogger | None,
98
return_tokens_as_token_ids: bool = False,
99
task_type: Literal["transcribe", "translate"] = "transcribe",
100
- log_error_stack: bool = False,
100
enable_force_include_usage: bool = False,
101
):
102
super().__init__(
@@ -105,7 +104,6 @@ class OpenAISpeechToText(OpenAIServing):
104
models=models,
105
request_logger=request_logger,
106
return_tokens_as_token_ids=return_tokens_as_token_ids,
108
- log_error_stack=log_error_stack,
107
)
108
109
self.default_sampling_params = self.model_config.get_diff_sampling_param()
@@ -517,69 +515,61 @@ class OpenAISpeechToText(OpenAIServing):
515
if raw_request:
516
raw_request.state.request_metadata = request_metadata
517
520
- try:
521
- lora_request = self._maybe_get_adapters(request)
522
-
523
- engine_prompts, duration_s = await self._preprocess_speech_to_text(
524
- request=request,
525
- audio_data=audio_data,
526
- request_id=request_id,
527
- )
518
+ lora_request = self._maybe_get_adapters(request)
519
529
- except ValueError as e:
530
- logger.exception("Error in preprocessing prompt inputs")
531
- return self.create_error_response(e)
520
+ engine_prompts, duration_s = await self._preprocess_speech_to_text(
521
+ request=request,
522
+ audio_data=audio_data,
523
+ request_id=request_id,
524
+ )
525
526
# Schedule the request and get the result generator.
527
max_model_len = self.model_config.max_model_len
528
list_result_generator: list[AsyncGenerator[RequestOutput, None]] | None = None
536
- try:
537
- # Unlike most decoder-only models, whisper generation length is not
538
- # constrained by the size of the input audio, which is mapped to a
539
- # fixed-size log-mel-spectogram. Still, allow for fewer tokens to be
540
- # generated by respecting the extra completion tokens arg.
541
- max_tokens = get_max_tokens(
542
- max_model_len,
543
- request.max_completion_tokens,
544
- 0,
545
- self.default_sampling_params,
546
- )
529
+ # Unlike most decoder-only models, whisper generation length is not
530
+ # constrained by the size of the input audio, which is mapped to a
531
+ # fixed-size log-mel-spectogram. Still, allow for fewer tokens to be
532
+ # generated by respecting the extra completion tokens arg.
533
+ max_tokens = get_max_tokens(
534
+ max_model_len,
535
+ request.max_completion_tokens,
536
+ 0,
537
+ self.default_sampling_params,
538
+ )
539
548
- sampling_params = request.to_sampling_params(
549
- max_tokens,
550
- self.default_sampling_params,
540
+ sampling_params = request.to_sampling_params(
541
+ max_tokens,
542
+ self.default_sampling_params,
543
+ )
544
+ if request.response_format == "verbose_json":
545
+ sampling_params.logprobs = 1
546
+
547
+ list_result_generator = []
548
+ for i, engine_prompt in enumerate(engine_prompts):
549
+ request_id_item = f"{request_id}_{i}"
550
+
551
+ self._log_inputs(
552
+ request_id_item,
553
+ engine_prompt,
554
+ params=sampling_params,
555
+ lora_request=lora_request,
556
)
552
- if request.response_format == "verbose_json":
553
- sampling_params.logprobs = 1
554
-
555
- list_result_generator = []
556
- for i, engine_prompt in enumerate(engine_prompts):
557
- request_id_item = f"{request_id}_{i}"
558
-
559
- self._log_inputs(
560
- request_id_item,
561
- engine_prompt,
562
- params=sampling_params,
563
- lora_request=lora_request,
564
- )
557
566
- trace_headers = (
567
- None
568
- if raw_request is None
569
- else await self._get_trace_headers(raw_request.headers)
570
- )
558
+ trace_headers = (
559
+ None
560
+ if raw_request is None
561
+ else await self._get_trace_headers(raw_request.headers)
562
+ )
563
572
- generator = self.engine_client.generate(
573
- engine_prompt,
574
- sampling_params,
575
- request_id_item,
576
- lora_request=lora_request,
577
- trace_headers=trace_headers,
578
- )
564
+ generator = self.engine_client.generate(
565
+ engine_prompt,
566
+ sampling_params,
567
+ request_id_item,
568
+ lora_request=lora_request,
569
+ trace_headers=trace_headers,
570
+ )
571
580
- list_result_generator.append(generator)
581
- except ValueError as e:
582
- return self.create_error_response(e)
572
+ list_result_generator.append(generator)
573
574
if request.stream:
575
return stream_generator_method(
@@ -663,8 +653,6 @@ class OpenAISpeechToText(OpenAIServing):
653
return final_response
654
except asyncio.CancelledError:
655
return self.create_error_response("Client disconnected")
666
- except ValueError as e:
667
- return self.create_error_response(e)
656
657
async def _speech_to_text_stream_generator(
658
self,
vllm/entrypoints/pooling/__init__.py
-4
@@ -72,7 +72,6 @@ def init_pooling_state(
72
chat_template=resolved_chat_template,
73
chat_template_content_format=args.chat_template_content_format,
74
trust_request_chat_template=args.trust_request_chat_template,
75
- log_error_stack=args.log_error_stack,
75
)
76
)
77
if any(t in supported_tasks for t in POOLING_TASKS)
@@ -86,7 +85,6 @@ def init_pooling_state(
85
chat_template=resolved_chat_template,
86
chat_template_content_format=args.chat_template_content_format,
87
trust_request_chat_template=args.trust_request_chat_template,
89
- log_error_stack=args.log_error_stack,
88
)
89
if "embed" in supported_tasks
90
else None
@@ -99,7 +97,6 @@ def init_pooling_state(
97
chat_template=resolved_chat_template,
98
chat_template_content_format=args.chat_template_content_format,
99
trust_request_chat_template=args.trust_request_chat_template,
102
- log_error_stack=args.log_error_stack,
100
)
101
if "classify" in supported_tasks
102
else None
@@ -114,7 +111,6 @@ def init_pooling_state(
111
state.openai_serving_models,
112
request_logger=request_logger,
113
score_template=resolved_chat_template,
117
- log_error_stack=args.log_error_stack,
114
use_gpu_for_pooling_score=getattr(args, "use_gpu_for_pooling_score", False),
115
)
116
if any(t in supported_tasks for t in ("embed", "score", "token_embed"))
vllm/entrypoints/pooling/base/serving.py
+16
-26
@@ -41,7 +41,6 @@ from vllm.tracing import (
41
from vllm.utils import random_uuid
42
from vllm.utils.async_utils import merge_async_iterators
43
44
-from ...utils import create_error_response
44
from .io_processor import PoolingIOProcessor
45
46
PoolingRequestT = TypeVar("PoolingRequestT", bound=AnyPoolingRequest)
@@ -112,34 +111,25 @@ class PoolingServing:
111
request: AnyPoolingRequest,
112
raw_request: Request,
113
) -> JSONResponse:
115
- try:
116
- model_name = self.models.model_name()
117
- request_id = (
118
- f"{self.request_id_prefix}-{self._base_request_id(raw_request)}"
119
- )
114
+ model_name = self.models.model_name()
115
+ request_id = f"{self.request_id_prefix}-{self._base_request_id(raw_request)}"
116
121
- await self._check_model(request)
117
+ await self._check_model(request)
118
123
- ctx = PoolingServeContext(
124
- request=request,
125
- raw_request=raw_request,
126
- model_name=model_name,
127
- request_id=request_id,
128
- )
119
+ ctx = PoolingServeContext(
120
+ request=request,
121
+ raw_request=raw_request,
122
+ model_name=model_name,
123
+ request_id=request_id,
124
+ )
125
130
- self._validate_request(ctx)
131
- self._maybe_get_adapters(ctx)
132
- await self._preprocess(ctx)
133
- await self._prepare_generators(ctx)
134
- await self._collect_batch(ctx)
135
- response = await self._build_response(ctx)
136
- return JSONResponse(content=response.model_dump())
137
- except Exception as e:
138
- error_response = create_error_response(e)
139
- return JSONResponse(
140
- content=error_response.model_dump(),
141
- status_code=error_response.error.code,
142
- )
126
+ self._validate_request(ctx)
127
+ self._maybe_get_adapters(ctx)
128
+ await self._preprocess(ctx)
129
+ await self._prepare_generators(ctx)
130
+ await self._collect_batch(ctx)
131
+ response = await self._build_response(ctx)
132
+ return JSONResponse(content=response.model_dump())
133
134
async def _preprocess(
135
self,
vllm/entrypoints/pooling/embed/api_router.py
+1
-4
@@ -61,10 +61,7 @@ async def create_embedding(
61
message="The model does not support Embeddings API"
62
)
63
64
- try:
65
- generator = await handler.create_embedding(request, raw_request)
66
- except Exception as e:
67
- generator = handler.create_error_response(e)
64
+ generator = await handler.create_embedding(request, raw_request)
65
66
if isinstance(generator, ErrorResponse):
67
return JSONResponse(
vllm/entrypoints/pooling/embed/serving.py
+197
-214
@@ -54,13 +54,11 @@ class OpenAIServingEmbedding(OpenAIServing):
54
chat_template: str | None,
55
chat_template_content_format: ChatTemplateContentFormatOption,
56
trust_request_chat_template: bool = False,
57
- log_error_stack: bool = False,
57
) -> None:
58
super().__init__(
59
engine_client=engine_client,
60
models=models,
61
request_logger=request_logger,
63
- log_error_stack=log_error_stack,
62
)
63
64
self.chat_template = chat_template
@@ -75,38 +73,34 @@ class OpenAIServingEmbedding(OpenAIServing):
73
self,
74
ctx: EmbeddingServeContext,
75
) -> ErrorResponse | None:
78
- try:
79
- ctx.lora_request = self._maybe_get_adapters(ctx.request)
80
-
81
- if isinstance(ctx.request, EmbeddingChatRequest):
82
- error_check_ret = self._validate_chat_template(
83
- request_chat_template=ctx.request.chat_template,
84
- chat_template_kwargs=ctx.request.chat_template_kwargs,
85
- trust_request_chat_template=self.trust_request_chat_template,
86
- )
87
- if error_check_ret is not None:
88
- return error_check_ret
89
-
90
- _, ctx.engine_prompts = await self._preprocess_chat(
91
- ctx.request,
92
- ctx.request.messages,
93
- default_template=self.chat_template,
94
- default_template_content_format=self.chat_template_content_format,
95
- default_template_kwargs=None,
96
- )
97
- elif isinstance(ctx.request, EmbeddingCompletionRequest):
98
- ctx.engine_prompts = await self._preprocess_completion(
99
- ctx.request,
100
- prompt_input=ctx.request.input,
101
- prompt_embeds=None,
102
- )
103
- else:
104
- return self.create_error_response("Invalid classification request type")
76
+ ctx.lora_request = self._maybe_get_adapters(ctx.request)
77
+
78
+ if isinstance(ctx.request, EmbeddingChatRequest):
79
+ error_check_ret = self._validate_chat_template(
80
+ request_chat_template=ctx.request.chat_template,
81
+ chat_template_kwargs=ctx.request.chat_template_kwargs,
82
+ trust_request_chat_template=self.trust_request_chat_template,
83
+ )
84
+ if error_check_ret is not None:
85
+ return error_check_ret
86
+
87
+ _, ctx.engine_prompts = await self._preprocess_chat(
88
+ ctx.request,
89
+ ctx.request.messages,
90
+ default_template=self.chat_template,
91
+ default_template_content_format=self.chat_template_content_format,
92
+ default_template_kwargs=None,
93
+ )
94
+ elif isinstance(ctx.request, EmbeddingCompletionRequest):
95
+ ctx.engine_prompts = await self._preprocess_completion(
96
+ ctx.request,
97
+ prompt_input=ctx.request.input,
98
+ prompt_embeds=None,
99
+ )
100
+ else:
101
+ return self.create_error_response("Invalid classification request type")
102
106
- return None
107
- except (ValueError, TypeError) as e:
108
- logger.exception("Error in preprocessing prompt inputs")
109
- return self.create_error_response(str(e))
103
+ return None
104
105
def request_output_to_embed_json_response(
106
self,
@@ -397,51 +391,47 @@ class OpenAIServingEmbedding(OpenAIServing):
391
# Custom logic for chunked processing
392
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
393
400
- try:
401
- trace_headers = (
402
- None
403
- if ctx.raw_request is None
404
- else await self._get_trace_headers(ctx.raw_request.headers)
405
- )
406
-
407
- pooling_params = self._create_pooling_params(ctx)
408
- if isinstance(pooling_params, ErrorResponse):
409
- return pooling_params
394
+ trace_headers = (
395
+ None
396
+ if ctx.raw_request is None
397
+ else await self._get_trace_headers(ctx.raw_request.headers)
398
+ )
399
411
- if ctx.engine_prompts is None:
412
- return self.create_error_response("Engine prompts not available")
400
+ pooling_params = self._create_pooling_params(ctx)
401
+ if isinstance(pooling_params, ErrorResponse):
402
+ return pooling_params
403
414
- max_pos_embeddings = self._get_max_position_embeddings()
404
+ if ctx.engine_prompts is None:
405
+ return self.create_error_response("Engine prompts not available")
406
416
- for i, engine_prompt in enumerate(ctx.engine_prompts):
417
- # Check if this specific prompt needs chunked processing
418
- if "prompt_token_ids" in engine_prompt:
419
- prompt_token_ids = engine_prompt["prompt_token_ids"] # type: ignore[typeddict-item]
420
-
421
- if len(prompt_token_ids) > max_pos_embeddings:
422
- # Use chunked processing for this prompt
423
- chunk_generators = await self._process_chunked_request(
424
- ctx,
425
- prompt_token_ids,
426
- pooling_params,
427
- trace_headers,
428
- i,
429
- )
430
- generators.extend(chunk_generators)
431
- continue
407
+ max_pos_embeddings = self._get_max_position_embeddings()
408
433
- # Normal processing for short prompts or non-token prompts
434
- generator = await self._create_single_prompt_generator(
435
- ctx, engine_prompt, pooling_params, trace_headers, i
436
- )
437
- generators.append(generator)
409
+ for i, engine_prompt in enumerate(ctx.engine_prompts):
410
+ # Check if this specific prompt needs chunked processing
411
+ if "prompt_token_ids" in engine_prompt:
412
+ prompt_token_ids = engine_prompt["prompt_token_ids"] # type: ignore[typeddict-item]
413
+
414
+ if len(prompt_token_ids) > max_pos_embeddings:
415
+ # Use chunked processing for this prompt
416
+ chunk_generators = await self._process_chunked_request(
417
+ ctx,
418
+ prompt_token_ids,
419
+ pooling_params,
420
+ trace_headers,
421
+ i,
422
+ )
423
+ generators.extend(chunk_generators)
424
+ continue
425
439
- ctx.result_generator = merge_async_iterators(*generators)
426
+ # Normal processing for short prompts or non-token prompts
427
+ generator = await self._create_single_prompt_generator(
428
+ ctx, engine_prompt, pooling_params, trace_headers, i
429
+ )
430
+ generators.append(generator)
431
441
- return None
432
+ ctx.result_generator = merge_async_iterators(*generators)
433
443
- except Exception as e:
444
- return self.create_error_response(e)
434
+ return None
435
436
async def _collect_batch(
437
self,
@@ -454,164 +444,157 @@ class OpenAIServingEmbedding(OpenAIServing):
444
minimize memory usage.
445
For regular requests, collects results normally.
446
"""
457
- try:
458
- if ctx.engine_prompts is None:
459
- return self.create_error_response("Engine prompts not available")
460
-
461
- # Check if we used chunked processing
462
- use_chunked = self._should_use_chunked_processing(ctx.request)
463
-
464
- if not use_chunked:
465
- return await super()._collect_batch(ctx=ctx)
466
-
467
- if ctx.result_generator is None:
468
- return self.create_error_response("Result generator not available")
469
-
470
- # Online aggregation for chunked requests to
471
- # minimize memory usage
472
- # Track aggregation state for each prompt
473
- prompt_aggregators: dict[int, dict[str, Any]] = {}
474
- short_prompts_results: dict[int, PoolingRequestOutput] = {}
475
-
476
- async for result_idx, result in ctx.result_generator:
477
- if "-chunk-" in result.request_id:
478
- # Extract prompt_idx from chunked request_id
479
- parts = result.request_id.split("-")
480
- try:
481
- prompt_idx = int(parts[parts.index("prompt") + 1])
482
- except (ValueError, IndexError):
483
- # Fallback: extract from result_idx if parsing fails
484
- prompt_idx = result_idx
485
-
486
- # Initialize aggregator for this prompt if needed
487
- if prompt_idx not in prompt_aggregators:
488
- prompt_aggregators[prompt_idx] = {
489
- "weighted_sum": None,
490
- "total_weight": 0,
491
- "chunk_count": 0,
492
- "request_id": result.request_id.split("-chunk-")[0],
493
- }
494
-
495
- aggregator = prompt_aggregators[prompt_idx]
496
-
497
- # MEAN pooling with online weighted averaging
498
- # Ensure result is PoolingRequestOutput
499
- # for embedding processing
500
- if not isinstance(result, PoolingRequestOutput):
501
- return self.create_error_response(
502
- f"Expected PoolingRequestOutput for "
503
- f"chunked embedding, got "
504
- f"{type(result).__name__}"
505
- )
447
+ if ctx.engine_prompts is None:
448
+ return self.create_error_response("Engine prompts not available")
449
507
- # Handle both PoolingOutput and
508
- # EmbeddingOutput types
509
- if hasattr(result.outputs, "data"):
510
- # PoolingOutput case
511
- embedding_data = result.outputs.data
512
- elif hasattr(result.outputs, "embedding"):
513
- # EmbeddingOutput case -
514
- # convert embedding list to tensor
515
- embedding_data = result.outputs.embedding
516
- else:
517
- return self.create_error_response(
518
- f"Unsupported output type: {type(result.outputs).__name__}"
519
- )
450
+ # Check if we used chunked processing
451
+ use_chunked = self._should_use_chunked_processing(ctx.request)
452
521
- if not isinstance(embedding_data, torch.Tensor):
522
- embedding_data = torch.tensor(
523
- embedding_data, dtype=torch.float32
524
- )
453
+ if not use_chunked:
454
+ return await super()._collect_batch(ctx=ctx)
455
+
456
+ if ctx.result_generator is None:
457
+ return self.create_error_response("Result generator not available")
458
+
459
+ # Online aggregation for chunked requests to
460
+ # minimize memory usage
461
+ # Track aggregation state for each prompt
462
+ prompt_aggregators: dict[int, dict[str, Any]] = {}
463
+ short_prompts_results: dict[int, PoolingRequestOutput] = {}
464
+
465
+ async for result_idx, result in ctx.result_generator:
466
+ if "-chunk-" in result.request_id:
467
+ # Extract prompt_idx from chunked request_id
468
+ parts = result.request_id.split("-")
469
+ try:
470
+ prompt_idx = int(parts[parts.index("prompt") + 1])
471
+ except (ValueError, IndexError):
472
+ # Fallback: extract from result_idx if parsing fails
473
+ prompt_idx = result_idx
474
+
475
+ # Initialize aggregator for this prompt if needed
476
+ if prompt_idx not in prompt_aggregators:
477
+ prompt_aggregators[prompt_idx] = {
478
+ "weighted_sum": None,
479
+ "total_weight": 0,
480
+ "chunk_count": 0,
481
+ "request_id": result.request_id.split("-chunk-")[0],
482
+ }
483
526
- if result.prompt_token_ids is None:
527
- return self.create_error_response(
528
- "prompt_token_ids cannot be None for chunked processing"
529
- )
530
- weight = len(result.prompt_token_ids)
484
+ aggregator = prompt_aggregators[prompt_idx]
485
+
486
+ # MEAN pooling with online weighted averaging
487
+ # Ensure result is PoolingRequestOutput
488
+ # for embedding processing
489
+ if not isinstance(result, PoolingRequestOutput):
490
+ return self.create_error_response(
491
+ f"Expected PoolingRequestOutput for "
492
+ f"chunked embedding, got "
493
+ f"{type(result).__name__}"
494
+ )
495
532
- weighted_embedding = embedding_data.to(dtype=torch.float32) * weight
496
+ # Handle both PoolingOutput and
497
+ # EmbeddingOutput types
498
+ if hasattr(result.outputs, "data"):
499
+ # PoolingOutput case
500
+ embedding_data = result.outputs.data
501
+ elif hasattr(result.outputs, "embedding"):
502
+ # EmbeddingOutput case -
503
+ # convert embedding list to tensor
504
+ embedding_data = result.outputs.embedding
505
+ else:
506
+ return self.create_error_response(
507
+ f"Unsupported output type: {type(result.outputs).__name__}"
508
+ )
509
534
- if aggregator["weighted_sum"] is None:
535
- # First chunk
536
- aggregator["weighted_sum"] = weighted_embedding
537
- else:
538
- # Accumulate
539
- aggregator["weighted_sum"] += weighted_embedding
510
+ if not isinstance(embedding_data, torch.Tensor):
511
+ embedding_data = torch.tensor(embedding_data, dtype=torch.float32)
512
541
- aggregator["total_weight"] += weight
542
- aggregator["chunk_count"] += 1
513
+ if result.prompt_token_ids is None:
514
+ return self.create_error_response(
515
+ "prompt_token_ids cannot be None for chunked processing"
516
+ )
517
+ weight = len(result.prompt_token_ids)
518
+
519
+ weighted_embedding = embedding_data.to(dtype=torch.float32) * weight
520
+
521
+ if aggregator["weighted_sum"] is None:
522
+ # First chunk
523
+ aggregator["weighted_sum"] = weighted_embedding
524
else:
544
- # Non-chunked result - extract prompt_idx from request_id
545
- parts = result.request_id.split("-")
546
- try:
547
- # Last part should be prompt index
548
- prompt_idx = int(parts[-1])
549
- except (ValueError, IndexError):
550
- prompt_idx = result_idx # Fallback to result_idx
551
-
552
- short_prompts_results[prompt_idx] = result
553
-
554
- # Finalize aggregated results
555
- final_res_batch: list[PoolingRequestOutput] = []
556
- num_prompts = len(ctx.engine_prompts)
557
-
558
- for prompt_idx in range(num_prompts):
559
- if prompt_idx in prompt_aggregators:
560
- # Finalize MEAN aggregation for this chunked prompt
561
- aggregator = prompt_aggregators[prompt_idx]
562
-
563
- weighted_sum = aggregator["weighted_sum"]
564
- total_weight = aggregator["total_weight"]
565
-
566
- if (
567
- weighted_sum is not None
568
- and isinstance(weighted_sum, torch.Tensor)
569
- and isinstance(total_weight, (int, float))
570
- and total_weight > 0
571
- ):
572
- # Compute final mean embedding
573
- final_embedding = weighted_sum / total_weight
574
-
575
- # Create a PoolingRequestOutput
576
- # for the aggregated result
577
- pooling_output_data = PoolingOutput(data=final_embedding)
578
-
579
- # Get original prompt token IDs for this prompt
580
- original_prompt = ctx.engine_prompts[prompt_idx]
581
- if "prompt_token_ids" not in original_prompt:
582
- return self.create_error_response(
583
- f"Chunked prompt {prompt_idx} does not contain "
584
- "token IDs"
585
- )
586
-
587
- original_token_ids = original_prompt["prompt_token_ids"] # type: ignore[typeddict-item]
588
-
589
- pooling_request_output = PoolingRequestOutput(
590
- request_id=aggregator["request_id"],
591
- prompt_token_ids=original_token_ids,
592
- outputs=pooling_output_data,
593
- num_cached_tokens=0,
594
- finished=True,
595
- )
525
+ # Accumulate
526
+ aggregator["weighted_sum"] += weighted_embedding
527
597
- final_res_batch.append(pooling_request_output)
598
- else:
528
+ aggregator["total_weight"] += weight
529
+ aggregator["chunk_count"] += 1
530
+ else:
531
+ # Non-chunked result - extract prompt_idx from request_id
532
+ parts = result.request_id.split("-")
533
+ try:
534
+ # Last part should be prompt index
535
+ prompt_idx = int(parts[-1])
536
+ except (ValueError, IndexError):
537
+ prompt_idx = result_idx # Fallback to result_idx
538
+
539
+ short_prompts_results[prompt_idx] = result
540
+
541
+ # Finalize aggregated results
542
+ final_res_batch: list[PoolingRequestOutput] = []
543
+ num_prompts = len(ctx.engine_prompts)
544
+
545
+ for prompt_idx in range(num_prompts):
546
+ if prompt_idx in prompt_aggregators:
547
+ # Finalize MEAN aggregation for this chunked prompt
548
+ aggregator = prompt_aggregators[prompt_idx]
549
+
550
+ weighted_sum = aggregator["weighted_sum"]
551
+ total_weight = aggregator["total_weight"]
552
+
553
+ if (
554
+ weighted_sum is not None
555
+ and isinstance(weighted_sum, torch.Tensor)
556
+ and isinstance(total_weight, (int, float))
557
+ and total_weight > 0
558
+ ):
559
+ # Compute final mean embedding
560
+ final_embedding = weighted_sum / total_weight
561
+
562
+ # Create a PoolingRequestOutput
563
+ # for the aggregated result
564
+ pooling_output_data = PoolingOutput(data=final_embedding)
565
+
566
+ # Get original prompt token IDs for this prompt
567
+ original_prompt = ctx.engine_prompts[prompt_idx]
568
+ if "prompt_token_ids" not in original_prompt:
569
return self.create_error_response(
600
- f"Failed to aggregate chunks for prompt {prompt_idx}"
570
+ f"Chunked prompt {prompt_idx} does not contain token IDs"
571
)
602
- elif prompt_idx in short_prompts_results:
603
- final_res_batch.append(short_prompts_results[prompt_idx])
572
+
573
+ original_token_ids = original_prompt["prompt_token_ids"] # type: ignore[typeddict-item]
574
+
575
+ pooling_request_output = PoolingRequestOutput(
576
+ request_id=aggregator["request_id"],
577
+ prompt_token_ids=original_token_ids,
578
+ outputs=pooling_output_data,
579
+ num_cached_tokens=0,
580
+ finished=True,
581
+ )
582
+
583
+ final_res_batch.append(pooling_request_output)
584
else:
585
return self.create_error_response(
606
- f"Result not found for prompt {prompt_idx}"
586
+ f"Failed to aggregate chunks for prompt {prompt_idx}"
587
)
588
+ elif prompt_idx in short_prompts_results:
589
+ final_res_batch.append(short_prompts_results[prompt_idx])
590
+ else:
591
+ return self.create_error_response(
592
+ f"Result not found for prompt {prompt_idx}"
593
+ )
594
609
- ctx.final_res_batch = final_res_batch
610
-
611
- return None
595
+ ctx.final_res_batch = final_res_batch
596
613
- except Exception as e:
614
- return self.create_error_response(e)
597
+ return None
598
599
async def create_embedding(
600
self,
vllm/entrypoints/pooling/pooling/api_router.py
+2
-4
@@ -41,10 +41,8 @@ async def create_pooling(request: PoolingRequest, raw_request: Request):
41
return base_server.create_error_response(
42
message="The model does not support Pooling API"
43
)
44
- try:
45
- generator = await handler.create_pooling(request, raw_request)
46
- except Exception as e:
47
- generator = handler.create_error_response(e)
44
+
45
+ generator = await handler.create_pooling(request, raw_request)
46
47
if isinstance(generator, ErrorResponse):
48
return JSONResponse(
vllm/entrypoints/pooling/pooling/serving.py
+76
-90
@@ -8,7 +8,6 @@ from collections.abc import AsyncGenerator, Callable, Sequence
8
from functools import partial
9
from typing import Final, Literal, cast
10
11
-import jinja2
11
from fastapi import Request
12
from typing_extensions import assert_never
13
@@ -53,13 +52,11 @@ class OpenAIServingPooling(OpenAIServing):
52
chat_template: str | None,
53
chat_template_content_format: ChatTemplateContentFormatOption,
54
trust_request_chat_template: bool = False,
56
- log_error_stack: bool = False,
55
) -> None:
56
super().__init__(
57
engine_client=engine_client,
58
models=models,
59
request_logger=request_logger,
62
- log_error_stack=log_error_stack,
60
)
61
62
self.chat_template = chat_template
@@ -84,101 +81,92 @@ class OpenAIServingPooling(OpenAIServing):
81
request_id = f"pool-{self._base_request_id(raw_request)}"
82
created_time = int(time.time())
83
87
- try:
88
- lora_request = self._maybe_get_adapters(request)
84
+ lora_request = self._maybe_get_adapters(request)
85
90
- if getattr(request, "dimensions", None) is not None:
91
- return self.create_error_response(
92
- "dimensions is currently not supported"
93
- )
86
+ if getattr(request, "dimensions", None) is not None:
87
+ return self.create_error_response("dimensions is currently not supported")
88
95
- engine_prompts: Sequence[ProcessorInputs]
96
- if use_io_processor := isinstance(request, IOProcessorRequest):
97
- if self.io_processor is None:
98
- raise ValueError(
99
- "No IOProcessor plugin installed. Please refer "
100
- "to the documentation and to the "
101
- "'prithvi_geospatial_mae_io_processor' "
102
- "offline inference example for more details."
103
- )
89
+ engine_prompts: Sequence[ProcessorInputs]
90
+ if use_io_processor := isinstance(request, IOProcessorRequest):
91
+ if self.io_processor is None:
92
+ raise ValueError(
93
+ "No IOProcessor plugin installed. Please refer "
94
+ "to the documentation and to the "
95
+ "'prithvi_geospatial_mae_io_processor' "
96
+ "offline inference example for more details."
97
+ )
98
105
- validated_prompt = self.io_processor.parse_data(request.data)
99
+ validated_prompt = self.io_processor.parse_data(request.data)
100
107
- raw_prompts = await self.io_processor.pre_process_async(
108
- prompt=validated_prompt, request_id=request_id
109
- )
110
- engine_prompts = await self._preprocess_cmpl(
111
- request,
112
- prompt_to_seq(raw_prompts),
113
- )
114
- elif isinstance(request, PoolingChatRequest):
115
- error_check_ret = self._validate_chat_template(
116
- request_chat_template=request.chat_template,
117
- chat_template_kwargs=request.chat_template_kwargs,
118
- trust_request_chat_template=self.trust_request_chat_template,
119
- )
120
- if error_check_ret is not None:
121
- return error_check_ret
122
-
123
- _, engine_prompts = await self._preprocess_chat(
124
- request,
125
- request.messages,
126
- default_template=self.chat_template,
127
- default_template_content_format=self.chat_template_content_format,
128
- default_template_kwargs=None,
129
- )
130
- elif isinstance(request, PoolingCompletionRequest):
131
- engine_prompts = await self._preprocess_completion(
132
- request,
133
- prompt_input=request.input,
134
- prompt_embeds=None,
135
- )
136
- else:
137
- raise ValueError(f"Unsupported request of type {type(request)}")
138
- except (ValueError, TypeError, jinja2.TemplateError) as e:
139
- logger.exception("Error in preprocessing prompt inputs")
140
- return self.create_error_response(str(e))
101
+ raw_prompts = await self.io_processor.pre_process_async(
102
+ prompt=validated_prompt, request_id=request_id
103
+ )
104
+ engine_prompts = await self._preprocess_cmpl(
105
+ request,
106
+ prompt_to_seq(raw_prompts),
107
+ )
108
+ elif isinstance(request, PoolingChatRequest):
109
+ error_check_ret = self._validate_chat_template(
110
+ request_chat_template=request.chat_template,
111
+ chat_template_kwargs=request.chat_template_kwargs,
112
+ trust_request_chat_template=self.trust_request_chat_template,
113
+ )
114
+ if error_check_ret is not None:
115
+ return error_check_ret
116
+
117
+ _, engine_prompts = await self._preprocess_chat(
118
+ request,
119
+ request.messages,
120
+ default_template=self.chat_template,
121
+ default_template_content_format=self.chat_template_content_format,
122
+ default_template_kwargs=None,
123
+ )
124
+ elif isinstance(request, PoolingCompletionRequest):
125
+ engine_prompts = await self._preprocess_completion(
126
+ request,
127
+ prompt_input=request.input,
128
+ prompt_embeds=None,
129
+ )
130
+ else:
131
+ raise ValueError(f"Unsupported request of type {type(request)}")
132
133
# Schedule the request and get the result generator.
134
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
144
- try:
145
- if use_io_processor:
146
- assert self.io_processor is not None
147
-
148
- pooling_params = self.io_processor.merge_pooling_params()
149
- if pooling_params.task is None:
150
- pooling_params.task = "plugin"
151
- else:
152
- pooling_params = request.to_pooling_params() # type: ignore
153
-
154
- for i, engine_prompt in enumerate(engine_prompts):
155
- request_id_item = f"{request_id}-{i}"
156
-
157
- self._log_inputs(
158
- request_id_item,
159
- engine_prompt,
160
- params=pooling_params,
161
- lora_request=lora_request,
162
- )
135
+ if use_io_processor:
136
+ assert self.io_processor is not None
137
164
- trace_headers = (
165
- None
166
- if raw_request is None
167
- else await self._get_trace_headers(raw_request.headers)
168
- )
138
+ pooling_params = self.io_processor.merge_pooling_params()
139
+ if pooling_params.task is None:
140
+ pooling_params.task = "plugin"
141
+ else:
142
+ pooling_params = request.to_pooling_params() # type: ignore
143
170
- generator = self.engine_client.encode(
171
- engine_prompt,
172
- pooling_params,
173
- request_id_item,
174
- lora_request=lora_request,
175
- trace_headers=trace_headers,
176
- priority=request.priority,
177
- )
144
+ for i, engine_prompt in enumerate(engine_prompts):
145
+ request_id_item = f"{request_id}-{i}"
146
+
147
+ self._log_inputs(
148
+ request_id_item,
149
+ engine_prompt,
150
+ params=pooling_params,
151
+ lora_request=lora_request,
152
+ )
153
+
154
+ trace_headers = (
155
+ None
156
+ if raw_request is None
157
+ else await self._get_trace_headers(raw_request.headers)
158
+ )
159
+
160
+ generator = self.engine_client.encode(
161
+ engine_prompt,
162
+ pooling_params,
163
+ request_id_item,
164
+ lora_request=lora_request,
165
+ trace_headers=trace_headers,
166
+ priority=request.priority,
167
+ )
168
179
- generators.append(generator)
180
- except ValueError as e:
181
- return self.create_error_response(e)
169
+ generators.append(generator)
170
171
result_generator = merge_async_iterators(*generators)
172
@@ -233,8 +221,6 @@ class OpenAIServingPooling(OpenAIServing):
221
)
222
except asyncio.CancelledError:
223
return self.create_error_response("Client disconnected")
236
- except ValueError as e:
237
- return self.create_error_response(e)
224
225
return response
226
vllm/entrypoints/pooling/score/api_router.py
+3
-8
@@ -49,10 +49,7 @@ async def create_score(request: ScoreRequest, raw_request: Request):
49
message="The model does not support Score API"
50
)
51
52
- try:
53
- generator = await handler.create_score(request, raw_request)
54
- except Exception as e:
55
- generator = handler.create_error_response(e)
52
+ generator = await handler.create_score(request, raw_request)
53
54
if isinstance(generator, ErrorResponse):
55
return JSONResponse(
@@ -100,10 +97,8 @@ async def do_rerank(request: RerankRequest, raw_request: Request):
97
return base_server.create_error_response(
98
message="The model does not support Rerank (Score) API"
99
)
103
- try:
104
- generator = await handler.do_rerank(request, raw_request)
105
- except Exception as e:
106
- generator = handler.create_error_response(e)
100
+
101
+ generator = await handler.do_rerank(request, raw_request)
102
103
if isinstance(generator, ErrorResponse):
104
return JSONResponse(
vllm/entrypoints/pooling/score/serving.py
-5
@@ -62,7 +62,6 @@ class ServingScores(OpenAIServing):
62
engine_client=engine_client,
63
models=models,
64
request_logger=request_logger,
65
- log_error_stack=log_error_stack,
65
)
66
self.score_template = score_template
67
self.use_gpu_for_pooling_score = use_gpu_for_pooling_score
@@ -518,8 +517,6 @@ class ServingScores(OpenAIServing):
517
)
518
except asyncio.CancelledError:
519
return self.create_error_response("Client disconnected")
521
- except ValueError as e:
522
- return self.create_error_response(e)
520
521
async def do_rerank(
522
self, request: RerankRequest, raw_request: Request | None = None
@@ -562,8 +559,6 @@ class ServingScores(OpenAIServing):
559
)
560
except asyncio.CancelledError:
561
return self.create_error_response("Client disconnected")
565
- except ValueError as e:
566
- return self.create_error_response(e)
562
563
def request_output_to_score_response(
564
self,
vllm/entrypoints/serve/disagg/api_router.py
+2
-4
@@ -64,10 +64,8 @@ async def generate(request: GenerateRequest, raw_request: Request):
64
return tokenization(raw_request).create_error_response(
65
message="The model does not support generate tokens API"
66
)
67
- try:
68
- generator = await handler.serve_tokens(request, raw_request)
69
- except Exception as e:
70
- generator = handler.create_error_response(e)
67
+
68
+ generator = await handler.serve_tokens(request, raw_request)
69
70
if isinstance(generator, ErrorResponse):
71
return JSONResponse(
vllm/entrypoints/serve/disagg/serving.py
+27
-38
@@ -49,7 +49,6 @@ class ServingTokens(OpenAIServing):
49
request_logger: RequestLogger | None,
50
force_no_detokenize: bool = False,
51
return_tokens_as_token_ids: bool = False,
52
- log_error_stack: bool = False,
52
enable_prompt_tokens_details: bool = False,
53
enable_log_outputs: bool = False,
54
):
@@ -58,7 +57,6 @@ class ServingTokens(OpenAIServing):
57
models=models,
58
request_logger=request_logger,
59
return_tokens_as_token_ids=return_tokens_as_token_ids,
61
- log_error_stack=log_error_stack,
60
)
61
self.enable_prompt_tokens_details = enable_prompt_tokens_details
62
self.enable_log_outputs = enable_log_outputs
@@ -108,45 +106,38 @@ class ServingTokens(OpenAIServing):
106
107
# Schedule the request and get the result generator.
108
result_generator: AsyncGenerator[RequestOutput, None] | None = None
111
- try:
112
- sampling_params = request.sampling_params
113
- if self.force_no_detokenize:
114
- sampling_params.detokenize = False
115
-
116
- self._log_inputs(
117
- request_id,
118
- engine_prompt,
119
- params=sampling_params,
120
- lora_request=lora_request,
121
- )
122
-
123
- trace_headers = (
124
- None
125
- if raw_request is None
126
- else await self._get_trace_headers(raw_request.headers)
127
- )
109
+ sampling_params = request.sampling_params
110
+ if self.force_no_detokenize:
111
+ sampling_params.detokenize = False
112
+
113
+ self._log_inputs(
114
+ request_id,
115
+ engine_prompt,
116
+ params=sampling_params,
117
+ lora_request=lora_request,
118
+ )
119
129
- result_generator = self.engine_client.generate(
130
- engine_prompt,
131
- sampling_params,
132
- request_id,
133
- lora_request=lora_request,
134
- trace_headers=trace_headers,
135
- priority=request.priority,
136
- )
120
+ trace_headers = (
121
+ None
122
+ if raw_request is None
123
+ else await self._get_trace_headers(raw_request.headers)
124
+ )
125
138
- except ValueError as e:
139
- return self.create_error_response(str(e))
126
+ result_generator = self.engine_client.generate(
127
+ engine_prompt,
128
+ sampling_params,
129
+ request_id,
130
+ lora_request=lora_request,
131
+ trace_headers=trace_headers,
132
+ priority=request.priority,
133
+ )
134
135
# TODO(NickLucche): Implement streaming response
136
143
- try:
144
- assert result_generator is not None
145
- return await self.serve_tokens_full_generator(
146
- request, result_generator, request_id, model_name, request_metadata
147
- )
148
- except ValueError as e:
149
- return self.create_error_response(str(e))
137
+ assert result_generator is not None
138
+ return await self.serve_tokens_full_generator(
139
+ request, result_generator, request_id, model_name, request_metadata
140
+ )
141
142
async def serve_tokens_full_generator(
143
self,
@@ -165,8 +156,6 @@ class ServingTokens(OpenAIServing):
156
final_res = res
157
except asyncio.CancelledError:
158
return self.create_error_response("Client disconnected")
168
- except ValueError as e:
169
- return self.create_error_response(str(e))
159
160
assert final_res is not None
161
vllm/entrypoints/serve/tokenize/api_router.py
+1
-4
@@ -49,10 +49,7 @@ router = APIRouter()
49
async def tokenize(request: TokenizeRequest, raw_request: Request):
50
handler = tokenization(raw_request)
51
52
- try:
53
- generator = await handler.create_tokenize(request, raw_request)
54
- except Exception as e:
55
- generator = handler.create_error_response(e)
52
+ generator = await handler.create_tokenize(request, raw_request)
53
54
if isinstance(generator, ErrorResponse):
55
return JSONResponse(
vllm/entrypoints/serve/tokenize/serving.py
+33
-43
@@ -3,7 +3,6 @@
3
from dataclasses import dataclass
4
from typing import Any, Final
5
6
-import jinja2
6
from fastapi import Request
7
8
from vllm.engine.protocol import EngineClient
@@ -37,13 +36,11 @@ class OpenAIServingTokenization(OpenAIServing):
36
chat_template: str | None,
37
chat_template_content_format: ChatTemplateContentFormatOption,
38
trust_request_chat_template: bool = False,
40
- log_error_stack: bool = False,
39
) -> None:
40
super().__init__(
41
engine_client=engine_client,
42
models=models,
43
request_logger=request_logger,
46
- log_error_stack=log_error_stack,
44
)
45
46
self.chat_template = chat_template
@@ -61,40 +58,36 @@ class OpenAIServingTokenization(OpenAIServing):
58
59
request_id = f"tokenize-{self._base_request_id(raw_request)}"
60
64
- try:
65
- lora_request = self._maybe_get_adapters(request)
66
-
67
- if isinstance(request, TokenizeChatRequest):
68
- tool_dicts = (
69
- None
70
- if request.tools is None
71
- else [tool.model_dump() for tool in request.tools]
72
- )
73
- error_check_ret = self._validate_chat_template(
74
- request_chat_template=request.chat_template,
75
- chat_template_kwargs=request.chat_template_kwargs,
76
- trust_request_chat_template=self.trust_request_chat_template,
77
- )
78
- if error_check_ret is not None:
79
- return error_check_ret
80
-
81
- _, engine_prompts = await self._preprocess_chat(
82
- request,
83
- request.messages,
84
- default_template=self.chat_template,
85
- default_template_content_format=self.chat_template_content_format,
86
- default_template_kwargs=None,
87
- tool_dicts=tool_dicts,
88
- )
89
- else:
90
- engine_prompts = await self._preprocess_completion(
91
- request,
92
- prompt_input=request.prompt,
93
- prompt_embeds=None,
94
- )
95
- except (ValueError, TypeError, jinja2.TemplateError) as e:
96
- logger.exception("Error in preprocessing prompt inputs")
97
- return self.create_error_response(f"{e} {e.__cause__}")
61
+ lora_request = self._maybe_get_adapters(request)
62
+
63
+ if isinstance(request, TokenizeChatRequest):
64
+ tool_dicts = (
65
+ None
66
+ if request.tools is None
67
+ else [tool.model_dump() for tool in request.tools]
68
+ )
69
+ error_check_ret = self._validate_chat_template(
70
+ request_chat_template=request.chat_template,
71
+ chat_template_kwargs=request.chat_template_kwargs,
72
+ trust_request_chat_template=self.trust_request_chat_template,
73
+ )
74
+ if error_check_ret is not None:
75
+ return error_check_ret
76
+
77
+ _, engine_prompts = await self._preprocess_chat(
78
+ request,
79
+ request.messages,
80
+ default_template=self.chat_template,
81
+ default_template_content_format=self.chat_template_content_format,
82
+ default_template_kwargs=None,
83
+ tool_dicts=tool_dicts,
84
+ )
85
+ else:
86
+ engine_prompts = await self._preprocess_completion(
87
+ request,
88
+ prompt_input=request.prompt,
89
+ prompt_embeds=None,
90
+ )
91
92
input_ids: list[int] = []
93
for engine_prompt in engine_prompts:
@@ -152,12 +145,9 @@ class OpenAIServingTokenization(OpenAIServing):
145
self,
146
) -> TokenizerInfoResponse | ErrorResponse:
147
"""Get comprehensive tokenizer information."""
155
- try:
156
- tokenizer = self.renderer.get_tokenizer()
157
- info = TokenizerInfo(tokenizer, self.chat_template).to_dict()
158
- return TokenizerInfoResponse(**info)
159
- except Exception as e:
160
- return self.create_error_response(f"Failed to get tokenizer info: {str(e)}")
148
+ tokenizer = self.renderer.get_tokenizer()
149
+ info = TokenizerInfo(tokenizer, self.chat_template).to_dict()
150
+ return TokenizerInfoResponse(**info)
151
152
153
@dataclass
vllm/entrypoints/utils.py
+15
-29
@@ -5,13 +5,10 @@ import asyncio
5
import dataclasses
6
import functools
7
import os
8
-import sys
9
-import traceback
8
from argparse import Namespace
9
from http import HTTPStatus
10
from logging import Logger
11
from string import Template
14
-from typing import TYPE_CHECKING
12
13
import regex as re
14
from fastapi import Request
@@ -20,24 +17,17 @@ from starlette.background import BackgroundTask, BackgroundTasks
17
18
from vllm import envs
19
from vllm.engine.arg_utils import EngineArgs
23
-from vllm.exceptions import VLLMValidationError
20
+from vllm.entrypoints.openai.engine.protocol import (
21
+ ErrorInfo,
22
+ ErrorResponse,
23
+ GenerationError,
24
+ StreamOptions,
25
+)
26
+from vllm.entrypoints.openai.models.protocol import LoRAModulePath
27
from vllm.logger import current_formatter_type, init_logger
28
from vllm.platforms import current_platform
29
from vllm.utils.argparse_utils import FlexibleArgumentParser
30
28
-if TYPE_CHECKING:
29
- from vllm.entrypoints.openai.engine.protocol import (
30
- ErrorInfo,
31
- ErrorResponse,
32
- StreamOptions,
33
- )
34
- from vllm.entrypoints.openai.models.protocol import LoRAModulePath
35
-else:
36
- ErrorResponse = object
37
- ErrorInfo = object
38
- LoRAModulePath = object
39
- StreamOptions = object
40
-
31
logger = init_logger(__name__)
32
33
VLLM_SUBCMD_PARSER_EPILOG = (
@@ -307,20 +297,19 @@ def create_error_response(
297
err_type: str = "BadRequestError",
298
status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
299
param: str | None = None,
310
- log_error_stack: bool = False,
311
-) -> "ErrorResponse":
300
+) -> ErrorResponse:
301
exc: Exception | None = None
302
314
- from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse
315
-
303
if isinstance(message, Exception):
304
exc = message
305
306
+ from vllm.exceptions import VLLMValidationError
307
+
308
if isinstance(exc, VLLMValidationError):
309
err_type = "BadRequestError"
310
status_code = HTTPStatus.BAD_REQUEST
311
param = exc.parameter
323
- elif isinstance(exc, (ValueError, TypeError, RuntimeError, OverflowError)):
312
+ elif isinstance(exc, (ValueError, TypeError, OverflowError)):
313
# Common validation errors from user input
314
err_type = "BadRequestError"
315
status_code = HTTPStatus.BAD_REQUEST
@@ -329,6 +318,10 @@ def create_error_response(
318
err_type = "NotImplementedError"
319
status_code = HTTPStatus.NOT_IMPLEMENTED
320
param = None
321
+ elif isinstance(exc, GenerationError):
322
+ err_type = "InternalServerError"
323
+ status_code = exc.status_code
324
+ param = None
325
elif exc.__class__.__name__ == "TemplateError":
326
# jinja2.TemplateError (avoid importing jinja2)
327
err_type = "BadRequestError"
@@ -341,13 +334,6 @@ def create_error_response(
334
335
message = str(exc)
336
344
- if log_error_stack:
345
- exc_type, _, _ = sys.exc_info()
346
- if exc_type is not None:
347
- traceback.print_exc()
348
- else:
349
- traceback.print_stack()
350
-
337
return ErrorResponse(
338
error=ErrorInfo(
339
message=sanitize_message(message),