A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 09:00 UTC
[Frontend][1/n] Improve pooling entrypoints | classify. (#35604)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Signed-off-by: wang.yuqi <noooop@126.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
wang.yuqi committed
Mar 3, 2026 at 22:05 UTC
ea463978bb987a4c15c9b51c0013d620a722aa67
12 files changed
+890
-171
vllm/entrypoints/chat_utils.py
+8
@@ -7,6 +7,7 @@ import warnings
7
from abc import ABC, abstractmethod
8
from collections import Counter, defaultdict
9
from collections.abc import Awaitable, Callable, Iterable
10
+from dataclasses import dataclass
11
from functools import cached_property, lru_cache, partial
12
from itertools import accumulate
13
from pathlib import Path
@@ -1024,6 +1025,13 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
1025
self._add_placeholder("video", placeholder)
1026
1027
1028
+@dataclass
1029
+class ChatTemplateConfig:
1030
+ chat_template: str | None = None
1031
+ chat_template_content_format: ChatTemplateContentFormatOption = "auto"
1032
+ trust_request_chat_template: bool = False
1033
+
1034
+
1035
def validate_chat_template(chat_template: Path | str | None):
1036
"""Raises if the provided chat template appears invalid."""
1037
if chat_template is None:
vllm/entrypoints/llm.py
+67
-25
@@ -3,6 +3,7 @@
3
4
import itertools
5
from collections.abc import Callable, Iterable, Sequence
6
+from pathlib import Path
7
from typing import TYPE_CHECKING, Any
8
9
import cloudpickle
@@ -40,8 +41,11 @@ from vllm.distributed.weight_transfer.base import (
41
from vllm.engine.arg_utils import EngineArgs
42
from vllm.entrypoints.chat_utils import (
43
ChatCompletionMessageParam,
44
+ ChatTemplateConfig,
45
ChatTemplateContentFormatOption,
46
+ load_chat_template,
47
)
48
+from vllm.entrypoints.pooling.io_processor_factories import init_pooling_io_processors
49
from vllm.entrypoints.pooling.score.utils import (
50
ScoreData,
51
ScoreMultiModalParam,
@@ -145,6 +149,7 @@ class LLM:
149
a tag name, or a commit id.
150
tokenizer_revision: The specific tokenizer version to use. It can be a
151
branch name, a tag name, or a commit id.
152
+ chat_template: The chat template to apply.
153
seed: The seed to initialize the random number generator for sampling.
154
gpu_memory_utilization: The ratio (between 0 and 1) of GPU memory to
155
reserve for the model weights, activations, and KV cache. Higher
@@ -232,6 +237,7 @@ class LLM:
237
quantization: QuantizationMethods | None = None,
238
revision: str | None = None,
239
tokenizer_revision: str | None = None,
240
+ chat_template: Path | str | None = None,
241
seed: int = 0,
242
gpu_memory_utilization: float = 0.9,
243
swap_space: float = 4,
@@ -384,9 +390,16 @@ class LLM:
390
391
self.model_config = self.llm_engine.model_config
392
self.renderer = self.llm_engine.renderer
393
+ self.chat_template = load_chat_template(chat_template)
394
self.io_processor = self.llm_engine.io_processor
395
self.input_processor = self.llm_engine.input_processor
389
-
396
+ self.chat_template_config = ChatTemplateConfig(chat_template=self.chat_template)
397
+ self.init_pooling_io_processors = init_pooling_io_processors(
398
+ supported_tasks=supported_tasks,
399
+ model_config=self.model_config,
400
+ renderer=self.renderer,
401
+ chat_template_config=self.chat_template_config,
402
+ )
403
# Cache for __repr__ to avoid repeated collective_rpc calls
404
self._cached_repr: str | None = None
405
@@ -1086,7 +1099,7 @@ class LLM:
1099
"pooling model."
1100
)
1101
1089
- if use_io_processor := (isinstance(prompts, dict) and "data" in prompts):
1102
+ if isinstance(prompts, dict) and "data" in prompts:
1103
if self.io_processor is None:
1104
raise ValueError(
1105
"No IOProcessor plugin installed. Please refer "
@@ -1120,6 +1133,31 @@ class LLM:
1133
for p in params_seq:
1134
if p.task is None:
1135
p.task = "plugin"
1136
+
1137
+ outputs = self._run_completion(
1138
+ prompts=prompts_seq,
1139
+ params=params_seq,
1140
+ output_type=PoolingRequestOutput,
1141
+ use_tqdm=use_tqdm,
1142
+ lora_request=lora_request,
1143
+ tokenization_kwargs=tokenization_kwargs,
1144
+ )
1145
+
1146
+ # get the post-processed model outputs
1147
+ assert self.io_processor is not None
1148
+ processed_outputs = self.io_processor.post_process(outputs)
1149
+
1150
+ return [
1151
+ PoolingRequestOutput[Any](
1152
+ request_id="",
1153
+ outputs=processed_outputs,
1154
+ num_cached_tokens=getattr(
1155
+ processed_outputs, "num_cached_tokens", 0
1156
+ ),
1157
+ prompt_token_ids=[],
1158
+ finished=True,
1159
+ )
1160
+ ]
1161
else:
1162
if pooling_params is None:
1163
# Use default pooling params.
@@ -1137,32 +1175,36 @@ class LLM:
1175
)
1176
raise ValueError(msg)
1177
1140
- outputs = self._run_completion(
1141
- prompts=prompts_seq,
1142
- params=params_seq,
1143
- output_type=PoolingRequestOutput,
1144
- use_tqdm=use_tqdm,
1145
- lora_request=lora_request,
1146
- tokenization_kwargs=tokenization_kwargs,
1147
- )
1148
-
1149
- if use_io_processor:
1150
- # get the post-processed model outputs
1151
- assert self.io_processor is not None
1152
- processed_outputs = self.io_processor.post_process(outputs)
1178
+ if pooling_task in self.init_pooling_io_processors:
1179
+ io_processor = self.init_pooling_io_processors[pooling_task]
1180
+ processor_inputs = io_processor.pre_process_offline(
1181
+ prompts_seq, tokenization_kwargs
1182
+ )
1183
+ seq_lora_requests = self._lora_request_to_seq(
1184
+ lora_request, len(prompts_seq)
1185
+ )
1186
+ seq_priority = self._priority_to_seq(None, len(prompts))
1187
1154
- return [
1155
- PoolingRequestOutput[Any](
1156
- request_id="",
1157
- outputs=processed_outputs,
1158
- num_cached_tokens=getattr(
1159
- processed_outputs, "num_cached_tokens", 0
1160
- ),
1161
- prompt_token_ids=[],
1162
- finished=True,
1188
+ self._render_and_add_requests(
1189
+ prompts=processor_inputs,
1190
+ params=params_seq,
1191
+ lora_requests=seq_lora_requests,
1192
+ priorities=seq_priority,
1193
)
1164
- ]
1194
1195
+ outputs = self._run_engine(
1196
+ use_tqdm=use_tqdm, output_type=PoolingRequestOutput
1197
+ )
1198
+ outputs = io_processor.post_process(outputs)
1199
+ else:
1200
+ outputs = self._run_completion(
1201
+ prompts=prompts_seq,
1202
+ params=params_seq,
1203
+ output_type=PoolingRequestOutput,
1204
+ use_tqdm=use_tqdm,
1205
+ lora_request=lora_request,
1206
+ tokenization_kwargs=tokenization_kwargs,
1207
+ )
1208
return outputs
1209
1210
def embed(
vllm/entrypoints/openai/engine/serving.py
+4
-17
@@ -62,11 +62,6 @@ from vllm.entrypoints.openai.speech_to_text.protocol import (
62
TranscriptionResponse,
63
TranslationRequest,
64
)
65
-from vllm.entrypoints.pooling.classify.protocol import (
66
- ClassificationChatRequest,
67
- ClassificationCompletionRequest,
68
- ClassificationResponse,
69
-)
65
from vllm.entrypoints.pooling.embed.protocol import (
66
EmbeddingBytesResponse,
67
EmbeddingChatRequest,
@@ -161,7 +156,6 @@ CompletionLikeRequest: TypeAlias = (
156
| TokenizeCompletionRequest
157
| DetokenizeRequest
158
| EmbeddingCompletionRequest
164
- | ClassificationCompletionRequest
159
| RerankRequest
160
| ScoreRequest
161
| PoolingCompletionRequest
@@ -171,7 +165,6 @@ ChatLikeRequest: TypeAlias = (
165
ChatCompletionRequest
166
| TokenizeChatRequest
167
| EmbeddingChatRequest
174
- | ClassificationChatRequest
168
| PoolingChatRequest
169
)
170
@@ -194,12 +187,10 @@ AnyResponse: TypeAlias = (
187
| TranscriptionResponse
188
| TokenizeResponse
189
| PoolingResponse
197
- | ClassificationResponse
190
| ScoreResponse
191
| GenerateResponse
192
)
193
202
-
194
RequestT = TypeVar("RequestT", bound=AnyRequest)
195
196
@@ -223,8 +214,8 @@ class ServeContext(Generic[RequestT]):
214
215
class OpenAIServing:
216
request_id_prefix: ClassVar[str] = """
226
- A short string prepended to every request’s ID (e.g. "embd", "classify")
227
- so you can easily tell “this ID came from Embedding vs Classification.”
217
+ A short string prepended to every request’s ID (e.g. "embd")
218
+ so you can easily tell “this ID came from Embedding.”
219
"""
220
221
def __init__(
@@ -456,7 +447,7 @@ class OpenAIServing:
447
) -> ErrorResponse | None:
448
"""
449
Default preprocessing hook. Subclasses may override
459
- to prepare `ctx` (classification, embedding, etc.).
450
+ to prepare `ctx` (embedding, etc.).
451
"""
452
return None
453
@@ -817,7 +808,7 @@ class OpenAIServing:
808
token_num = len(input_ids)
809
max_model_len = self.model_config.max_model_len
810
820
- # Note: EmbeddingRequest, ClassificationRequest,
811
+ # Note: EmbeddingRequest,
812
# and ScoreRequest doesn't have max_tokens
813
if isinstance(
814
request,
@@ -828,8 +819,6 @@ class OpenAIServing:
819
ScoreTextRequest,
820
ScoreQueriesDocumentsRequest,
821
RerankRequest,
831
- ClassificationCompletionRequest,
832
- ClassificationChatRequest,
822
),
823
):
824
# Note: input length can be up to the entire model context length
@@ -839,8 +828,6 @@ class OpenAIServing:
828
ScoreDataRequest: "score",
829
ScoreTextRequest: "score",
830
ScoreQueriesDocumentsRequest: "score",
842
- ClassificationCompletionRequest: "classification",
843
- ClassificationChatRequest: "classification",
831
}
832
operation = operations.get(type(request), "embedding generation")
833
raise VLLMValidationError(
vllm/entrypoints/pooling/base/io_processor.py
new
+189
@@ -0,0 +1,189 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from collections.abc import Callable, Sequence
5
+from concurrent.futures import ThreadPoolExecutor
6
+from typing import Any, Final
7
+
8
+from vllm import PoolingRequestOutput, PromptType
9
+from vllm.config import ModelConfig
10
+from vllm.entrypoints.chat_utils import (
11
+ ChatCompletionMessageParam,
12
+ ChatTemplateConfig,
13
+ ChatTemplateContentFormatOption,
14
+ ConversationMessage,
15
+)
16
+from vllm.entrypoints.openai.engine.serving import RendererChatRequest, RendererRequest
17
+from vllm.inputs import ProcessorInputs, SingletonPrompt
18
+from vllm.renderers import BaseRenderer, merge_kwargs
19
+from vllm.renderers.inputs import TokPrompt
20
+from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
21
+from vllm.tokenizers import TokenizerLike
22
+from vllm.tool_parsers import ToolParser
23
+from vllm.utils.mistral import is_mistral_tokenizer
24
+
25
+
26
+class PoolingIOProcessor:
27
+ def __init__(
28
+ self,
29
+ model_config: ModelConfig,
30
+ renderer: BaseRenderer,
31
+ chat_template_config: ChatTemplateConfig,
32
+ ):
33
+ self._tokenizer_executor = ThreadPoolExecutor(max_workers=1)
34
+
35
+ self.model_config = model_config
36
+ self.renderer = renderer
37
+
38
+ self.chat_template = chat_template_config.chat_template
39
+ self.chat_template_content_format: Final = (
40
+ chat_template_config.chat_template_content_format
41
+ )
42
+ self.trust_request_chat_template = (
43
+ chat_template_config.trust_request_chat_template
44
+ )
45
+
46
+ def pre_process_online(self, *args, **kwargs):
47
+ raise NotImplementedError
48
+
49
+ async def pre_process_online_async(self, *args, **kwargs):
50
+ return self.pre_process_online(*args, **kwargs)
51
+
52
+ def pre_process_offline(self, *args, **kwargs):
53
+ raise NotImplementedError
54
+
55
+ async def pre_process_offline_async(self, *args, **kwargs):
56
+ return self.pre_process_offline(*args, **kwargs)
57
+
58
+ def post_process(
59
+ self, outputs: list[PoolingRequestOutput]
60
+ ) -> list[PoolingRequestOutput]:
61
+ return outputs
62
+
63
+ async def post_process_async(
64
+ self, outputs: list[PoolingRequestOutput]
65
+ ) -> list[PoolingRequestOutput]:
66
+ return self.post_process(outputs)
67
+
68
+ def create_pooling_params(self, request):
69
+ return request.to_pooling_params()
70
+
71
+ def _preprocess_completion_online(
72
+ self,
73
+ request: RendererRequest,
74
+ prompt_input: str | list[str] | list[int] | list[list[int]] | None,
75
+ prompt_embeds: bytes | list[bytes] | None,
76
+ ) -> list[TokPrompt]:
77
+ renderer = self.renderer
78
+ model_config = self.model_config
79
+
80
+ prompts = list[SingletonPrompt | bytes]()
81
+ if prompt_embeds is not None: # embeds take higher priority
82
+ prompts.extend(prompt_to_seq(prompt_embeds))
83
+ if prompt_input is not None:
84
+ prompts.extend(prompt_to_seq(prompt_input))
85
+
86
+ parsed_prompts = [
87
+ (
88
+ prompt
89
+ if isinstance(prompt, bytes)
90
+ else parse_model_prompt(model_config, prompt)
91
+ )
92
+ for prompt in prompts
93
+ ]
94
+ tok_params = request.build_tok_params(model_config)
95
+
96
+ return renderer.render_cmpl(
97
+ parsed_prompts,
98
+ tok_params,
99
+ prompt_extras={
100
+ k: v
101
+ for k in ("mm_processor_kwargs", "cache_salt")
102
+ if (v := getattr(request, k, None)) is not None
103
+ },
104
+ )
105
+
106
+ def _preprocess_chat_online(
107
+ self,
108
+ request: RendererChatRequest,
109
+ messages: list[ChatCompletionMessageParam],
110
+ default_template: str | None,
111
+ default_template_content_format: ChatTemplateContentFormatOption,
112
+ default_template_kwargs: dict[str, Any] | None,
113
+ tool_dicts: list[dict[str, Any]] | None = None,
114
+ tool_parser: Callable[[TokenizerLike], ToolParser] | None = None,
115
+ ) -> tuple[list[ConversationMessage], list[TokPrompt]]:
116
+ renderer = self.renderer
117
+
118
+ default_template_kwargs = merge_kwargs(
119
+ default_template_kwargs,
120
+ dict(
121
+ tools=tool_dicts,
122
+ tokenize=is_mistral_tokenizer(renderer.tokenizer),
123
+ ),
124
+ )
125
+
126
+ tok_params = request.build_tok_params(self.model_config)
127
+ chat_params = request.build_chat_params(
128
+ default_template, default_template_content_format
129
+ ).with_defaults(default_template_kwargs)
130
+
131
+ (conversation,), (engine_prompt,) = renderer.render_chat(
132
+ [messages],
133
+ chat_params,
134
+ tok_params,
135
+ prompt_extras={
136
+ k: v
137
+ for k in ("mm_processor_kwargs", "cache_salt")
138
+ if (v := getattr(request, k, None)) is not None
139
+ },
140
+ )
141
+
142
+ return conversation, [engine_prompt]
143
+
144
+ def _preprocess_completion_offline(
145
+ self,
146
+ prompts: PromptType | Sequence[PromptType],
147
+ tokenization_kwargs: dict[str, Any] | None = None,
148
+ ) -> Sequence[ProcessorInputs]:
149
+ renderer = self.renderer
150
+ model_config = self.model_config
151
+
152
+ prompts = prompt_to_seq(prompts)
153
+
154
+ parsed_prompts = [
155
+ (
156
+ prompt
157
+ if isinstance(prompt, bytes)
158
+ else parse_model_prompt(model_config, prompt)
159
+ )
160
+ for prompt in prompts
161
+ ]
162
+ tok_params = renderer.default_cmpl_tok_params.with_kwargs(
163
+ **(tokenization_kwargs or {})
164
+ )
165
+
166
+ return renderer.render_cmpl(
167
+ parsed_prompts,
168
+ tok_params,
169
+ )
170
+
171
+ def _validate_chat_template(
172
+ self,
173
+ request_chat_template: str | None,
174
+ chat_template_kwargs: dict[str, Any] | None,
175
+ trust_request_chat_template: bool,
176
+ ):
177
+ if not trust_request_chat_template and (
178
+ request_chat_template is not None
179
+ or (
180
+ chat_template_kwargs
181
+ and chat_template_kwargs.get("chat_template") is not None
182
+ )
183
+ ):
184
+ raise ValueError(
185
+ "Chat template is passed with request, but "
186
+ "--trust-request-chat-template is not set. "
187
+ "Refused request with untrusted chat template."
188
+ )
189
+ return None
vllm/entrypoints/pooling/base/serving.py
new
+378
@@ -0,0 +1,378 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import time
4
+from collections.abc import AsyncGenerator, Mapping
5
+from dataclasses import dataclass, field
6
+from http import HTTPStatus
7
+from typing import ClassVar, Generic, TypeVar
8
+
9
+from fastapi import Request
10
+from pydantic import ConfigDict
11
+from starlette.datastructures import Headers
12
+from starlette.responses import JSONResponse
13
+
14
+from vllm import (
15
+ PoolingParams,
16
+ PoolingRequestOutput,
17
+ PromptType,
18
+ SamplingParams,
19
+ envs,
20
+)
21
+from vllm.config import ModelConfig
22
+from vllm.engine.protocol import EngineClient
23
+from vllm.entrypoints.chat_utils import (
24
+ ChatTemplateConfig,
25
+ ChatTemplateContentFormatOption,
26
+)
27
+from vllm.entrypoints.logger import RequestLogger
28
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
29
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
30
+from vllm.entrypoints.pooling.typing import AnyPoolingRequest, AnyPoolingResponse
31
+from vllm.inputs import ProcessorInputs
32
+from vllm.lora.request import LoRARequest
33
+from vllm.renderers import BaseRenderer
34
+from vllm.renderers.inputs.preprocess import extract_prompt_components
35
+from vllm.sampling_params import BeamSearchParams
36
+from vllm.tracing import (
37
+ contains_trace_headers,
38
+ extract_trace_headers,
39
+ log_tracing_disabled_warning,
40
+)
41
+from vllm.utils import random_uuid
42
+from vllm.utils.async_utils import merge_async_iterators
43
+
44
+from ...utils import create_error_response
45
+from .io_processor import PoolingIOProcessor
46
+
47
+PoolingRequestT = TypeVar("PoolingRequestT", bound=AnyPoolingRequest)
48
+
49
+
50
+@dataclass(kw_only=True)
51
+class PoolingServeContext(Generic[PoolingRequestT]):
52
+ request: PoolingRequestT
53
+ raw_request: Request | None = None
54
+ model_name: str
55
+ request_id: str
56
+ created_time: int = field(default_factory=lambda: int(time.time()))
57
+ lora_request: LoRARequest | None = None
58
+ engine_prompts: list[ProcessorInputs] | None = None
59
+
60
+ result_generator: AsyncGenerator[tuple[int, PoolingRequestOutput], None] | None = (
61
+ None
62
+ )
63
+ final_res_batch: list[PoolingRequestOutput] = field(default_factory=list)
64
+
65
+ model_config = ConfigDict(arbitrary_types_allowed=True)
66
+
67
+
68
+class PoolingServing:
69
+ request_id_prefix: ClassVar[str]
70
+
71
+ def __init__(
72
+ self,
73
+ engine_client: EngineClient,
74
+ models: OpenAIServingModels,
75
+ *,
76
+ request_logger: RequestLogger | None,
77
+ chat_template: str | None = None,
78
+ chat_template_content_format: ChatTemplateContentFormatOption = "auto",
79
+ trust_request_chat_template: bool = False,
80
+ return_tokens_as_token_ids: bool = False,
81
+ log_error_stack: bool = False,
82
+ ):
83
+ super().__init__()
84
+ self.engine_client = engine_client
85
+ self.models = models
86
+ self.model_config = models.model_config
87
+ self.max_model_len = self.model_config.max_model_len
88
+ self.request_logger = request_logger
89
+ self.return_tokens_as_token_ids = return_tokens_as_token_ids
90
+ self.log_error_stack = log_error_stack
91
+ self.chat_template_config = ChatTemplateConfig(
92
+ chat_template=chat_template,
93
+ chat_template_content_format=chat_template_content_format,
94
+ trust_request_chat_template=trust_request_chat_template,
95
+ )
96
+ self.io_processor = self.init_io_processor(
97
+ model_config=models.model_config,
98
+ renderer=models.renderer,
99
+ chat_template_config=self.chat_template_config,
100
+ )
101
+
102
+ def init_io_processor(
103
+ self,
104
+ model_config: ModelConfig,
105
+ renderer: BaseRenderer,
106
+ chat_template_config: ChatTemplateConfig,
107
+ ) -> PoolingIOProcessor:
108
+ raise NotImplementedError
109
+
110
+ async def __call__(
111
+ self,
112
+ request: AnyPoolingRequest,
113
+ raw_request: Request,
114
+ ) -> JSONResponse:
115
+ try:
116
+ model_name = self.models.model_name()
117
+ request_id = (
118
+ f"{self.request_id_prefix}-{self._base_request_id(raw_request)}"
119
+ )
120
+
121
+ await self._check_model(request)
122
+
123
+ ctx = PoolingServeContext(
124
+ request=request,
125
+ raw_request=raw_request,
126
+ model_name=model_name,
127
+ request_id=request_id,
128
+ )
129
+
130
+ self._validate_request(ctx)
131
+ self._maybe_get_adapters(ctx)
132
+ await self._preprocess(ctx)
133
+ await self._prepare_generators(ctx)
134
+ await self._collect_batch(ctx)
135
+ response = await self._build_response(ctx)
136
+ return JSONResponse(content=response.model_dump())
137
+ except Exception as e:
138
+ error_response = create_error_response(e)
139
+ return JSONResponse(
140
+ content=error_response.model_dump(),
141
+ status_code=error_response.error.code,
142
+ )
143
+
144
+ async def _preprocess(
145
+ self,
146
+ ctx: PoolingServeContext,
147
+ ):
148
+ ctx.engine_prompts = await self.io_processor.pre_process_online_async(
149
+ ctx.request
150
+ )
151
+
152
+ async def _prepare_generators(
153
+ self,
154
+ ctx: PoolingServeContext,
155
+ ):
156
+ if ctx.engine_prompts is None:
157
+ raise ValueError("Engine prompts not available")
158
+
159
+ generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
160
+
161
+ trace_headers = (
162
+ None
163
+ if ctx.raw_request is None
164
+ else await self._get_trace_headers(ctx.raw_request.headers)
165
+ )
166
+
167
+ pooling_params = self.io_processor.create_pooling_params(ctx.request)
168
+
169
+ for i, engine_prompt in enumerate(ctx.engine_prompts):
170
+ request_id_item = f"{ctx.request_id}-{i}"
171
+
172
+ self._log_inputs(
173
+ request_id_item,
174
+ engine_prompt,
175
+ params=pooling_params,
176
+ lora_request=ctx.lora_request,
177
+ )
178
+
179
+ generator = self.engine_client.encode(
180
+ engine_prompt,
181
+ pooling_params,
182
+ request_id_item,
183
+ lora_request=ctx.lora_request,
184
+ trace_headers=trace_headers,
185
+ priority=getattr(ctx.request, "priority", 0),
186
+ )
187
+
188
+ generators.append(generator)
189
+
190
+ ctx.result_generator = merge_async_iterators(*generators)
191
+
192
+ async def _collect_batch(
193
+ self,
194
+ ctx: PoolingServeContext,
195
+ ):
196
+ if ctx.engine_prompts is None:
197
+ raise ValueError("Engine prompts not available")
198
+
199
+ if ctx.result_generator is None:
200
+ raise ValueError("Result generator not available")
201
+
202
+ num_prompts = len(ctx.engine_prompts)
203
+ final_res_batch: list[PoolingRequestOutput | None]
204
+ final_res_batch = [None] * num_prompts
205
+
206
+ async for i, res in ctx.result_generator:
207
+ final_res_batch[i] = res
208
+
209
+ if None in final_res_batch:
210
+ raise ValueError("Failed to generate results for all prompts")
211
+
212
+ ctx.final_res_batch = [res for res in final_res_batch if res is not None]
213
+
214
+ async def _build_response(
215
+ self,
216
+ ctx: PoolingServeContext,
217
+ ) -> AnyPoolingResponse:
218
+ raise NotImplementedError
219
+
220
+ @staticmethod
221
+ def _base_request_id(
222
+ raw_request: Request | None, default: str | None = None
223
+ ) -> str | None:
224
+ """Pulls the request id to use from a header, if provided"""
225
+ if raw_request is not None and (
226
+ (req_id := raw_request.headers.get("X-Request-Id")) is not None
227
+ ):
228
+ return req_id
229
+
230
+ return random_uuid() if default is None else default
231
+
232
+ def _is_model_supported(self, model_name: str | None) -> bool:
233
+ if not model_name:
234
+ return True
235
+ return self.models.is_base_model(model_name)
236
+
237
+ async def _check_model(
238
+ self,
239
+ request: AnyPoolingRequest,
240
+ ) -> ErrorResponse | None:
241
+ if self._is_model_supported(request.model):
242
+ return None
243
+ if request.model in self.models.lora_requests:
244
+ return None
245
+ if (
246
+ envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING
247
+ and request.model
248
+ and (load_result := await self.models.resolve_lora(request.model))
249
+ ):
250
+ if isinstance(load_result, LoRARequest):
251
+ return None
252
+ if (
253
+ isinstance(load_result, ErrorResponse)
254
+ and load_result.error.code == HTTPStatus.BAD_REQUEST.value
255
+ ):
256
+ raise ValueError(load_result.error.message)
257
+ return None
258
+
259
+ def _validate_request(self, ctx: PoolingServeContext) -> None:
260
+ truncate_prompt_tokens = getattr(ctx.request, "truncate_prompt_tokens", None)
261
+
262
+ if (
263
+ truncate_prompt_tokens is not None
264
+ and truncate_prompt_tokens > self.max_model_len
265
+ ):
266
+ raise ValueError(
267
+ "truncate_prompt_tokens value is "
268
+ "greater than max_model_len."
269
+ " Please, select a smaller truncation size."
270
+ )
271
+ return None
272
+
273
+ async def _get_trace_headers(
274
+ self,
275
+ headers: Headers,
276
+ ) -> Mapping[str, str] | None:
277
+ is_tracing_enabled = await self.engine_client.is_tracing_enabled()
278
+
279
+ if is_tracing_enabled:
280
+ return extract_trace_headers(headers)
281
+
282
+ if contains_trace_headers(headers):
283
+ log_tracing_disabled_warning()
284
+
285
+ return None
286
+
287
+ def _maybe_get_adapters(
288
+ self,
289
+ ctx: PoolingServeContext,
290
+ supports_default_mm_loras: bool = False,
291
+ ):
292
+ request = ctx.request
293
+ if request.model in self.models.lora_requests:
294
+ ctx.lora_request = self.models.lora_requests[request.model]
295
+
296
+ # Currently only support default modality specific loras
297
+ # if we have exactly one lora matched on the request.
298
+ if supports_default_mm_loras:
299
+ default_mm_lora = self._get_active_default_mm_loras(request)
300
+ if default_mm_lora is not None:
301
+ ctx.lora_request = default_mm_lora
302
+
303
+ if self._is_model_supported(request.model):
304
+ return None
305
+
306
+ # if _check_model has been called earlier, this will be unreachable
307
+ raise ValueError(f"The model `{request.model}` does not exist.")
308
+
309
+ def _get_active_default_mm_loras(
310
+ self, request: AnyPoolingRequest
311
+ ) -> LoRARequest | None:
312
+ """Determine if there are any active default multimodal loras."""
313
+ # TODO: Currently this is only enabled for chat completions
314
+ # to be better aligned with only being enabled for .generate
315
+ # when run offline. It would be nice to support additional
316
+ # tasks types in the future.
317
+ message_types = self._get_message_types(request)
318
+ default_mm_loras = set()
319
+
320
+ for lora in self.models.lora_requests.values():
321
+ # Best effort match for default multimodal lora adapters;
322
+ # There is probably a better way to do this, but currently
323
+ # this matches against the set of 'types' in any content lists
324
+ # up until '_', e.g., to match audio_url -> audio
325
+ if lora.lora_name in message_types:
326
+ default_mm_loras.add(lora)
327
+
328
+ # Currently only support default modality specific loras if
329
+ # we have exactly one lora matched on the request.
330
+ if len(default_mm_loras) == 1:
331
+ return default_mm_loras.pop()
332
+ return None
333
+
334
+ def _get_message_types(self, request: AnyPoolingRequest) -> set[str]:
335
+ """Retrieve the set of types from message content dicts up
336
+ until `_`; we use this to match potential multimodal data
337
+ with default per modality loras.
338
+ """
339
+ message_types: set[str] = set()
340
+
341
+ if not hasattr(request, "messages"):
342
+ return message_types
343
+
344
+ messages = request.messages
345
+ if messages is None or isinstance(messages, (str, bytes)):
346
+ return message_types
347
+
348
+ for message in messages:
349
+ if (
350
+ isinstance(message, dict)
351
+ and "content" in message
352
+ and isinstance(message["content"], list)
353
+ ):
354
+ for content_dict in message["content"]:
355
+ if "type" in content_dict:
356
+ message_types.add(content_dict["type"].split("_")[0])
357
+ return message_types
358
+
359
+ def _log_inputs(
360
+ self,
361
+ request_id: str,
362
+ inputs: PromptType | ProcessorInputs,
363
+ params: SamplingParams | PoolingParams | BeamSearchParams | None,
364
+ lora_request: LoRARequest | None,
365
+ ) -> None:
366
+ if self.request_logger is None:
367
+ return
368
+
369
+ components = extract_prompt_components(self.model_config, inputs)
370
+
371
+ self.request_logger.log_inputs(
372
+ request_id,
373
+ components.text,
374
+ components.token_ids,
375
+ components.embeds,
376
+ params=params,
377
+ lora_request=lora_request,
378
+ )
vllm/entrypoints/pooling/classify/api_router.py
+12
-19
@@ -3,16 +3,17 @@
3
4
from fastapi import APIRouter, Depends, Request
5
from starlette.responses import JSONResponse
6
-from typing_extensions import assert_never
6
8
-from vllm.entrypoints.openai.engine.protocol import ErrorResponse
7
from vllm.entrypoints.openai.utils import validate_json_request
8
from vllm.entrypoints.pooling.classify.protocol import (
9
ClassificationRequest,
12
- ClassificationResponse,
10
)
11
from vllm.entrypoints.pooling.classify.serving import ServingClassification
15
-from vllm.entrypoints.utils import load_aware_call, with_cancellation
12
+from vllm.entrypoints.utils import (
13
+ create_error_response,
14
+ load_aware_call,
15
+ with_cancellation,
16
+)
17
18
router = APIRouter()
19
@@ -24,25 +25,17 @@ def classify(request: Request) -> ServingClassification | None:
25
@router.post("/classify", dependencies=[Depends(validate_json_request)])
26
@with_cancellation
27
@load_aware_call
27
-async def create_classify(request: ClassificationRequest, raw_request: Request):
28
+async def create_classify(
29
+ request: ClassificationRequest, raw_request: Request
30
+) -> JSONResponse:
31
handler = classify(raw_request)
32
if handler is None:
30
- base_server = raw_request.app.state.openai_serving_tokenization
31
- return base_server.create_error_response(
33
+ error_response = create_error_response(
34
message="The model does not support Classification API"
35
)
34
-
35
- try:
36
- generator = await handler.create_classify(request, raw_request)
37
- except Exception as e:
38
- generator = handler.create_error_response(e)
39
-
40
- if isinstance(generator, ErrorResponse):
36
return JSONResponse(
42
- content=generator.model_dump(), status_code=generator.error.code
37
+ content=error_response.model_dump(),
38
+ status_code=error_response.error.code,
39
)
40
45
- elif isinstance(generator, ClassificationResponse):
46
- return JSONResponse(content=generator.model_dump())
47
-
48
- assert_never(generator)
41
+ return await handler(request, raw_request)
vllm/entrypoints/pooling/classify/io_processor.py
new
+50
@@ -0,0 +1,50 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from collections.abc import Sequence
4
+from typing import Any
5
+
6
+from vllm import PromptType
7
+from vllm.entrypoints.pooling.base.io_processor import PoolingIOProcessor
8
+from vllm.entrypoints.pooling.classify.protocol import (
9
+ ClassificationChatRequest,
10
+ ClassificationCompletionRequest,
11
+)
12
+from vllm.inputs import ProcessorInputs
13
+from vllm.renderers.inputs import TokPrompt
14
+
15
+
16
+class ClassifyIOProcessor(PoolingIOProcessor):
17
+ def pre_process_online(
18
+ self, request: ClassificationCompletionRequest | ClassificationChatRequest
19
+ ) -> list[TokPrompt] | None:
20
+ if isinstance(request, ClassificationChatRequest):
21
+ self._validate_chat_template(
22
+ request_chat_template=request.chat_template,
23
+ chat_template_kwargs=request.chat_template_kwargs,
24
+ trust_request_chat_template=self.trust_request_chat_template,
25
+ )
26
+ _, engine_prompts = self._preprocess_chat_online(
27
+ request,
28
+ request.messages,
29
+ default_template=self.chat_template,
30
+ default_template_content_format=self.chat_template_content_format,
31
+ default_template_kwargs=None,
32
+ )
33
+ elif isinstance(request, ClassificationCompletionRequest):
34
+ engine_prompts = self._preprocess_completion_online(
35
+ request,
36
+ prompt_input=request.input,
37
+ prompt_embeds=None,
38
+ )
39
+ else:
40
+ raise ValueError("Invalid classification request type")
41
+ return engine_prompts
42
+
43
+ def pre_process_offline(
44
+ self,
45
+ prompts: PromptType | Sequence[PromptType],
46
+ tokenization_kwargs: dict[str, Any] | None = None,
47
+ ) -> Sequence[ProcessorInputs]:
48
+ return self._preprocess_completion_offline(
49
+ prompts=prompts, tokenization_kwargs=tokenization_kwargs
50
+ )
vllm/entrypoints/pooling/classify/serving.py
+30
-106
@@ -1,116 +1,57 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-from typing import Final, TypeAlias
4
+from typing import TypeAlias
5
6
-import jinja2
6
import numpy as np
8
-from fastapi import Request
9
-
10
-from vllm.engine.protocol import EngineClient
11
-from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
12
-from vllm.entrypoints.logger import RequestLogger
13
-from vllm.entrypoints.openai.engine.protocol import ErrorResponse, UsageInfo
14
-from vllm.entrypoints.openai.engine.serving import OpenAIServing, ServeContext
15
-from vllm.entrypoints.openai.models.serving import OpenAIServingModels
16
-from vllm.entrypoints.pooling.classify.protocol import (
17
- ClassificationChatRequest,
18
- ClassificationCompletionRequest,
7
+
8
+from vllm import ClassificationOutput
9
+from vllm.config import ModelConfig
10
+from vllm.entrypoints.chat_utils import ChatTemplateConfig
11
+from vllm.entrypoints.openai.engine.protocol import UsageInfo
12
+from vllm.entrypoints.pooling.base.serving import PoolingServeContext, PoolingServing
13
+from vllm.logger import init_logger
14
+from vllm.renderers import BaseRenderer
15
+
16
+from .io_processor import ClassifyIOProcessor
17
+from .protocol import (
18
ClassificationData,
19
ClassificationRequest,
20
ClassificationResponse,
21
)
23
-from vllm.logger import init_logger
24
-from vllm.outputs import ClassificationOutput
22
23
logger = init_logger(__name__)
24
25
29
-ClassificationServeContext: TypeAlias = ServeContext[ClassificationRequest]
26
+ClassificationServeContext: TypeAlias = PoolingServeContext[ClassificationRequest]
27
28
32
-class ServingClassification(OpenAIServing):
29
+class ServingClassification(PoolingServing):
30
request_id_prefix = "classify"
31
35
- def __init__(
32
+ def init_io_processor(
33
self,
37
- engine_client: EngineClient,
38
- models: OpenAIServingModels,
39
- *,
40
- request_logger: RequestLogger | None,
41
- chat_template: str | None = None,
42
- chat_template_content_format: ChatTemplateContentFormatOption = "auto",
43
- trust_request_chat_template: bool = False,
44
- log_error_stack: bool = False,
45
- ) -> None:
46
- super().__init__(
47
- engine_client=engine_client,
48
- models=models,
49
- request_logger=request_logger,
50
- log_error_stack=log_error_stack,
34
+ model_config: ModelConfig,
35
+ renderer: BaseRenderer,
36
+ chat_template_config: ChatTemplateConfig,
37
+ ) -> ClassifyIOProcessor:
38
+ return ClassifyIOProcessor(
39
+ model_config=model_config,
40
+ renderer=renderer,
41
+ chat_template_config=chat_template_config,
42
)
43
53
- self.chat_template = chat_template
54
- self.chat_template_content_format: Final = chat_template_content_format
55
- self.trust_request_chat_template = trust_request_chat_template
56
-
57
- async def _preprocess(
44
+ async def _build_response(
45
self,
46
ctx: ClassificationServeContext,
60
- ) -> ErrorResponse | None:
61
- """
62
- Process classification inputs: tokenize text, resolve adapters,
63
- and prepare model-specific inputs.
64
- """
65
- try:
66
- ctx.lora_request = self._maybe_get_adapters(ctx.request)
67
-
68
- if isinstance(ctx.request, ClassificationChatRequest):
69
- error_check_ret = self._validate_chat_template(
70
- request_chat_template=ctx.request.chat_template,
71
- chat_template_kwargs=ctx.request.chat_template_kwargs,
72
- trust_request_chat_template=self.trust_request_chat_template,
73
- )
74
- if error_check_ret:
75
- return error_check_ret
76
-
77
- _, ctx.engine_prompts = await self._preprocess_chat(
78
- ctx.request,
79
- ctx.request.messages,
80
- default_template=self.chat_template,
81
- default_template_content_format=self.chat_template_content_format,
82
- default_template_kwargs=None,
83
- )
84
- elif isinstance(ctx.request, ClassificationCompletionRequest):
85
- ctx.engine_prompts = await self._preprocess_completion(
86
- ctx.request,
87
- prompt_input=ctx.request.input,
88
- prompt_embeds=None,
89
- )
90
- else:
91
- return self.create_error_response("Invalid classification request type")
92
-
93
- return None
94
-
95
- except (ValueError, TypeError, jinja2.TemplateError) as e:
96
- logger.exception("Error in preprocessing prompt inputs")
97
- return self.create_error_response(str(e))
98
-
99
- def _build_response(
100
- self,
101
- ctx: ClassificationServeContext,
102
- ) -> ClassificationResponse | ErrorResponse:
103
- """
104
- Convert model outputs to a formatted classification response
105
- with probabilities and labels.
106
- """
107
- id2label = getattr(self.model_config.hf_config, "id2label", {})
47
+ ) -> ClassificationResponse:
48
+ final_res_batch_checked = await self.io_processor.post_process_async(
49
+ ctx.final_res_batch
50
+ )
51
109
- items: list[ClassificationData] = []
52
+ id2label = getattr(self.model_config.hf_config, "id2label", {})
53
num_prompt_tokens = 0
111
-
112
- final_res_batch_checked = ctx.final_res_batch
113
-
54
+ items: list[ClassificationData] = []
55
for idx, final_res in enumerate(final_res_batch_checked):
56
classify_res = ClassificationOutput.from_base(final_res.outputs)
57
@@ -141,20 +82,3 @@ class ServingClassification(OpenAIServing):
82
data=items,
83
usage=usage,
84
)
144
-
145
- async def create_classify(
146
- self,
147
- request: ClassificationRequest,
148
- raw_request: Request,
149
- ) -> ClassificationResponse | ErrorResponse:
150
- model_name = self.models.model_name()
151
- request_id = f"{self.request_id_prefix}-{self._base_request_id(raw_request)}"
152
-
153
- ctx = ClassificationServeContext(
154
- request=request,
155
- raw_request=raw_request,
156
- model_name=model_name,
157
- request_id=request_id,
158
- )
159
-
160
- return await self.handle(ctx) # type: ignore[return-value]
vllm/entrypoints/pooling/io_processor_factories.py
new
+31
@@ -0,0 +1,31 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+from vllm.config import ModelConfig
6
+from vllm.entrypoints.chat_utils import ChatTemplateConfig
7
+from vllm.entrypoints.pooling.base.io_processor import PoolingIOProcessor
8
+from vllm.renderers import BaseRenderer
9
+from vllm.tasks import SupportedTask
10
+
11
+
12
+def init_pooling_io_processors(
13
+ supported_tasks: tuple[SupportedTask, ...],
14
+ model_config: ModelConfig,
15
+ renderer: BaseRenderer,
16
+ chat_template_config: ChatTemplateConfig,
17
+) -> dict[str, PoolingIOProcessor]:
18
+ pooling_io_processors: dict[str, PoolingIOProcessor] = {}
19
+
20
+ if "classify" in supported_tasks:
21
+ from vllm.entrypoints.pooling.classify.io_processor import (
22
+ ClassifyIOProcessor,
23
+ )
24
+
25
+ pooling_io_processors["classify"] = ClassifyIOProcessor(
26
+ model_config=model_config,
27
+ renderer=renderer,
28
+ chat_template_config=chat_template_config,
29
+ )
30
+
31
+ return pooling_io_processors
vllm/entrypoints/pooling/typing.py
new
+51
@@ -0,0 +1,51 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from typing import TypeAlias
5
+
6
+from vllm.entrypoints.pooling.classify.protocol import (
7
+ ClassificationChatRequest,
8
+ ClassificationCompletionRequest,
9
+ ClassificationResponse,
10
+)
11
+from vllm.entrypoints.pooling.embed.protocol import (
12
+ EmbeddingBytesResponse,
13
+ EmbeddingChatRequest,
14
+ EmbeddingCompletionRequest,
15
+ EmbeddingResponse,
16
+)
17
+from vllm.entrypoints.pooling.pooling.protocol import (
18
+ IOProcessorRequest,
19
+ PoolingChatRequest,
20
+ PoolingCompletionRequest,
21
+ PoolingResponse,
22
+)
23
+from vllm.entrypoints.pooling.score.protocol import (
24
+ RerankRequest,
25
+ ScoreRequest,
26
+ ScoreResponse,
27
+)
28
+
29
+PoolingCompletionLikeRequest: TypeAlias = (
30
+ EmbeddingCompletionRequest
31
+ | ClassificationCompletionRequest
32
+ | RerankRequest
33
+ | ScoreRequest
34
+ | PoolingCompletionRequest
35
+)
36
+
37
+PoolingChatLikeRequest: TypeAlias = (
38
+ EmbeddingChatRequest | ClassificationChatRequest | PoolingChatRequest
39
+)
40
+
41
+AnyPoolingRequest: TypeAlias = (
42
+ PoolingCompletionLikeRequest | PoolingChatLikeRequest | IOProcessorRequest
43
+)
44
+
45
+AnyPoolingResponse: TypeAlias = (
46
+ ClassificationResponse
47
+ | EmbeddingResponse
48
+ | EmbeddingBytesResponse
49
+ | PoolingResponse
50
+ | ScoreResponse
51
+)
vllm/entrypoints/sagemaker/api_router.py
+2
-1
@@ -13,6 +13,7 @@ from fastapi.responses import JSONResponse, Response
13
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
14
from vllm.entrypoints.openai.engine.serving import OpenAIServing
15
from vllm.entrypoints.openai.utils import validate_json_request
16
+from vllm.entrypoints.pooling.base.serving import PoolingServing
17
from vllm.entrypoints.serve.instrumentator.basic import base
18
from vllm.entrypoints.serve.instrumentator.health import health
19
from vllm.tasks import POOLING_TASKS, SupportedTask
@@ -20,7 +21,7 @@ from vllm.tasks import POOLING_TASKS, SupportedTask
21
# TODO: RequestType = TypeForm[BaseModel] when recognized by type checkers
22
# (requires typing_extensions >= 4.13)
23
RequestType = Any
23
-GetHandlerFn = Callable[[Request], OpenAIServing | None]
24
+GetHandlerFn = Callable[[Request], OpenAIServing | PoolingServing | None]
25
EndpointFn = Callable[[RequestType, Request], Awaitable[Any]]
26
27
vllm/entrypoints/utils.py
+68
-3
@@ -5,7 +5,10 @@ import asyncio
5
import dataclasses
6
import functools
7
import os
8
+import sys
9
+import traceback
10
from argparse import Namespace
11
+from http import HTTPStatus
12
from logging import Logger
13
from string import Template
14
from typing import TYPE_CHECKING
@@ -17,17 +20,23 @@ from starlette.background import BackgroundTask, BackgroundTasks
20
21
from vllm import envs
22
from vllm.engine.arg_utils import EngineArgs
23
+from vllm.exceptions import VLLMValidationError
24
from vllm.logger import current_formatter_type, init_logger
25
from vllm.platforms import current_platform
26
from vllm.utils.argparse_utils import FlexibleArgumentParser
27
28
if TYPE_CHECKING:
25
- from vllm.entrypoints.openai.engine.protocol import StreamOptions
29
+ from vllm.entrypoints.openai.engine.protocol import (
30
+ ErrorInfo,
31
+ ErrorResponse,
32
+ StreamOptions,
33
+ )
34
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
35
else:
28
- StreamOptions = object
36
+ ErrorResponse = object
37
+ ErrorInfo = object
38
LoRAModulePath = object
30
-
39
+ StreamOptions = object
40
41
logger = init_logger(__name__)
42
@@ -291,3 +300,59 @@ def log_version_and_model(lgr: Logger, version: str, model_name: str) -> None:
300
message = logo_template.substitute(colors)
301
302
lgr.info(message, version, model_name)
303
+
304
+
305
+def create_error_response(
306
+ message: str | Exception,
307
+ err_type: str = "BadRequestError",
308
+ status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
309
+ param: str | None = None,
310
+ log_error_stack: bool = False,
311
+) -> "ErrorResponse":
312
+ exc: Exception | None = None
313
+
314
+ from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse
315
+
316
+ if isinstance(message, Exception):
317
+ exc = message
318
+
319
+ if isinstance(exc, VLLMValidationError):
320
+ err_type = "BadRequestError"
321
+ status_code = HTTPStatus.BAD_REQUEST
322
+ param = exc.parameter
323
+ elif isinstance(exc, (ValueError, TypeError, RuntimeError, OverflowError)):
324
+ # Common validation errors from user input
325
+ err_type = "BadRequestError"
326
+ status_code = HTTPStatus.BAD_REQUEST
327
+ param = None
328
+ elif isinstance(exc, NotImplementedError):
329
+ err_type = "NotImplementedError"
330
+ status_code = HTTPStatus.NOT_IMPLEMENTED
331
+ param = None
332
+ elif exc.__class__.__name__ == "TemplateError":
333
+ # jinja2.TemplateError (avoid importing jinja2)
334
+ err_type = "BadRequestError"
335
+ status_code = HTTPStatus.BAD_REQUEST
336
+ param = None
337
+ else:
338
+ err_type = "InternalServerError"
339
+ status_code = HTTPStatus.INTERNAL_SERVER_ERROR
340
+ param = None
341
+
342
+ message = str(exc)
343
+
344
+ if log_error_stack:
345
+ exc_type, _, _ = sys.exc_info()
346
+ if exc_type is not None:
347
+ traceback.print_exc()
348
+ else:
349
+ traceback.print_stack()
350
+
351
+ return ErrorResponse(
352
+ error=ErrorInfo(
353
+ message=sanitize_message(message),
354
+ type=err_type,
355
+ code=status_code.value,
356
+ param=param,
357
+ )
358
+ )