@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c831911be2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 9, 14:45 UTC

[Frontend] Reduce mixin usage in serving pooling (#33101)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 27, 2026 at 11:50 UTC c831911be29179bdc5042428556e6a43d191d169
3 files changed +134 -239
vllm/entrypoints/openai/engine/serving.py
+13 -48
@@ -64,13 +64,12 @@ from vllm.entrypoints.openai.translations.protocol import (
64 from vllm.entrypoints.pooling.classify.protocol import (
65 ClassificationChatRequest,
66 ClassificationCompletionRequest,
67 - ClassificationRequest,
67 ClassificationResponse,
68 )
69 from vllm.entrypoints.pooling.embed.protocol import (
70 + EmbeddingBytesResponse,
71 EmbeddingChatRequest,
72 EmbeddingCompletionRequest,
73 - EmbeddingRequest,
73 EmbeddingResponse,
74 )
75 from vllm.entrypoints.pooling.pooling.protocol import (
@@ -170,6 +169,7 @@ AnyResponse: TypeAlias = (
169 CompletionResponse
170 | ChatCompletionResponse
171 | EmbeddingResponse
172 + | EmbeddingBytesResponse
173 | TranscriptionResponse
174 | TokenizeResponse
175 | PoolingResponse
@@ -183,51 +183,21 @@ RequestT = TypeVar("RequestT", bound=AnyRequest)
183
184
185 @dataclass(kw_only=True)
186 -class RequestProcessingMixin:
187 - """
188 - Mixin for request processing,
189 - handling prompt preparation and engine input.
190 - """
191 -
192 - engine_prompts: list[TokensPrompt] | None = field(default_factory=list)
193 -
194 -
195 -@dataclass(kw_only=True)
196 -class ResponseGenerationMixin:
197 - """
198 - Mixin for response generation,
199 - managing result generators and final batch results.
200 - """
201 -
202 - result_generator: (
203 - AsyncGenerator[tuple[int, RequestOutput | PoolingRequestOutput], None] | None
204 - ) = None
205 - final_res_batch: list[RequestOutput | PoolingRequestOutput] = field(
206 - default_factory=list
207 - )
208 -
209 - model_config = ConfigDict(arbitrary_types_allowed=True)
210 -
211 -
212 -@dataclass(kw_only=True)
213 -class ServeContext(RequestProcessingMixin, ResponseGenerationMixin, Generic[RequestT]):
186 +class ServeContext(Generic[RequestT]):
187 request: RequestT
188 raw_request: Request | None = None
189 model_name: str
190 request_id: str
191 created_time: int = field(default_factory=lambda: int(time.time()))
192 lora_request: LoRARequest | None = None
193 + engine_prompts: list[TokensPrompt] | None = None
194
195 + result_generator: AsyncGenerator[tuple[int, PoolingRequestOutput], None] | None = (
196 + None
197 + )
198 + final_res_batch: list[PoolingRequestOutput] = field(default_factory=list)
199
222 -@dataclass(kw_only=True)
223 -class ClassificationServeContext(ServeContext[ClassificationRequest]):
224 - pass
225 -
226 -
227 -@dataclass(kw_only=True)
228 -class EmbeddingServeContext(ServeContext[EmbeddingRequest]):
229 - chat_template: str | None = None
230 - chat_template_content_format: ChatTemplateContentFormatOption
200 + model_config = ConfigDict(arbitrary_types_allowed=True)
201
202
203 class OpenAIServing:
@@ -605,10 +575,7 @@ class OpenAIServing:
575 self,
576 ctx: ServeContext,
577 ) -> AnyResponse | ErrorResponse:
608 - generation: AsyncGenerator[AnyResponse | ErrorResponse, None]
609 - generation = self._pipeline(ctx)
610 -
611 - async for response in generation:
578 + async for response in self._pipeline(ctx):
579 return response
580
581 return self.create_error_response("No response yielded from pipeline")
@@ -667,9 +634,7 @@ class OpenAIServing:
634 ctx: ServeContext,
635 ) -> ErrorResponse | None:
636 """Schedule the request and get the result generator."""
670 - generators: list[
671 - AsyncGenerator[RequestOutput | PoolingRequestOutput, None]
672 - ] = []
637 + generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
638
639 try:
640 trace_headers = (
@@ -723,7 +688,7 @@ class OpenAIServing:
688 return self.create_error_response("Engine prompts not available")
689
690 num_prompts = len(ctx.engine_prompts)
726 - final_res_batch: list[RequestOutput | PoolingRequestOutput | None]
691 + final_res_batch: list[PoolingRequestOutput | None]
692 final_res_batch = [None] * num_prompts
693
694 if ctx.result_generator is None:
@@ -1011,7 +976,7 @@ class OpenAIServing:
976
977 def _validate_input(
978 self,
1014 - request: AnyRequest,
979 + request: object,
980 input_ids: list[int],
981 input_text: str,
982 ) -> TokensPrompt:
vllm/entrypoints/pooling/classify/serving.py
+57 -89
@@ -2,7 +2,7 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 from http import HTTPStatus
5 -from typing import cast
5 +from typing import Final, cast
6
7 import jinja2
8 import numpy as np
@@ -11,18 +11,8 @@ from fastapi import Request
11 from vllm.engine.protocol import EngineClient
12 from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
13 from vllm.entrypoints.logger import RequestLogger
14 -from vllm.entrypoints.openai.chat_completion.protocol import (
15 - ChatCompletionRequest,
16 -)
17 -from vllm.entrypoints.openai.engine.protocol import (
18 - ErrorResponse,
19 - UsageInfo,
20 -)
21 -from vllm.entrypoints.openai.engine.serving import (
22 - ClassificationServeContext,
23 - OpenAIServing,
24 - ServeContext,
25 -)
14 +from vllm.entrypoints.openai.engine.protocol import ErrorResponse, UsageInfo
15 +from vllm.entrypoints.openai.engine.serving import OpenAIServing, ServeContext
16 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
17 from vllm.entrypoints.pooling.classify.protocol import (
18 ClassificationChatRequest,
@@ -39,60 +29,68 @@ from vllm.pooling_params import PoolingParams
29 logger = init_logger(__name__)
30
31
42 -class ClassificationMixin(OpenAIServing):
43 - chat_template: str | None
44 - chat_template_content_format: ChatTemplateContentFormatOption
45 - trust_request_chat_template: bool
32 +ClassificationServeContext = ServeContext[ClassificationRequest]
33 +
34 +
35 +class ServingClassification(OpenAIServing):
36 + request_id_prefix = "classify"
37 +
38 + def __init__(
39 + self,
40 + engine_client: EngineClient,
41 + models: OpenAIServingModels,
42 + *,
43 + request_logger: RequestLogger | None,
44 + chat_template: str | None = None,
45 + chat_template_content_format: ChatTemplateContentFormatOption = "auto",
46 + trust_request_chat_template: bool = False,
47 + log_error_stack: bool = False,
48 + ) -> None:
49 + super().__init__(
50 + engine_client=engine_client,
51 + models=models,
52 + request_logger=request_logger,
53 + log_error_stack=log_error_stack,
54 + )
55 +
56 + self.chat_template = chat_template
57 + self.chat_template_content_format: Final = chat_template_content_format
58 + self.trust_request_chat_template = trust_request_chat_template
59
60 async def _preprocess(
61 self,
49 - ctx: ServeContext,
62 + ctx: ClassificationServeContext,
63 ) -> ErrorResponse | None:
64 """
65 Process classification inputs: tokenize text, resolve adapters,
66 and prepare model-specific inputs.
67 """
55 - ctx = cast(ClassificationServeContext, ctx)
68 try:
57 - request_obj = ctx.request
58 -
59 - if isinstance(request_obj, ClassificationChatRequest):
60 - chat_request = request_obj
61 - messages = chat_request.messages
62 - trust_request_chat_template = getattr(
63 - self,
64 - "trust_request_chat_template",
65 - False,
66 - )
67 - ret = self._validate_chat_template(
68 - request_chat_template=chat_request.chat_template,
69 - chat_template_kwargs=chat_request.chat_template_kwargs,
70 - trust_request_chat_template=trust_request_chat_template,
69 + ctx.lora_request = self._maybe_get_adapters(ctx.request)
70 +
71 + if isinstance(ctx.request, ClassificationChatRequest):
72 + error_check_ret = self._validate_chat_template(
73 + request_chat_template=ctx.request.chat_template,
74 + chat_template_kwargs=ctx.request.chat_template_kwargs,
75 + trust_request_chat_template=self.trust_request_chat_template,
76 )
72 - if ret:
73 - return ret
77 + if error_check_ret:
78 + return error_check_ret
79
80 _, engine_prompts = await self._preprocess_chat(
76 - cast(ChatCompletionRequest, chat_request),
81 + ctx.request,
82 self.renderer,
78 - messages,
79 - chat_template=(
80 - chat_request.chat_template
81 - or getattr(self, "chat_template", None)
82 - ),
83 - chat_template_content_format=cast(
84 - ChatTemplateContentFormatOption,
85 - getattr(self, "chat_template_content_format", "auto"),
86 - ),
87 - add_generation_prompt=chat_request.add_generation_prompt,
88 - continue_final_message=chat_request.continue_final_message,
89 - add_special_tokens=chat_request.add_special_tokens,
83 + ctx.request.messages,
84 + chat_template=ctx.request.chat_template or self.chat_template,
85 + chat_template_content_format=self.chat_template_content_format,
86 + add_generation_prompt=ctx.request.add_generation_prompt,
87 + continue_final_message=ctx.request.continue_final_message,
88 + add_special_tokens=ctx.request.add_special_tokens,
89 )
90 ctx.engine_prompts = engine_prompts
91
93 - elif isinstance(request_obj, ClassificationCompletionRequest):
94 - completion_request = request_obj
95 - input_data = completion_request.input
92 + elif isinstance(ctx.request, ClassificationCompletionRequest):
93 + input_data = ctx.request.input
94 if input_data in (None, ""):
95 return self.create_error_response(
96 "Input or messages must be provided",
@@ -106,13 +104,10 @@ class ClassificationMixin(OpenAIServing):
104 prompt_input = cast(str | list[str], input_data)
105 ctx.engine_prompts = await renderer.render_prompt(
106 prompt_or_prompts=prompt_input,
109 - config=self._build_render_config(completion_request),
107 + config=self._build_render_config(ctx.request),
108 )
109 else:
112 - return self.create_error_response(
113 - "Invalid classification request type",
114 - status_code=HTTPStatus.BAD_REQUEST,
115 - )
110 + return self.create_error_response("Invalid classification request type")
111
112 return None
113
@@ -122,13 +117,14 @@ class ClassificationMixin(OpenAIServing):
117
118 def _build_response(
119 self,
125 - ctx: ServeContext,
120 + ctx: ClassificationServeContext,
121 ) -> ClassificationResponse | ErrorResponse:
122 """
123 Convert model outputs to a formatted classification response
124 with probabilities and labels.
125 """
131 - ctx = cast(ClassificationServeContext, ctx)
126 + id2label = getattr(self.model_config.hf_config, "id2label", {})
127 +
128 items: list[ClassificationData] = []
129 num_prompt_tokens = 0
130
@@ -139,9 +135,7 @@ class ClassificationMixin(OpenAIServing):
135
136 probs = classify_res.probs
137 predicted_index = int(np.argmax(probs))
142 - label = getattr(self.model_config.hf_config, "id2label", {}).get(
143 - predicted_index
144 - )
138 + label = id2label.get(predicted_index)
139
140 item = ClassificationData(
141 index=idx,
@@ -174,32 +168,6 @@ class ClassificationMixin(OpenAIServing):
168 add_special_tokens=request.add_special_tokens,
169 )
170
177 -
178 -class ServingClassification(ClassificationMixin):
179 - request_id_prefix = "classify"
180 -
181 - def __init__(
182 - self,
183 - engine_client: EngineClient,
184 - models: OpenAIServingModels,
185 - *,
186 - request_logger: RequestLogger | None,
187 - chat_template: str | None = None,
188 - chat_template_content_format: ChatTemplateContentFormatOption = "auto",
189 - trust_request_chat_template: bool = False,
190 - log_error_stack: bool = False,
191 - ) -> None:
192 - super().__init__(
193 - engine_client=engine_client,
194 - models=models,
195 - request_logger=request_logger,
196 - log_error_stack=log_error_stack,
197 - )
198 -
199 - self.chat_template = chat_template
200 - self.chat_template_content_format = chat_template_content_format
201 - self.trust_request_chat_template = trust_request_chat_template
202 -
171 async def create_classify(
172 self,
173 request: ClassificationRequest,
@@ -215,11 +183,11 @@ class ServingClassification(ClassificationMixin):
183 request_id=request_id,
184 )
185
218 - return await super().handle(ctx) # type: ignore
186 + return await self.handle(ctx) # type: ignore[return-value]
187
188 def _create_pooling_params(
189 self,
222 - ctx: ServeContext[ClassificationRequest],
190 + ctx: ClassificationServeContext,
191 ) -> PoolingParams | ErrorResponse:
192 pooling_params = super()._create_pooling_params(ctx)
193 if isinstance(pooling_params, ErrorResponse):
vllm/entrypoints/pooling/embed/serving.py
+64 -102
@@ -6,21 +6,13 @@ from typing import Any, Final, cast
6
7 import torch
8 from fastapi import Request
9 -from fastapi.responses import Response
10 -from typing_extensions import assert_never, override
9 +from typing_extensions import assert_never
10
11 from vllm.engine.protocol import EngineClient
12 from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
13 from vllm.entrypoints.logger import RequestLogger
15 -from vllm.entrypoints.openai.engine.protocol import (
16 - ErrorResponse,
17 - UsageInfo,
18 -)
19 -from vllm.entrypoints.openai.engine.serving import (
20 - EmbeddingServeContext,
21 - OpenAIServing,
22 - ServeContext,
23 -)
14 +from vllm.entrypoints.openai.engine.protocol import ErrorResponse, UsageInfo
15 +from vllm.entrypoints.openai.engine.serving import OpenAIServing, ServeContext
16 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
17 from vllm.entrypoints.pooling.embed.protocol import (
18 EmbeddingBytesResponse,
@@ -33,19 +25,11 @@ from vllm.entrypoints.pooling.embed.protocol import (
25 from vllm.entrypoints.renderer import RenderConfig
26 from vllm.inputs.data import TokensPrompt
27 from vllm.logger import init_logger
36 -from vllm.outputs import (
37 - EmbeddingRequestOutput,
38 - PoolingOutput,
39 - PoolingRequestOutput,
40 - RequestOutput,
41 -)
28 +from vllm.outputs import PoolingOutput, PoolingRequestOutput
29 from vllm.pooling_params import PoolingParams
30 from vllm.utils.async_utils import merge_async_iterators
31 from vllm.utils.collection_utils import chunk_list
32 from vllm.utils.serial_utils import (
46 - EmbedDType,
47 - EncodingFormat,
48 - Endianness,
33 encode_pooling_bytes,
34 encode_pooling_output,
35 )
@@ -53,9 +37,33 @@ from vllm.utils.serial_utils import (
37 logger = init_logger(__name__)
38
39
56 -class EmbeddingMixin(OpenAIServing):
57 - def __init__(self, *args, **kwargs):
58 - super().__init__(*args, **kwargs)
40 +EmbeddingServeContext = ServeContext[EmbeddingRequest]
41 +
42 +
43 +class OpenAIServingEmbedding(OpenAIServing):
44 + request_id_prefix = "embd"
45 +
46 + def __init__(
47 + self,
48 + engine_client: EngineClient,
49 + models: OpenAIServingModels,
50 + *,
51 + request_logger: RequestLogger | None,
52 + chat_template: str | None,
53 + chat_template_content_format: ChatTemplateContentFormatOption,
54 + trust_request_chat_template: bool = False,
55 + log_error_stack: bool = False,
56 + ) -> None:
57 + super().__init__(
58 + engine_client=engine_client,
59 + models=models,
60 + request_logger=request_logger,
61 + log_error_stack=log_error_stack,
62 + )
63 +
64 + self.chat_template = chat_template
65 + self.chat_template_content_format: Final = chat_template_content_format
66 + self.trust_request_chat_template = trust_request_chat_template
67
68 pooler_config = self.model_config.pooler_config
69
@@ -69,32 +77,41 @@ class EmbeddingMixin(OpenAIServing):
77 else None
78 )
79
72 - @override
80 async def _preprocess(
81 self,
75 - ctx: ServeContext,
82 + ctx: EmbeddingServeContext,
83 ) -> ErrorResponse | None:
77 - ctx = cast(EmbeddingServeContext, ctx)
84 try:
85 ctx.lora_request = self._maybe_get_adapters(ctx.request)
86
87 if isinstance(ctx.request, EmbeddingChatRequest):
88 + error_check_ret = self._validate_chat_template(
89 + request_chat_template=ctx.request.chat_template,
90 + chat_template_kwargs=ctx.request.chat_template_kwargs,
91 + trust_request_chat_template=self.trust_request_chat_template,
92 + )
93 + if error_check_ret is not None:
94 + return error_check_ret
95 +
96 _, ctx.engine_prompts = await self._preprocess_chat(
97 ctx.request,
98 self.renderer,
99 ctx.request.messages,
86 - chat_template=ctx.request.chat_template or ctx.chat_template,
87 - chat_template_content_format=ctx.chat_template_content_format,
100 + chat_template=ctx.request.chat_template or self.chat_template,
101 + chat_template_content_format=self.chat_template_content_format,
102 add_generation_prompt=ctx.request.add_generation_prompt,
103 continue_final_message=ctx.request.continue_final_message,
104 add_special_tokens=ctx.request.add_special_tokens,
105 )
92 - else:
106 + elif isinstance(ctx.request, EmbeddingCompletionRequest):
107 renderer = self._get_completion_renderer()
108 ctx.engine_prompts = await renderer.render_prompt(
109 prompt_or_prompts=ctx.request.input,
110 config=self._build_render_config(ctx.request),
111 )
112 + else:
113 + return self.create_error_response("Invalid classification request type")
114 +
115 return None
116 except (ValueError, TypeError) as e:
117 logger.exception("Error in preprocessing prompt inputs")
@@ -113,16 +130,15 @@ class EmbeddingMixin(OpenAIServing):
130 add_special_tokens=request.add_special_tokens,
131 )
132
116 - @override
133 def _build_response(
134 self,
119 - ctx: ServeContext,
120 - ) -> EmbeddingResponse | Response | ErrorResponse:
121 - final_res_batch_checked = cast(list[PoolingRequestOutput], ctx.final_res_batch)
135 + ctx: EmbeddingServeContext,
136 + ) -> EmbeddingResponse | EmbeddingBytesResponse | ErrorResponse:
137 + final_res_batch_checked = ctx.final_res_batch
138
123 - encoding_format: EncodingFormat = ctx.request.encoding_format
124 - embed_dtype: EmbedDType = ctx.request.embed_dtype
125 - endianness: Endianness = ctx.request.endianness
139 + encoding_format = ctx.request.encoding_format
140 + embed_dtype = ctx.request.embed_dtype
141 + endianness = ctx.request.endianness
142
143 def encode_float_base64():
144 items: list[EmbeddingResponseData] = []
@@ -203,8 +219,8 @@ class EmbeddingMixin(OpenAIServing):
219 self,
220 ctx: EmbeddingServeContext,
221 token_ids: list[int],
206 - pooling_params,
207 - trace_headers,
222 + pooling_params: PoolingParams,
223 + trace_headers: Mapping[str, str] | None,
224 prompt_idx: int,
225 ) -> list[AsyncGenerator[PoolingRequestOutput, None]]:
226 """Process a single prompt using chunked processing."""
@@ -246,7 +262,7 @@ class EmbeddingMixin(OpenAIServing):
262
263 def _validate_input(
264 self,
249 - request,
265 + request: object,
266 input_ids: list[int],
267 input_text: str,
268 ) -> TokensPrompt:
@@ -326,7 +342,7 @@ class EmbeddingMixin(OpenAIServing):
342 pooling_params: PoolingParams,
343 trace_headers: Mapping[str, str] | None,
344 prompt_index: int,
329 - ) -> AsyncGenerator[RequestOutput | PoolingRequestOutput, None]:
345 + ) -> AsyncGenerator[PoolingRequestOutput, None]:
346 """Create a generator for a single prompt using standard processing."""
347 request_id_item = f"{ctx.request_id}-{prompt_index}"
348
@@ -347,7 +363,6 @@ class EmbeddingMixin(OpenAIServing):
363 priority=getattr(ctx.request, "priority", 0),
364 )
365
350 - @override
366 async def _prepare_generators(
367 self,
368 ctx: ServeContext,
@@ -363,9 +378,7 @@ class EmbeddingMixin(OpenAIServing):
378 return await super()._prepare_generators(ctx)
379
380 # Custom logic for chunked processing
366 - generators: list[
367 - AsyncGenerator[RequestOutput | PoolingRequestOutput, None]
368 - ] = []
381 + generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
382
383 try:
384 trace_headers = (
@@ -419,10 +432,9 @@ class EmbeddingMixin(OpenAIServing):
432 # TODO: Use a vllm-specific Validation Error
433 return self.create_error_response(str(e))
434
422 - @override
435 async def _collect_batch(
436 self,
425 - ctx: ServeContext,
437 + ctx: EmbeddingServeContext,
438 ) -> ErrorResponse | None:
439 """Collect and aggregate batch results
440 with support for chunked processing.
@@ -431,7 +443,6 @@ class EmbeddingMixin(OpenAIServing):
443 minimize memory usage.
444 For regular requests, collects results normally.
445 """
434 - ctx = cast(EmbeddingServeContext, ctx)
446 try:
447 if ctx.engine_prompts is None:
448 return self.create_error_response("Engine prompts not available")
@@ -527,12 +538,10 @@ class EmbeddingMixin(OpenAIServing):
538 except (ValueError, IndexError):
539 prompt_idx = result_idx # Fallback to result_idx
540
530 - short_prompts_results[prompt_idx] = cast(
531 - PoolingRequestOutput, result
532 - )
541 + short_prompts_results[prompt_idx] = result
542
543 # Finalize aggregated results
535 - final_res_batch: list[PoolingRequestOutput | EmbeddingRequestOutput] = []
544 + final_res_batch: list[PoolingRequestOutput] = []
545 num_prompts = len(ctx.engine_prompts)
546
547 for prompt_idx in range(num_prompts):
@@ -580,49 +589,19 @@ class EmbeddingMixin(OpenAIServing):
589 f"Failed to aggregate chunks for prompt {prompt_idx}"
590 )
591 elif prompt_idx in short_prompts_results:
583 - final_res_batch.append(
584 - cast(PoolingRequestOutput, short_prompts_results[prompt_idx])
585 - )
592 + final_res_batch.append(short_prompts_results[prompt_idx])
593 else:
594 return self.create_error_response(
595 f"Result not found for prompt {prompt_idx}"
596 )
597
591 - ctx.final_res_batch = cast(
592 - list[RequestOutput | PoolingRequestOutput], final_res_batch
593 - )
598 + ctx.final_res_batch = final_res_batch
599
600 return None
601
602 except Exception as e:
603 return self.create_error_response(str(e))
604
600 -
601 -class OpenAIServingEmbedding(EmbeddingMixin):
602 - request_id_prefix = "embd"
603 -
604 - def __init__(
605 - self,
606 - engine_client: EngineClient,
607 - models: OpenAIServingModels,
608 - *,
609 - request_logger: RequestLogger | None,
610 - chat_template: str | None,
611 - chat_template_content_format: ChatTemplateContentFormatOption,
612 - trust_request_chat_template: bool = False,
613 - log_error_stack: bool = False,
614 - ) -> None:
615 - super().__init__(
616 - engine_client=engine_client,
617 - models=models,
618 - request_logger=request_logger,
619 - log_error_stack=log_error_stack,
620 - )
621 -
622 - self.chat_template = chat_template
623 - self.chat_template_content_format: Final = chat_template_content_format
624 - self.trust_request_chat_template = trust_request_chat_template
625 -
605 async def create_embedding(
606 self,
607 request: EmbeddingRequest,
@@ -645,16 +624,13 @@ class OpenAIServingEmbedding(EmbeddingMixin):
624 raw_request=raw_request,
625 model_name=model_name,
626 request_id=request_id,
648 - chat_template=self.chat_template,
649 - chat_template_content_format=self.chat_template_content_format,
627 )
628
652 - return await super().handle(ctx) # type: ignore
629 + return await self.handle(ctx) # type: ignore[return-value]
630
654 - @override
631 def _create_pooling_params(
632 self,
657 - ctx: ServeContext[EmbeddingRequest],
633 + ctx: EmbeddingServeContext,
634 ) -> PoolingParams | ErrorResponse:
635 pooling_params = super()._create_pooling_params(ctx)
636 if isinstance(pooling_params, ErrorResponse):
@@ -666,17 +642,3 @@ class OpenAIServingEmbedding(EmbeddingMixin):
642 return self.create_error_response(str(e))
643
644 return pooling_params
669 -
670 - async def _preprocess(
671 - self,
672 - ctx: ServeContext,
673 - ) -> ErrorResponse | None:
674 - if isinstance(ctx.request, EmbeddingChatRequest):
675 - error_check_ret = self._validate_chat_template(
676 - request_chat_template=ctx.request.chat_template,
677 - chat_template_kwargs=ctx.request.chat_template_kwargs,
678 - trust_request_chat_template=self.trust_request_chat_template,
679 - )
680 - if error_check_ret is not None:
681 - return error_check_ret
682 - return await super()._preprocess(ctx)