A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 14:30 UTC
[Frontend] Complete OpenAI render delegation (#37287)
Signed-off-by: Sage Ahrac <sagiahrak@gmail.com>
Sage committed
Mar 17, 2026 at 15:53 UTC
59192dfd39512b9d05563709cfc0fe78746a8fe3
8 files changed
+140
-244
tests/entrypoints/openai/test_serving_responses.py
+4
@@ -159,6 +159,7 @@ class TestInitializeToolSessions:
159
instance = OpenAIServingResponses(
160
engine_client=engine_client,
161
models=models,
162
+ openai_serving_render=MagicMock(),
163
request_logger=None,
164
chat_template=None,
165
chat_template_content_format="auto",
@@ -245,6 +246,7 @@ class TestValidateGeneratorInput:
246
instance = OpenAIServingResponses(
247
engine_client=engine_client,
248
models=models,
249
+ openai_serving_render=MagicMock(),
250
request_logger=None,
251
chat_template=None,
252
chat_template_content_format="auto",
@@ -308,6 +310,7 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
310
serving = OpenAIServingResponses(
311
engine_client=engine_client,
312
models=models,
313
+ openai_serving_render=MagicMock(),
314
request_logger=None,
315
chat_template=None,
316
chat_template_content_format="auto",
@@ -607,6 +610,7 @@ def _make_serving_instance_with_reasoning():
610
serving = OpenAIServingResponses(
611
engine_client=engine_client,
612
models=models,
613
+ openai_serving_render=MagicMock(),
614
request_logger=None,
615
chat_template=None,
616
chat_template_content_format="auto",
vllm/entrypoints/openai/engine/serving.py
+3
-226
@@ -4,7 +4,7 @@ import asyncio
4
import contextlib
5
import json
6
import time
7
-from collections.abc import AsyncGenerator, Callable, Mapping, Sequence
7
+from collections.abc import AsyncGenerator, Callable, Mapping
8
from dataclasses import dataclass, field
9
from http import HTTPStatus
10
from typing import Any, ClassVar, Generic, Protocol, TypeAlias, TypeVar
@@ -22,9 +22,7 @@ from vllm.beam_search import BeamSearchSequence, create_sort_beams_key_function
22
from vllm.config import ModelConfig
23
from vllm.engine.protocol import EngineClient
24
from vllm.entrypoints.chat_utils import (
25
- ChatCompletionMessageParam,
25
ChatTemplateContentFormatOption,
27
- ConversationMessage,
26
)
27
from vllm.entrypoints.logger import RequestLogger
28
from vllm.entrypoints.openai.chat_completion.protocol import (
@@ -43,19 +41,9 @@ from vllm.entrypoints.openai.engine.protocol import (
41
GenerationError,
42
)
43
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
46
-from vllm.entrypoints.openai.responses.context import (
47
- ConversationContext,
48
- HarmonyContext,
49
- ParsableContext,
50
- StreamingHarmonyContext,
51
-)
44
from vllm.entrypoints.openai.responses.protocol import (
53
- ResponseInputOutputItem,
45
ResponsesRequest,
46
)
56
-from vllm.entrypoints.openai.responses.utils import (
57
- construct_input_messages,
58
-)
47
from vllm.entrypoints.openai.speech_to_text.protocol import (
48
TranscriptionRequest,
49
TranscriptionResponse,
@@ -82,26 +70,22 @@ from vllm.entrypoints.serve.tokenize.protocol import (
70
TokenizeCompletionRequest,
71
TokenizeResponse,
72
)
85
-from vllm.entrypoints.utils import create_error_response, get_max_tokens
73
+from vllm.entrypoints.utils import create_error_response
74
from vllm.exceptions import VLLMValidationError
75
from vllm.inputs.data import (
76
ProcessorInputs,
77
PromptType,
90
- SingletonPrompt,
78
TokensPrompt,
92
- token_inputs,
79
)
80
from vllm.logger import init_logger
81
from vllm.logprobs import Logprob, PromptLogprobs
82
from vllm.lora.request import LoRARequest
83
from vllm.outputs import CompletionOutput, PoolingRequestOutput, RequestOutput
84
from vllm.pooling_params import PoolingParams
99
-from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
85
+from vllm.renderers import ChatParams, TokenizeParams
86
from vllm.renderers.inputs.preprocess import (
87
extract_prompt_components,
88
extract_prompt_len,
103
- parse_model_prompt,
104
- prompt_to_seq,
89
)
90
from vllm.sampling_params import BeamSearchParams, SamplingParams
91
from vllm.tokenizers import TokenizerLike
@@ -116,7 +100,6 @@ from vllm.utils.async_utils import (
100
collect_from_async_generator,
101
merge_async_iterators,
102
)
119
-from vllm.utils.mistral import is_mistral_tokenizer
103
104
logger = init_logger(__name__)
105
@@ -823,109 +806,6 @@ class OpenAIServing:
806
# Apply server defaults first, then request kwargs override.
807
return default_chat_template_kwargs | request_chat_template_kwargs
808
826
- async def _preprocess_completion(
827
- self,
828
- request: RendererRequest,
829
- prompt_input: str | list[str] | list[int] | list[list[int]] | None,
830
- prompt_embeds: bytes | list[bytes] | None,
831
- ) -> list[ProcessorInputs]:
832
- prompts = list[SingletonPrompt | bytes]()
833
- if prompt_embeds is not None: # embeds take higher priority
834
- prompts.extend(prompt_to_seq(prompt_embeds))
835
- if prompt_input is not None:
836
- prompts.extend(prompt_to_seq(prompt_input))
837
-
838
- return await self._preprocess_cmpl(request, prompts)
839
-
840
- async def _preprocess_cmpl(
841
- self,
842
- request: RendererRequest,
843
- prompts: Sequence[PromptType | bytes],
844
- ) -> list[ProcessorInputs]:
845
- renderer = self.renderer
846
- model_config = self.model_config
847
-
848
- parsed_prompts = [
849
- (
850
- prompt
851
- if isinstance(prompt, bytes)
852
- else parse_model_prompt(model_config, prompt)
853
- )
854
- for prompt in prompts
855
- ]
856
- tok_params = request.build_tok_params(model_config)
857
-
858
- return await renderer.render_cmpl_async(
859
- parsed_prompts,
860
- tok_params,
861
- prompt_extras={
862
- k: v
863
- for k in ("mm_processor_kwargs", "cache_salt")
864
- if (v := getattr(request, k, None)) is not None
865
- },
866
- )
867
-
868
- async def _preprocess_chat(
869
- self,
870
- request: RendererChatRequest,
871
- messages: list[ChatCompletionMessageParam],
872
- default_template: str | None,
873
- default_template_content_format: ChatTemplateContentFormatOption,
874
- default_template_kwargs: dict[str, Any] | None,
875
- tool_dicts: list[dict[str, Any]] | None = None,
876
- tool_parser: Callable[[TokenizerLike], ToolParser] | None = None,
877
- ) -> tuple[list[ConversationMessage], list[ProcessorInputs]]:
878
- renderer = self.renderer
879
-
880
- default_template_kwargs = merge_kwargs(
881
- default_template_kwargs,
882
- dict(
883
- tools=tool_dicts,
884
- tokenize=is_mistral_tokenizer(renderer.tokenizer),
885
- ),
886
- )
887
-
888
- mm_config = self.model_config.multimodal_config
889
-
890
- tok_params = request.build_tok_params(self.model_config)
891
- chat_params = request.build_chat_params(
892
- default_template, default_template_content_format
893
- ).with_defaults(
894
- default_template_kwargs,
895
- default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
896
- default_mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
897
- )
898
-
899
- (conversation,), (engine_prompt,) = await renderer.render_chat_async(
900
- [messages],
901
- chat_params,
902
- tok_params,
903
- prompt_extras={
904
- k: v
905
- for k in ("mm_processor_kwargs", "cache_salt")
906
- if (v := getattr(request, k, None)) is not None
907
- },
908
- )
909
-
910
- # tool parsing is done only if a tool_parser has been set and if
911
- # tool_choice is not "none" (if tool_choice is "none" but a tool_parser
912
- # is set, we want to prevent parsing a tool_call hallucinated by the LLM
913
- if tool_parser is not None:
914
- tool_choice = getattr(request, "tool_choice", "none")
915
- if tool_choice != "none":
916
- if not isinstance(request, ChatCompletionRequest | ResponsesRequest):
917
- msg = (
918
- "Tool usage is only supported for Chat Completions API "
919
- "or Responses API requests."
920
- )
921
- raise NotImplementedError(msg)
922
-
923
- # TODO: Update adjust_request to accept ResponsesRequest
924
- tokenizer = renderer.get_tokenizer()
925
- request = tool_parser(tokenizer).adjust_request(request=request) # type: ignore[arg-type]
926
-
927
- return conversation, [engine_prompt]
928
-
809
def _extract_prompt_components(self, prompt: PromptType | ProcessorInputs):
810
return extract_prompt_components(self.model_config, prompt)
811
@@ -935,109 +815,6 @@ class OpenAIServing:
815
def _extract_prompt_len(self, prompt: ProcessorInputs):
816
return extract_prompt_len(self.model_config, prompt)
817
938
- async def _render_next_turn(
939
- self,
940
- request: ResponsesRequest,
941
- messages: list[ResponseInputOutputItem],
942
- tool_dicts: list[dict[str, Any]] | None,
943
- tool_parser: Callable[[TokenizerLike], ToolParser] | None,
944
- chat_template: str | None,
945
- chat_template_content_format: ChatTemplateContentFormatOption,
946
- ):
947
- new_messages = construct_input_messages(
948
- request_input=messages,
949
- )
950
-
951
- _, engine_prompts = await self._preprocess_chat(
952
- request,
953
- new_messages,
954
- default_template=chat_template,
955
- default_template_content_format=chat_template_content_format,
956
- default_template_kwargs=None,
957
- tool_dicts=tool_dicts,
958
- tool_parser=tool_parser,
959
- )
960
- return engine_prompts
961
-
962
- async def _generate_with_builtin_tools(
963
- self,
964
- request_id: str,
965
- engine_prompt: ProcessorInputs,
966
- sampling_params: SamplingParams,
967
- context: ConversationContext,
968
- lora_request: LoRARequest | None = None,
969
- priority: int = 0,
970
- trace_headers: Mapping[str, str] | None = None,
971
- ):
972
- max_model_len = self.model_config.max_model_len
973
-
974
- orig_priority = priority
975
- sub_request = 0
976
- while True:
977
- # Ensure that each sub-request has a unique request id.
978
- sub_request_id = f"{request_id}_{sub_request}"
979
-
980
- self._log_inputs(
981
- sub_request_id,
982
- engine_prompt,
983
- params=sampling_params,
984
- lora_request=lora_request,
985
- )
986
-
987
- generator = self.engine_client.generate(
988
- engine_prompt,
989
- sampling_params,
990
- sub_request_id,
991
- lora_request=lora_request,
992
- trace_headers=trace_headers,
993
- priority=priority,
994
- )
995
-
996
- async for res in generator:
997
- context.append_output(res)
998
- # NOTE(woosuk): The stop condition is handled by the engine.
999
- yield context
1000
-
1001
- if not context.need_builtin_tool_call():
1002
- # The model did not ask for a tool call, so we're done.
1003
- break
1004
-
1005
- # Call the tool and update the context with the result.
1006
- tool_output = await context.call_tool()
1007
- context.append_tool_output(tool_output)
1008
-
1009
- # TODO: uncomment this and enable tool output streaming
1010
- # yield context
1011
-
1012
- # Create inputs for the next turn.
1013
- # Render the next prompt token ids and update sampling_params.
1014
- if isinstance(context, (HarmonyContext, StreamingHarmonyContext)):
1015
- token_ids = context.render_for_completion()
1016
- engine_prompt = token_inputs(token_ids)
1017
-
1018
- sampling_params.max_tokens = max_model_len - len(token_ids)
1019
- elif isinstance(context, ParsableContext):
1020
- (engine_prompt,) = await self._render_next_turn(
1021
- context.request,
1022
- context.parser.response_messages,
1023
- context.tool_dicts,
1024
- context.tool_parser_cls,
1025
- context.chat_template,
1026
- context.chat_template_content_format,
1027
- )
1028
-
1029
- sampling_params.max_tokens = get_max_tokens(
1030
- max_model_len,
1031
- context.request.max_output_tokens,
1032
- self._extract_prompt_len(engine_prompt),
1033
- self.default_sampling_params, # type: ignore
1034
- self.override_max_tokens, # type: ignore
1035
- )
1036
-
1037
- # OPTIMIZATION
1038
- priority = orig_priority - 1
1039
- sub_request += 1
1040
-
818
def _log_inputs(
819
self,
820
request_id: str,
vllm/entrypoints/openai/generate/api_router.py
+2
@@ -80,6 +80,7 @@ async def init_generate_state(
80
OpenAIServingResponses(
81
engine_client,
82
state.openai_serving_models,
83
+ state.openai_serving_render,
84
request_logger=request_logger,
85
chat_template=resolved_chat_template,
86
chat_template_content_format=args.chat_template_content_format,
@@ -157,6 +158,7 @@ async def init_generate_state(
158
ServingTokens(
159
engine_client,
160
state.openai_serving_models,
161
+ state.openai_serving_render,
162
request_logger=request_logger,
163
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
164
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
vllm/entrypoints/openai/responses/serving.py
+112
-3
@@ -5,11 +5,11 @@ import asyncio
5
import time
6
import uuid
7
from collections import deque
8
-from collections.abc import AsyncGenerator, AsyncIterator, Callable, Sequence
8
+from collections.abc import AsyncGenerator, AsyncIterator, Callable, Mapping, Sequence
9
from contextlib import AsyncExitStack
10
from copy import copy
11
from http import HTTPStatus
12
-from typing import Final
12
+from typing import Any, Final
13
14
from fastapi import Request
15
from openai.types.responses import (
@@ -86,6 +86,7 @@ from vllm.entrypoints.openai.responses.protocol import (
86
ResponseCompletedEvent,
87
ResponseCreatedEvent,
88
ResponseInProgressEvent,
89
+ ResponseInputOutputItem,
90
ResponseInputOutputMessage,
91
ResponseReasoningPartAddedEvent,
92
ResponseReasoningPartDoneEvent,
@@ -105,16 +106,19 @@ from vllm.entrypoints.openai.responses.utils import (
106
construct_tool_dicts,
107
extract_tool_types,
108
)
109
+from vllm.entrypoints.serve.render.serving import OpenAIServingRender
110
from vllm.entrypoints.utils import get_max_tokens
111
from vllm.exceptions import VLLMValidationError
112
from vllm.inputs.data import ProcessorInputs, token_inputs
113
from vllm.logger import init_logger
114
from vllm.logprobs import Logprob as SampleLogprob
115
from vllm.logprobs import SampleLogprobs
116
+from vllm.lora.request import LoRARequest
117
from vllm.outputs import CompletionOutput
118
from vllm.parser import ParserManager
119
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
120
from vllm.tokenizers import TokenizerLike
121
+from vllm.tool_parsers import ToolParser
122
from vllm.utils import random_uuid
123
from vllm.utils.collection_utils import as_list
124
@@ -165,6 +169,7 @@ class OpenAIServingResponses(OpenAIServing):
169
self,
170
engine_client: EngineClient,
171
models: OpenAIServingModels,
172
+ openai_serving_render: OpenAIServingRender,
173
*,
174
request_logger: RequestLogger | None,
175
chat_template: str | None,
@@ -185,6 +190,7 @@ class OpenAIServingResponses(OpenAIServing):
190
return_tokens_as_token_ids=return_tokens_as_token_ids,
191
)
192
193
+ self.openai_serving_render = openai_serving_render
194
self.chat_template = chat_template
195
self.chat_template_content_format: Final = chat_template_content_format
196
self.enable_log_outputs = enable_log_outputs
@@ -587,7 +593,7 @@ class OpenAIServingResponses(OpenAIServing):
593
prev_response_output=prev_response.output if prev_response else None,
594
)
595
590
- _, engine_prompts = await self._preprocess_chat(
596
+ _, engine_prompts = await self.openai_serving_render.preprocess_chat(
597
request,
598
messages,
599
default_template=self.chat_template,
@@ -598,6 +604,109 @@ class OpenAIServingResponses(OpenAIServing):
604
)
605
return messages, engine_prompts
606
607
+ async def _render_next_turn(
608
+ self,
609
+ request: ResponsesRequest,
610
+ messages: list[ResponseInputOutputItem],
611
+ tool_dicts: list[dict[str, Any]] | None,
612
+ tool_parser: Callable[[TokenizerLike], ToolParser] | None,
613
+ chat_template: str | None,
614
+ chat_template_content_format: ChatTemplateContentFormatOption,
615
+ ):
616
+ new_messages = construct_input_messages(
617
+ request_input=messages,
618
+ )
619
+
620
+ _, engine_prompts = await self.openai_serving_render.preprocess_chat(
621
+ request,
622
+ new_messages,
623
+ default_template=chat_template,
624
+ default_template_content_format=chat_template_content_format,
625
+ default_template_kwargs=None,
626
+ tool_dicts=tool_dicts,
627
+ tool_parser=tool_parser,
628
+ )
629
+ return engine_prompts
630
+
631
+ async def _generate_with_builtin_tools(
632
+ self,
633
+ request_id: str,
634
+ engine_prompt: ProcessorInputs,
635
+ sampling_params: SamplingParams,
636
+ context: ConversationContext,
637
+ lora_request: LoRARequest | None = None,
638
+ priority: int = 0,
639
+ trace_headers: Mapping[str, str] | None = None,
640
+ ):
641
+ max_model_len = self.model_config.max_model_len
642
+
643
+ orig_priority = priority
644
+ sub_request = 0
645
+ while True:
646
+ # Ensure that each sub-request has a unique request id.
647
+ sub_request_id = f"{request_id}_{sub_request}"
648
+
649
+ self._log_inputs(
650
+ sub_request_id,
651
+ engine_prompt,
652
+ params=sampling_params,
653
+ lora_request=lora_request,
654
+ )
655
+
656
+ generator = self.engine_client.generate(
657
+ engine_prompt,
658
+ sampling_params,
659
+ sub_request_id,
660
+ lora_request=lora_request,
661
+ trace_headers=trace_headers,
662
+ priority=priority,
663
+ )
664
+
665
+ async for res in generator:
666
+ context.append_output(res)
667
+ # NOTE(woosuk): The stop condition is handled by the engine.
668
+ yield context
669
+
670
+ if not context.need_builtin_tool_call():
671
+ # The model did not ask for a tool call, so we're done.
672
+ break
673
+
674
+ # Call the tool and update the context with the result.
675
+ tool_output = await context.call_tool()
676
+ context.append_tool_output(tool_output)
677
+
678
+ # TODO: uncomment this and enable tool output streaming
679
+ # yield context
680
+
681
+ # Create inputs for the next turn.
682
+ # Render the next prompt token ids and update sampling_params.
683
+ if isinstance(context, (HarmonyContext, StreamingHarmonyContext)):
684
+ token_ids = context.render_for_completion()
685
+ engine_prompt = token_inputs(token_ids)
686
+
687
+ sampling_params.max_tokens = max_model_len - len(token_ids)
688
+ elif isinstance(context, ParsableContext):
689
+ (engine_prompt,) = await self._render_next_turn(
690
+ context.request,
691
+ context.parser.response_messages,
692
+ context.tool_dicts,
693
+ context.tool_parser_cls,
694
+ context.chat_template,
695
+ context.chat_template_content_format,
696
+ )
697
+
698
+ sampling_params.max_tokens = get_max_tokens(
699
+ max_model_len,
700
+ context.request.max_output_tokens,
701
+ self._extract_prompt_len(engine_prompt),
702
+ self.default_sampling_params, # type: ignore
703
+ self.override_max_tokens, # type: ignore
704
+ )
705
+
706
+ # OPTIMIZATION
707
+ priority = orig_priority - 1
708
+ sub_request += 1
709
+
710
def _make_request_with_harmony(
711
self,
712
request: ResponsesRequest,
vllm/entrypoints/pooling/__init__.py
+1
@@ -68,6 +68,7 @@ def init_pooling_state(
68
OpenAIServingPooling(
69
engine_client,
70
state.openai_serving_models,
71
+ state.openai_serving_render,
72
request_logger=request_logger,
73
chat_template=resolved_chat_template,
74
chat_template_content_format=args.chat_template_content_format,
vllm/entrypoints/pooling/pooling/serving.py
+7
-4
@@ -32,6 +32,7 @@ from vllm.entrypoints.pooling.utils import (
32
encode_pooling_output_base64,
33
encode_pooling_output_float,
34
)
35
+from vllm.entrypoints.serve.render.serving import OpenAIServingRender
36
from vllm.inputs import ProcessorInputs
37
from vllm.logger import init_logger
38
from vllm.outputs import PoolingRequestOutput
@@ -47,6 +48,7 @@ class OpenAIServingPooling(OpenAIServing):
48
self,
49
engine_client: EngineClient,
50
models: OpenAIServingModels,
51
+ openai_serving_render: OpenAIServingRender,
52
*,
53
request_logger: RequestLogger | None,
54
chat_template: str | None,
@@ -59,6 +61,7 @@ class OpenAIServingPooling(OpenAIServing):
61
request_logger=request_logger,
62
)
63
64
+ self.openai_serving_render = openai_serving_render
65
self.chat_template = chat_template
66
self.chat_template_content_format: Final = chat_template_content_format
67
self.trust_request_chat_template = trust_request_chat_template
@@ -101,12 +104,12 @@ class OpenAIServingPooling(OpenAIServing):
104
raw_prompts = await self.io_processor.pre_process_async(
105
prompt=validated_prompt, request_id=request_id
106
)
104
- engine_prompts = await self._preprocess_cmpl(
107
+ engine_prompts = await self.openai_serving_render.preprocess_cmpl(
108
request,
109
prompt_to_seq(raw_prompts),
110
)
111
elif isinstance(request, PoolingChatRequest):
109
- error_check_ret = self._validate_chat_template(
112
+ error_check_ret = self.openai_serving_render.validate_chat_template(
113
request_chat_template=request.chat_template,
114
chat_template_kwargs=request.chat_template_kwargs,
115
trust_request_chat_template=self.trust_request_chat_template,
@@ -114,7 +117,7 @@ class OpenAIServingPooling(OpenAIServing):
117
if error_check_ret is not None:
118
return error_check_ret
119
117
- _, engine_prompts = await self._preprocess_chat(
120
+ _, engine_prompts = await self.openai_serving_render.preprocess_chat(
121
request,
122
request.messages,
123
default_template=self.chat_template,
@@ -122,7 +125,7 @@ class OpenAIServingPooling(OpenAIServing):
125
default_template_kwargs=None,
126
)
127
elif isinstance(request, PoolingCompletionRequest):
125
- engine_prompts = await self._preprocess_completion(
128
+ engine_prompts = await self.openai_serving_render.preprocess_completion(
129
request,
130
prompt_input=request.input,
131
prompt_embeds=None,
vllm/entrypoints/serve/disagg/serving.py
+4
-1
@@ -29,6 +29,7 @@ from vllm.entrypoints.serve.disagg.protocol import (
29
GenerateResponse,
30
GenerateResponseChoice,
31
)
32
+from vllm.entrypoints.serve.render.serving import OpenAIServingRender
33
from vllm.logger import init_logger
34
from vllm.logprobs import Logprob
35
from vllm.outputs import RequestOutput
@@ -45,6 +46,7 @@ class ServingTokens(OpenAIServing):
46
self,
47
engine_client: EngineClient,
48
models: OpenAIServingModels,
49
+ openai_serving_render: OpenAIServingRender,
50
*,
51
request_logger: RequestLogger | None,
52
force_no_detokenize: bool = False,
@@ -58,6 +60,7 @@ class ServingTokens(OpenAIServing):
60
request_logger=request_logger,
61
return_tokens_as_token_ids=return_tokens_as_token_ids,
62
)
63
+ self.openai_serving_render = openai_serving_render
64
self.enable_prompt_tokens_details = enable_prompt_tokens_details
65
self.enable_log_outputs = enable_log_outputs
66
self.force_no_detokenize = force_no_detokenize
@@ -96,7 +99,7 @@ class ServingTokens(OpenAIServing):
99
if raw_request:
100
raw_request.state.request_metadata = request_metadata
101
99
- engine_prompts = await self._preprocess_completion(
102
+ engine_prompts = await self.openai_serving_render.preprocess_completion(
103
request,
104
prompt_input=request.token_ids,
105
prompt_embeds=None,
vllm/entrypoints/serve/render/serving.py
+7
-10
@@ -24,6 +24,7 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
24
parse_chat_inputs_to_harmony_messages,
25
render_for_completion,
26
)
27
+from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
28
from vllm.entrypoints.serve.disagg.protocol import (
29
GenerateRequest,
30
MultiModalFeatures,
@@ -459,9 +460,9 @@ class OpenAIServingRender:
460
prompts.extend(prompt_to_seq(prompt_embeds))
461
if prompt_input is not None:
462
prompts.extend(prompt_to_seq(prompt_input))
462
- return await self._preprocess_cmpl(request, prompts)
463
+ return await self.preprocess_cmpl(request, prompts)
464
464
- async def _preprocess_cmpl(
465
+ async def preprocess_cmpl(
466
self,
467
request: Any,
468
prompts: Sequence[PromptType | bytes],
@@ -500,11 +501,7 @@ class OpenAIServingRender:
501
tool_dicts: list[dict[str, Any]] | None = None,
502
tool_parser: Callable[[TokenizerLike], ToolParser] | None = None,
503
) -> tuple[list[ConversationMessage], list[ProcessorInputs]]:
503
- """Copied from OpenAIServing._preprocess_chat.
504
-
505
- Differences: isinstance check is ChatCompletionRequest-only
506
- (ResponsesRequest not supported here); TODO comment dropped accordingly.
507
- """
504
+ """Copied from OpenAIServing._preprocess_chat."""
505
renderer = self.renderer
506
mm_config = self.model_config.multimodal_config
507
@@ -542,11 +539,11 @@ class OpenAIServingRender:
539
if tool_parser is not None:
540
tool_choice = getattr(request, "tool_choice", "none")
541
if tool_choice != "none":
545
- if not isinstance(request, ChatCompletionRequest):
542
+ if not isinstance(request, ChatCompletionRequest | ResponsesRequest):
543
msg = (
544
"Tool usage is only supported "
548
- " for ChatCompletionRequest, but got "
549
- f"{type(request).__name__}"
545
+ "for Chat Completions API or Responses API requests, "
546
+ f"but got {type(request).__name__}"
547
)
548
raise NotImplementedError(msg)
549
tokenizer = renderer.get_tokenizer()