A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 21:30 UTC
[2/N] move responses/serving _make_response_output_items logic to parser (#33281)
Signed-off-by: Andrew Xia <axia@fb.com> Signed-off-by: Andrew Xia <axia@meta.com> Co-authored-by: Andrew Xia <axia@fb.com>
Andrew Xia committed
Feb 5, 2026 at 00:46 UTC
9595afda183bdd79b0ee2d38d2b0049fe86e6628
2 files changed
+241
-99
vllm/entrypoints/openai/responses/serving.py
+41
-99
@@ -63,7 +63,6 @@ from vllm.engine.protocol import EngineClient
63
from vllm.entrypoints.chat_utils import (
64
ChatCompletionMessageParam,
65
ChatTemplateContentFormatOption,
66
- make_tool_call_id,
66
)
67
from vllm.entrypoints.logger import RequestLogger
68
from vllm.entrypoints.mcp.tool_server import ToolServer
@@ -915,114 +914,57 @@ class OpenAIServingResponses(OpenAIServing):
914
final_output: CompletionOutput,
915
tokenizer: TokenizerLike,
916
) -> list[ResponseOutputItem]:
918
- if self.parser and self.parser.reasoning_parser_cls:
919
- try:
920
- reasoning_parser = self.parser.reasoning_parser_cls(tokenizer)
921
- except RuntimeError as e:
922
- logger.exception("Error in reasoning parser creation.")
923
- raise e
924
-
925
- reasoning, content = reasoning_parser.extract_reasoning(
926
- final_output.text, request=request
927
- )
928
- else:
929
- reasoning = None
930
- content = final_output.text
931
-
917
# Log complete response if output logging is enabled
918
if self.enable_log_outputs and self.request_logger:
934
- output_text = ""
935
- if content:
936
- output_text = content
937
- elif reasoning:
938
- output_text = f"[reasoning: {reasoning}]"
939
-
940
- if output_text:
941
- self.request_logger.log_outputs(
942
- request_id=request.request_id,
943
- outputs=output_text,
944
- output_token_ids=final_output.token_ids,
945
- finish_reason=final_output.finish_reason,
946
- is_streaming=False,
947
- delta=False,
948
- )
919
+ self.request_logger.log_outputs(
920
+ request_id=request.request_id,
921
+ outputs=final_output.text,
922
+ output_token_ids=final_output.token_ids,
923
+ finish_reason=final_output.finish_reason,
924
+ is_streaming=False,
925
+ delta=False,
926
+ )
927
+
928
+ # Compute logprobs if requested
929
+ logprobs = None
930
+ if request.is_include_output_logprobs() and final_output.logprobs:
931
+ logprobs = self._create_response_logprobs(
932
+ token_ids=final_output.token_ids,
933
+ logprobs=final_output.logprobs,
934
+ tokenizer=tokenizer,
935
+ top_logprobs=request.top_logprobs,
936
+ )
937
950
- reasoning_item = None
951
- message_item = None
952
- if reasoning:
953
- reasoning_item = ResponseReasoningItem(
954
- id=f"rs_{random_uuid()}",
955
- summary=[],
956
- type="reasoning",
957
- content=[
958
- ResponseReasoningTextContent(text=reasoning, type="reasoning_text")
959
- ],
960
- status=None, # NOTE: Only the last output item has status.
938
+ # Use parser to extract and create response output items
939
+ if self.parser:
940
+ parser = self.parser(tokenizer)
941
+ return parser.extract_response_outputs(
942
+ model_output=final_output.text,
943
+ request=request,
944
+ enable_auto_tools=self.enable_auto_tools,
945
+ tool_call_id_type=self.tool_call_id_type,
946
+ logprobs=logprobs,
947
)
962
- tool_calls, content = self._parse_tool_calls_from_content(
963
- request=request,
964
- tokenizer=tokenizer,
965
- content=content,
966
- enable_auto_tools=self.enable_auto_tools,
967
- tool_parser_cls=self.parser.tool_parser_cls if self.parser else None,
968
- )
948
970
- if content or (self.use_harmony and tool_calls):
971
- res_text_part = None
972
- if content:
973
- res_text_part = ResponseOutputText(
974
- text=content,
975
- annotations=[], # TODO
976
- type="output_text",
977
- logprobs=(
978
- self._create_response_logprobs(
979
- token_ids=final_output.token_ids,
980
- logprobs=final_output.logprobs,
981
- tokenizer=tokenizer,
982
- top_logprobs=request.top_logprobs,
983
- )
984
- if request.is_include_output_logprobs()
985
- else None
986
- ),
987
- )
988
- message_item = ResponseOutputMessage(
949
+ # Fallback when no parser is configured
950
+ return [
951
+ ResponseOutputMessage(
952
id=f"msg_{random_uuid()}",
990
- content=[res_text_part] if res_text_part else [],
953
+ content=[
954
+ ResponseOutputText(
955
+ text=final_output.text,
956
+ annotations=[],
957
+ type="output_text",
958
+ logprobs=logprobs,
959
+ )
960
+ ]
961
+ if final_output.text
962
+ else [],
963
role="assistant",
964
status="completed",
965
type="message",
966
)
995
- outputs = []
996
-
997
- if reasoning_item:
998
- outputs.append(reasoning_item)
999
- if message_item:
1000
- outputs.append(message_item)
1001
- if tool_calls:
1002
- # We use a simple counter for history_tool_call_count because
1003
- # we don't track the history of tool calls in the Responses API yet.
1004
- # This means that the tool call index will start from 0 for each
1005
- # request.
1006
- tool_call_items = []
1007
- for history_tool_call_cnt, tool_call in enumerate(tool_calls):
1008
- tool_call_items.append(
1009
- ResponseFunctionToolCall(
1010
- id=f"fc_{random_uuid()}",
1011
- call_id=tool_call.id
1012
- if tool_call.id
1013
- else make_tool_call_id(
1014
- id_type=self.tool_call_id_type,
1015
- func_name=tool_call.name,
1016
- idx=history_tool_call_cnt,
1017
- ),
1018
- type="function_call",
1019
- status="completed",
1020
- name=tool_call.name,
1021
- arguments=tool_call.arguments,
1022
- )
1023
- )
1024
- outputs.extend(tool_call_items)
1025
- return outputs
967
+ ]
968
969
def _make_response_output_items_with_harmony(
970
self,
vllm/parser/abstract_parser.py
+200
@@ -1,23 +1,46 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+import json
5
from abc import abstractmethod
6
from collections.abc import Sequence
7
from functools import cached_property
8
9
+from openai.types.responses import (
10
+ ResponseFunctionToolCall,
11
+ ResponseOutputItem,
12
+ ResponseOutputMessage,
13
+ ResponseOutputText,
14
+ ResponseReasoningItem,
15
+ ToolChoiceFunction,
16
+)
17
+from openai.types.responses.response_output_text import Logprob
18
+from openai.types.responses.response_reasoning_item import (
19
+ Content as ResponseReasoningTextContent,
20
+)
21
+from pydantic import TypeAdapter
22
+
23
+from vllm.entrypoints.chat_utils import make_tool_call_id
24
from vllm.entrypoints.openai.chat_completion.protocol import (
25
+ ChatCompletionNamedToolChoiceParam,
26
ChatCompletionRequest,
27
)
28
from vllm.entrypoints.openai.engine.protocol import (
29
DeltaMessage,
30
ExtractedToolCallInformation,
31
+ FunctionCall,
32
+ FunctionDefinition,
33
)
34
from vllm.entrypoints.openai.responses.protocol import (
35
ResponsesRequest,
36
)
37
+from vllm.logger import init_logger
38
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
39
from vllm.tokenizers import TokenizerLike
40
from vllm.tool_parsers.abstract_tool_parser import ToolParser
41
+from vllm.utils import random_uuid
42
+
43
+logger = init_logger(__name__)
44
45
46
class Parser:
@@ -128,6 +151,33 @@ class Parser:
151
The extracted content token IDs.
152
"""
153
154
+ @abstractmethod
155
+ def extract_response_outputs(
156
+ self,
157
+ model_output: str,
158
+ request: ResponsesRequest,
159
+ enable_auto_tools: bool = False,
160
+ tool_call_id_type: str = "random",
161
+ logprobs: list[Logprob] | None = None,
162
+ ) -> list[ResponseOutputItem]:
163
+ """
164
+ Extract reasoning, content, and tool calls from a complete
165
+ model-generated string and return as ResponseOutputItem objects.
166
+
167
+ Used for non-streaming responses where we have the entire model
168
+ response available before sending to the client.
169
+
170
+ Args:
171
+ model_output: The complete model-generated string.
172
+ request: The request object used to generate the output.
173
+ enable_auto_tools: Whether to enable automatic tool call parsing.
174
+ tool_call_id_type: Type of tool call ID generation ("random", etc).
175
+ logprobs: Pre-computed logprobs for the output text, if any.
176
+
177
+ Returns:
178
+ A list of ResponseOutputItem objects.
179
+ """
180
+
181
@abstractmethod
182
def extract_reasoning(
183
self,
@@ -260,6 +310,156 @@ class DelegatingParser(Parser):
310
return None, model_output
311
return self._reasoning_parser.extract_reasoning(model_output, request)
312
313
+ def extract_response_outputs(
314
+ self,
315
+ model_output: str,
316
+ request: ResponsesRequest,
317
+ enable_auto_tools: bool = False,
318
+ tool_call_id_type: str = "random",
319
+ logprobs: list[Logprob] | None = None,
320
+ ) -> list[ResponseOutputItem]:
321
+ # First extract reasoning
322
+ reasoning, content = self.extract_reasoning(model_output, request)
323
+
324
+ # Then parse tool calls from the content
325
+ tool_calls, content = self._parse_tool_calls(
326
+ request=request,
327
+ content=content,
328
+ enable_auto_tools=enable_auto_tools,
329
+ )
330
+
331
+ # Build output items
332
+ outputs: list[ResponseOutputItem] = []
333
+
334
+ # Add reasoning item if present
335
+ if reasoning:
336
+ reasoning_item = ResponseReasoningItem(
337
+ id=f"rs_{random_uuid()}",
338
+ summary=[],
339
+ type="reasoning",
340
+ content=[
341
+ ResponseReasoningTextContent(text=reasoning, type="reasoning_text")
342
+ ],
343
+ status=None, # NOTE: Only the last output item has status.
344
+ )
345
+ outputs.append(reasoning_item)
346
+
347
+ # Add message item if there's content
348
+ if content:
349
+ res_text_part = ResponseOutputText(
350
+ text=content,
351
+ annotations=[],
352
+ type="output_text",
353
+ logprobs=logprobs,
354
+ )
355
+ message_item = ResponseOutputMessage(
356
+ id=f"msg_{random_uuid()}",
357
+ content=[res_text_part],
358
+ role="assistant",
359
+ status="completed",
360
+ type="message",
361
+ )
362
+ outputs.append(message_item)
363
+
364
+ if tool_calls:
365
+ # We use a simple counter for history_tool_call_count because
366
+ # we don't track the history of tool calls in the Responses API yet.
367
+ # This means that the tool call index will start from 0 for each
368
+ # request.
369
+ for history_tool_call_cnt, tool_call in enumerate(tool_calls):
370
+ tool_call_item = ResponseFunctionToolCall(
371
+ id=f"fc_{random_uuid()}",
372
+ call_id=tool_call.id
373
+ if tool_call.id
374
+ else make_tool_call_id(
375
+ id_type=tool_call_id_type,
376
+ func_name=tool_call.name,
377
+ idx=history_tool_call_cnt,
378
+ ),
379
+ type="function_call",
380
+ status="completed",
381
+ name=tool_call.name,
382
+ arguments=tool_call.arguments,
383
+ )
384
+ outputs.append(tool_call_item)
385
+
386
+ return outputs
387
+
388
+ def _parse_tool_calls(
389
+ self,
390
+ request: ResponsesRequest,
391
+ content: str | None,
392
+ enable_auto_tools: bool,
393
+ ) -> tuple[list[FunctionCall], str | None]:
394
+ """
395
+ TODO(qandrew): merge _parse_tool_calls_from_content
396
+ for ChatCompletions into this function
397
+ Parse tool calls from content based on request tool_choice settings.
398
+
399
+ Returns:
400
+ A tuple of (function_calls, remaining_content) if tool calls
401
+ were parsed
402
+ """
403
+ function_calls: list[FunctionCall] = []
404
+
405
+ if request.tool_choice and isinstance(request.tool_choice, ToolChoiceFunction):
406
+ # Forced Function Call (Responses API style)
407
+ assert content is not None
408
+ function_calls.append(
409
+ FunctionCall(name=request.tool_choice.name, arguments=content)
410
+ )
411
+ return function_calls, None # Clear content since tool is called.
412
+
413
+ if request.tool_choice and isinstance(
414
+ request.tool_choice, ChatCompletionNamedToolChoiceParam
415
+ ):
416
+ # Forced Function Call (Chat Completion API style)
417
+ assert content is not None
418
+ function_calls.append(
419
+ FunctionCall(name=request.tool_choice.function.name, arguments=content)
420
+ )
421
+ return function_calls, None # Clear content since tool is called.
422
+
423
+ if request.tool_choice == "required":
424
+ # Required tool calls - parse JSON
425
+ assert content is not None
426
+ tool_calls = TypeAdapter(list[FunctionDefinition]).validate_json(content)
427
+ function_calls.extend(
428
+ FunctionCall(
429
+ name=tool_call.name,
430
+ arguments=json.dumps(tool_call.parameters, ensure_ascii=False),
431
+ )
432
+ for tool_call in tool_calls
433
+ )
434
+ return function_calls, None # Clear content since tool is called.
435
+
436
+ if (
437
+ self._tool_parser is not None
438
+ and enable_auto_tools
439
+ and (request.tool_choice == "auto" or request.tool_choice is None)
440
+ ):
441
+ # Automatic Tool Call Parsing
442
+ tool_call_info = self._tool_parser.extract_tool_calls(
443
+ content if content is not None else "",
444
+ request=request, # type: ignore
445
+ )
446
+ if tool_call_info is not None and tool_call_info.tools_called:
447
+ function_calls.extend(
448
+ FunctionCall(
449
+ id=tool_call.id,
450
+ name=tool_call.function.name,
451
+ arguments=tool_call.function.arguments,
452
+ )
453
+ for tool_call in tool_call_info.tool_calls
454
+ )
455
+ remaining_content = tool_call_info.content
456
+ if remaining_content and remaining_content.strip() == "":
457
+ remaining_content = None
458
+ return function_calls, remaining_content
459
+
460
+ # No tool calls
461
+ return [], content
462
+
463
def extract_reasoning_streaming(
464
self,
465
previous_text: str,