A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 06:00 UTC
[Refactor][Parser] Migrate chat completion auto-tool/reasoning/plain streaming to parse_delta (#39446)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Flora Feng committed
Apr 14, 2026 at 00:39 UTC
db8a6d66bffe32d2232e82baf691f6a379eb066b
2 files changed
+35
-116
vllm/entrypoints/openai/chat_completion/serving.py
+29
-114
@@ -68,11 +68,11 @@ from vllm.logger import init_logger
68
from vllm.logprobs import Logprob
69
from vllm.outputs import CompletionOutput, RequestOutput
70
from vllm.parser import ParserManager
71
+from vllm.parser.abstract_parser import Parser
72
from vllm.reasoning import ReasoningParser
73
from vllm.renderers import ChatParams
74
from vllm.sampling_params import BeamSearchParams, SamplingParams
75
from vllm.tokenizers import TokenizerLike
75
-from vllm.tool_parsers import ToolParser
76
from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
77
from vllm.tool_parsers.utils import partial_json_loads
78
from vllm.utils.collection_utils import as_list
@@ -134,6 +134,12 @@ class OpenAIServingChat(OpenAIServing):
134
enable_auto_tools=enable_auto_tools,
135
model_name=self.model_config.model,
136
)
137
+ self.parser_cls = ParserManager.get_parser(
138
+ tool_parser_name=tool_parser,
139
+ reasoning_parser_name=reasoning_parser,
140
+ enable_auto_tools=enable_auto_tools,
141
+ model_name=self.model_config.model,
142
+ )
143
self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
144
145
self.enable_prompt_tokens_details = enable_prompt_tokens_details
@@ -216,13 +222,12 @@ class OpenAIServingChat(OpenAIServing):
222
# Streaming response
223
tokenizer = self.renderer.tokenizer
224
assert tokenizer is not None
225
+ chat_template_kwargs = self._prepare_extra_chat_template_kwargs(
226
+ request.chat_template_kwargs,
227
+ self.default_chat_template_kwargs,
228
+ )
229
reasoning_parser: ReasoningParser | None = None
230
if self.reasoning_parser_cls:
221
- # Pass the same chat template kwargs as used in tokenization
222
- chat_template_kwargs = self._prepare_extra_chat_template_kwargs(
223
- request.chat_template_kwargs,
224
- self.default_chat_template_kwargs,
225
- )
231
reasoning_parser = self.reasoning_parser_cls(
232
tokenizer,
233
chat_template_kwargs=chat_template_kwargs, # type: ignore[call-arg]
@@ -338,6 +343,7 @@ class OpenAIServingChat(OpenAIServing):
343
tokenizer,
344
request_metadata,
345
reasoning_parser,
346
+ chat_template_kwargs=chat_template_kwargs,
347
)
348
349
return await self.chat_completion_full_generator(
@@ -505,6 +511,7 @@ class OpenAIServingChat(OpenAIServing):
511
tokenizer: TokenizerLike,
512
request_metadata: RequestResponseMetadata,
513
reasoning_parser: ReasoningParser | None = None,
514
+ chat_template_kwargs: dict[str, Any] | None = None,
515
) -> AsyncGenerator[str, None]:
516
created_time = int(time.time())
517
chunk_object_type: Final = "chat.completion.chunk"
@@ -549,29 +556,29 @@ class OpenAIServingChat(OpenAIServing):
556
if tool_choice_auto or reasoning_parser:
557
# These are only required in "auto" tool choice case
558
all_previous_token_ids = [[] for _ in range(num_choices)]
552
- # For reasoning parser and tool call all enabled
553
- added_content_delta_arr = [False] * num_choices
559
reasoning_end_arr = [False] * num_choices
560
prompt_is_reasoning_end_arr: list[bool | None] = [None] * num_choices
561
else:
562
all_previous_token_ids = None
563
559
- # Prepare the tool parser if it's needed
564
try:
561
- if tool_choice_auto and self.tool_parser:
565
+ if self.parser_cls is not None:
566
if tokenizer is None:
567
raise ValueError(
568
"Tokenizer not available when `skip_tokenizer_init=True`"
569
)
566
-
567
- tool_parsers: list[ToolParser | None] = [
568
- self.tool_parser(tokenizer, request.tools)
570
+ parsers: list[Parser | None] = [
571
+ self.parser_cls(
572
+ tokenizer,
573
+ request.tools,
574
+ chat_template_kwargs=chat_template_kwargs,
575
+ )
576
for _ in range(num_choices)
577
]
578
else:
572
- tool_parsers = [None] * num_choices
579
+ parsers = [None] * num_choices
580
except Exception as e:
574
- logger.exception("Error in tool parser creation.")
581
+ logger.exception("Error in parser creation.")
582
data = self.create_streaming_error_response(e)
583
yield f"data: {data}\n\n"
584
yield "data: [DONE]\n\n"
@@ -675,7 +682,8 @@ class OpenAIServingChat(OpenAIServing):
682
683
for output in res.outputs:
684
i = output.index
678
- tool_parser = tool_parsers[i]
685
+ parser = parsers[i]
686
+ tool_parser = parser.tool_parser if parser is not None else None
687
688
if (
689
reasoning_parser
@@ -903,109 +911,16 @@ class OpenAIServingChat(OpenAIServing):
911
history_tool_call_cnt += 1
912
tools_streamed[i] = True
913
906
- # handle streaming deltas for tools with "auto" tool choice
907
- # and reasoning parser
908
- elif tool_choice_auto and reasoning_parser:
909
- assert tool_parser is not None
910
- assert added_content_delta_arr is not None
911
- assert reasoning_end_arr is not None
912
- output_token_ids = as_list(output.token_ids)
913
- if not reasoning_end_arr[i]:
914
- # When encountering think end id in prompt_token_ids
915
- # i.e {"enable_thinking": False},
916
- # set reasoning status to end.
917
- if prompt_is_reasoning_end_arr[i]:
918
- reasoning_end_arr[i] = True
919
- current_token_ids = output_token_ids
920
- # Don't update current_text, keep it as is from delta
921
- else:
922
- delta_message = (
923
- reasoning_parser.extract_reasoning_streaming(
924
- previous_text,
925
- current_text,
926
- delta_text,
927
- previous_token_ids,
928
- current_token_ids,
929
- output_token_ids,
930
- )
931
- )
932
-
933
- # When encountering think end id in delta_token_ids,
934
- # set reasoning status to end.
935
- # Remove the text and token ids related
936
- # to 'reasoning'.
937
- if reasoning_parser.is_reasoning_end(output_token_ids):
938
- reasoning_end_arr[i] = True
939
- current_token_ids = (
940
- reasoning_parser.extract_content_ids(
941
- output_token_ids
942
- )
943
- )
944
- if delta_message and delta_message.content:
945
- current_text = delta_message.content
946
- delta_message.content = None
947
- else:
948
- current_text = ""
949
-
950
- # handle tool calls only after reasoning is done,
951
- if reasoning_end_arr[i]:
952
- delta_token_ids = output_token_ids
953
- # First time to tool call,
954
- # add the remaining text and token ids
955
- # to delta from previous
956
- if not added_content_delta_arr[i]:
957
- added_content_delta_arr[i] = True
958
- previous_text = ""
959
- previous_token_ids = []
960
- delta_text = current_text
961
- delta_token_ids = current_token_ids
962
-
963
- delta_message = tool_parser.extract_tool_calls_streaming(
964
- previous_text=previous_text,
965
- current_text=current_text,
966
- delta_text=delta_text,
967
- previous_token_ids=previous_token_ids,
968
- current_token_ids=current_token_ids,
969
- delta_token_ids=delta_token_ids,
970
- request=request,
971
- )
972
- if delta_message and delta_message.tool_calls:
973
- tools_streamed[i] = True
974
- # when only tool calls
975
- elif tool_choice_auto:
976
- assert tool_parser is not None
977
- delta_message = tool_parser.extract_tool_calls_streaming(
978
- previous_text=previous_text,
979
- current_text=current_text,
914
+ elif parser is not None:
915
+ delta_message = parser.parse_delta(
916
delta_text=delta_text,
981
- previous_token_ids=previous_token_ids,
982
- current_token_ids=current_token_ids,
983
- delta_token_ids=output.token_ids,
917
+ delta_token_ids=as_list(output.token_ids),
918
request=request,
919
+ prompt_token_ids=res.prompt_token_ids,
920
)
921
if delta_message and delta_message.tool_calls:
922
tools_streamed[i] = True
988
-
989
- # when only reasoning
990
- elif reasoning_parser:
991
- # When encountering think end id in prompt_token_ids
992
- # i.e {"enable_thinking": False},
993
- # set reasoning status to end.
994
- # Route all generated tokens as content directly.
995
- if prompt_is_reasoning_end_arr[i]:
996
- delta_message = DeltaMessage(content=delta_text)
997
- else:
998
- delta_message = (
999
- reasoning_parser.extract_reasoning_streaming(
1000
- previous_text,
1001
- current_text,
1002
- delta_text,
1003
- previous_token_ids,
1004
- current_token_ids,
1005
- output.token_ids,
1006
- )
1007
- )
1008
- # handle streaming just a content delta
923
+ # handle streaming just a content delta (no parsers)
924
else:
925
delta_message = DeltaMessage(content=delta_text)
926
vllm/parser/abstract_parser.py
+6
-2
@@ -665,10 +665,14 @@ class _WrappedParser(DelegatingParser):
665
reasoning_parser_cls: type[ReasoningParser] | None = None
666
tool_parser_cls: type[ToolParser] | None = None
667
668
- def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
668
+ def __init__(
669
+ self, tokenizer: TokenizerLike, tools: list[Tool] | None = None, **kwargs
670
+ ):
671
super().__init__(tokenizer)
672
# Instantiate the underlying parsers from class attributes
673
if self.__class__.reasoning_parser_cls is not None:
672
- self._reasoning_parser = self.__class__.reasoning_parser_cls(tokenizer)
674
+ self._reasoning_parser = self.__class__.reasoning_parser_cls(
675
+ tokenizer, **kwargs
676
+ )
677
if self.__class__.tool_parser_cls is not None:
678
self._tool_parser = self.__class__.tool_parser_cls(tokenizer, tools)