A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 03:45 UTC
[Refactor] Consolidate required/named tool_choice streaming into DelegatingParser (#41876)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Flora Feng committed
May 7, 2026 at 12:50 UTC
8eb401134e750781a202c0b6dc4059616cdb4954
3 files changed
+41
-195
tests/tool_use/test_tool_choice_required.py
+15
-22
@@ -2,7 +2,6 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import json
4
from copy import deepcopy
5
-from unittest.mock import MagicMock
5
6
import pytest
7
import regex as re
@@ -11,7 +10,7 @@ from pydantic import TypeAdapter
10
from vllm.entrypoints.openai.chat_completion.protocol import (
11
ChatCompletionToolsParam,
12
)
14
-from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
13
+from vllm.tool_parsers.streaming import extract_required_tool_call_streaming
14
from vllm.tool_parsers.utils import get_json_schema_from_tools
15
16
pytestmark = pytest.mark.cpu_test
@@ -281,8 +280,6 @@ def test_structured_outputs_json_without_parameters(
280
@pytest.mark.parametrize("empty_params", [False, True])
281
@pytest.mark.parametrize("delta_len", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
282
def test_streaming_output_valid(output, empty_params, delta_len):
284
- self = MagicMock()
285
-
283
output = deepcopy(output)
284
if empty_params:
285
output = [{"name": o["name"], "parameters": {}} for o in output]
@@ -295,14 +292,13 @@ def test_streaming_output_valid(output, empty_params, delta_len):
292
delta_text = output_json[i : i + delta_len]
293
current_text = previous_text + delta_text
294
298
- delta_message, function_name_returned = (
299
- OpenAIServingChat.extract_tool_call_required_streaming(
300
- self,
301
- previous_text=previous_text,
302
- current_text=current_text,
303
- delta_text=delta_text,
304
- function_name_returned=function_name_returned,
305
- )
295
+ delta_message, function_name_returned = extract_required_tool_call_streaming(
296
+ previous_text=previous_text,
297
+ current_text=current_text,
298
+ delta_text=delta_text,
299
+ function_name_returned=function_name_returned,
300
+ tool_call_idx=None,
301
+ tool_call_id_type="random",
302
)
303
304
if delta_message:
@@ -332,8 +328,6 @@ def test_streaming_output_valid(output, empty_params, delta_len):
328
329
330
def test_streaming_output_valid_with_trailing_extra_data():
335
- self = MagicMock()
336
-
331
output = [{"name": "get_current_weather", "parameters": {"city": "Vienna"}}]
332
output_json = json.dumps(output) + "\nDONE"
333
@@ -345,14 +339,13 @@ def test_streaming_output_valid_with_trailing_extra_data():
339
delta_text = output_json[i : i + delta_len]
340
current_text = previous_text + delta_text
341
348
- delta_message, function_name_returned = (
349
- OpenAIServingChat.extract_tool_call_required_streaming(
350
- self,
351
- previous_text=previous_text,
352
- current_text=current_text,
353
- delta_text=delta_text,
354
- function_name_returned=function_name_returned,
355
- )
342
+ delta_message, function_name_returned = extract_required_tool_call_streaming(
343
+ previous_text=previous_text,
344
+ current_text=current_text,
345
+ delta_text=delta_text,
346
+ function_name_returned=function_name_returned,
347
+ tool_call_idx=None,
348
+ tool_call_id_type="random",
349
)
350
351
if delta_message:
vllm/entrypoints/openai/chat_completion/serving.py
+10
-169
@@ -70,10 +70,6 @@ from vllm.reasoning import ReasoningParser
70
from vllm.renderers import ChatParams
71
from vllm.sampling_params import BeamSearchParams, SamplingParams
72
from vllm.tokenizers import TokenizerLike
73
-from vllm.tool_parsers.streaming import (
74
- extract_named_tool_call_streaming,
75
- extract_required_tool_call_streaming,
76
-)
73
from vllm.utils.collection_utils import as_list
74
from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser
75
@@ -389,23 +385,6 @@ class OpenAIServingChat(OpenAIServing):
385
return self.response_role
386
return request.messages[-1]["role"]
387
392
- def extract_tool_call_required_streaming(
393
- self,
394
- previous_text: str,
395
- current_text: str | None,
396
- delta_text: str,
397
- function_name_returned: bool,
398
- tool_call_idx: int | None = None,
399
- ) -> tuple[DeltaMessage | None, bool]:
400
- return extract_required_tool_call_streaming(
401
- previous_text=previous_text,
402
- current_text=current_text,
403
- delta_text=delta_text,
404
- function_name_returned=function_name_returned,
405
- tool_call_idx=tool_call_idx,
406
- tool_call_id_type=self.tool_call_id_type,
407
- )
408
-
388
async def chat_completion_stream_generator(
389
self,
390
request: ChatCompletionRequest,
@@ -448,22 +427,7 @@ class OpenAIServingChat(OpenAIServing):
427
and self._should_stream_with_auto_tool_parsing(request)
428
)
429
451
- # Determine whether required/named tool_choice should fall back to
452
- # the auto tool_parser path instead of the standard JSON-based parsing.
453
- # This happens when the parser declares supports_required_and_named=False
454
- # (e.g. GLM models that output XML instead of JSON).
455
- tool_choice_uses_parser = (
456
- self.tool_parser is not None
457
- and not self.tool_parser.supports_required_and_named
458
- and request.tools
459
- and (
460
- request.tool_choice == "required"
461
- or isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
462
- )
463
- )
464
-
430
all_previous_token_ids: list[list[int]] | None
466
- function_name_returned = [False] * num_choices
431
if self.tool_call_id_type == "kimi_k2":
432
history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
433
else:
@@ -477,10 +441,10 @@ class OpenAIServingChat(OpenAIServing):
441
if (
442
is_mistral_grammar_path
443
or tool_choice_auto
480
- or tool_choice_uses_parser
444
+ or tool_choice_function_name
445
+ or request.tool_choice == "required"
446
or reasoning_parser
447
):
483
- # These are only required in "auto" tool choice case
448
all_previous_token_ids = [[] for _ in range(num_choices)]
449
reasoning_end_arr = [False] * num_choices
450
prompt_is_reasoning_end_arr: list[bool | None] = [None] * num_choices
@@ -501,6 +465,10 @@ class OpenAIServingChat(OpenAIServing):
465
)
466
for _ in range(num_choices)
467
]
468
+ for p in parsers:
469
+ if p is not None:
470
+ p._stream_state.tool_call_id_type = self.tool_call_id_type
471
+ p._stream_state.history_tool_call_cnt = history_tool_call_cnt
472
else:
473
parsers = [None] * num_choices
474
except Exception as e:
@@ -677,7 +645,8 @@ class OpenAIServingChat(OpenAIServing):
645
if (
646
is_mistral_grammar_path
647
or tool_choice_auto
680
- or tool_choice_uses_parser
648
+ or tool_choice_function_name
649
+ or request.tool_choice == "required"
650
or reasoning_parser
651
):
652
assert previous_texts is not None
@@ -731,135 +700,6 @@ class OpenAIServingChat(OpenAIServing):
700
current_token_ids = result.current_token_ids
701
if result.tools_called:
702
tools_streamed[i] = True
734
- # handle streaming deltas for tools with named tool_choice
735
- # Skip when tool_choice_uses_parser so it falls through
736
- # to the auto tool_parser branches below.
737
- elif tool_choice_function_name and not tool_choice_uses_parser:
738
- # When encountering think end id in prompt_token_ids
739
- # i.e {"enable_thinking": False},
740
- # check BEFORE calling the parser to avoid a spurious
741
- # reasoning delta on the first chunk.
742
- if (
743
- reasoning_parser
744
- and not reasoning_end_arr[i]
745
- and prompt_is_reasoning_end_arr[i]
746
- ):
747
- reasoning_end_arr[i] = True
748
-
749
- if (
750
- reasoning_parser
751
- and not reasoning_end_arr[i]
752
- and not reasoning_parser.is_reasoning_end(
753
- previous_token_ids
754
- )
755
- ):
756
- assert reasoning_parser is not None
757
- delta_message = (
758
- reasoning_parser.extract_reasoning_streaming(
759
- previous_text,
760
- current_text,
761
- delta_text,
762
- previous_token_ids,
763
- current_token_ids,
764
- output.token_ids,
765
- )
766
- )
767
- # When encountering think end id in delta_token_ids,
768
- # set reasoning status to end.
769
- # Only keep 'content', remove 'reasoning'.
770
- if reasoning_parser.is_reasoning_end(
771
- as_list(output.token_ids)
772
- ):
773
- reasoning_end_arr[i] = True
774
- if delta_message and delta_message.content:
775
- current_text = delta_message.content
776
- delta_message.content = None
777
- else:
778
- current_text = ""
779
- else:
780
- # Just to add remaining `content`
781
- if reasoning_parser:
782
- delta_text = previous_text + delta_text
783
- current_text = ""
784
-
785
- delta_message, function_name_returned[i] = (
786
- extract_named_tool_call_streaming(
787
- delta_text=delta_text,
788
- function_name=tool_choice_function_name,
789
- function_name_returned=function_name_returned[i],
790
- tool_call_idx=history_tool_call_cnt,
791
- tool_call_id_type=self.tool_call_id_type,
792
- tokenizer=tokenizer,
793
- tool_call_array_index=i,
794
- )
795
- )
796
- if (
797
- delta_message
798
- and delta_message.tool_calls
799
- and delta_message.tool_calls[0].id is not None
800
- ):
801
- history_tool_call_cnt += 1
802
- tools_streamed[i] = True
803
-
804
- # Skip when tool_choice_uses_parser so it falls through
805
- # to the auto tool_parser branches below.
806
- elif (
807
- request.tool_choice == "required"
808
- and not tool_choice_uses_parser
809
- ):
810
- assert previous_texts is not None
811
- previous_text = previous_texts[i]
812
- current_text = previous_text + delta_text
813
- fn_name_returned = function_name_returned[i]
814
- output_token_ids = as_list(output.token_ids)
815
-
816
- if (
817
- reasoning_parser is not None
818
- and not reasoning_end_arr[i]
819
- and prompt_is_reasoning_end_arr[i]
820
- ):
821
- reasoning_end_arr[i] = True
822
-
823
- if reasoning_parser and not reasoning_end_arr[i]:
824
- delta_message = (
825
- reasoning_parser.extract_reasoning_streaming(
826
- previous_text,
827
- current_text,
828
- delta_text,
829
- previous_token_ids,
830
- current_token_ids,
831
- output_token_ids,
832
- )
833
- )
834
- if reasoning_parser.is_reasoning_end(output_token_ids):
835
- reasoning_end_arr[i] = True
836
- if delta_message and delta_message.content:
837
- current_text = delta_message.content
838
- delta_message.content = None
839
- else:
840
- # reasoning ended
841
- current_text = ""
842
-
843
- else:
844
- # either finished reasoning or no reasoning at all
845
- content = current_text
846
-
847
- delta_message, function_name_returned[i] = (
848
- self.extract_tool_call_required_streaming(
849
- previous_text=previous_text,
850
- current_text=content,
851
- delta_text=delta_text,
852
- function_name_returned=fn_name_returned,
853
- tool_call_idx=history_tool_call_cnt,
854
- )
855
- )
856
- if (
857
- delta_message
858
- and delta_message.tool_calls
859
- and delta_message.tool_calls[0].id is not None
860
- ):
861
- history_tool_call_cnt += 1
862
- tools_streamed[i] = True
703
704
elif parser is not None:
705
delta_message = parser.parse_delta(
@@ -878,7 +718,8 @@ class OpenAIServingChat(OpenAIServing):
718
if (
719
is_mistral_grammar_path
720
or tool_choice_auto
881
- or tool_choice_uses_parser
721
+ or tool_choice_function_name
722
+ or request.tool_choice == "required"
723
or reasoning_parser
724
) and not self.use_harmony:
725
assert previous_texts is not None
vllm/parser/abstract_parser.py
+16
-4
@@ -587,9 +587,15 @@ class DelegatingParser(Parser):
587
tool_call_id_type: str = "random",
588
function_name_returned: bool = False,
589
) -> tuple[DeltaMessage | None, bool]:
590
- if request.tool_choice and isinstance(
591
- request.tool_choice,
592
- (ToolChoiceFunction, ChatCompletionNamedToolChoiceParam),
590
+ assert self._tool_parser is not None
591
+ supports_required_and_named = self._tool_parser.supports_required_and_named
592
+ if (
593
+ supports_required_and_named
594
+ and request.tool_choice
595
+ and isinstance(
596
+ request.tool_choice,
597
+ (ToolChoiceFunction, ChatCompletionNamedToolChoiceParam),
598
+ )
599
):
600
delta_message, function_name_returned = extract_named_tool_call_streaming(
601
delta_text=delta_text,
@@ -601,7 +607,7 @@ class DelegatingParser(Parser):
607
)
608
return delta_message, function_name_returned
609
604
- if request.tool_choice == "required":
610
+ if supports_required_and_named and request.tool_choice == "required":
611
delta_message, function_name_returned = (
612
extract_required_tool_call_streaming(
613
previous_text=previous_text,
@@ -706,6 +712,12 @@ class DelegatingParser(Parser):
712
function_name_returned=state.function_name_returned,
713
)
714
)
715
+ if (
716
+ delta_message
717
+ and delta_message.tool_calls
718
+ and delta_message.tool_calls[0].id is not None
719
+ ):
720
+ state.history_tool_call_cnt += 1
721
722
# No phase active: pass through as content
723
if (