A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 13:45 UTC
[Frontend]Responses API supports Tool/Function calling with streaming with named tool/function (#41110)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Apr 29, 2026 at 17:11 UTC
3885d340a4779c54662b10892555ae6928b3a090
4 files changed
+57
-60
tests/entrypoints/openai/responses/test_function_call.py
+2
-2
@@ -323,7 +323,7 @@ async def test_function_calling_with_streaming_expected_arguments(
323
@pytest.mark.parametrize("model_name", [MODEL_NAME])
324
@pytest.mark.parametrize(
325
"tool_choice",
326
- ["auto", "required"],
326
+ ["auto", "required", {"type": "function", "name": "get_current_weather"}],
327
)
328
async def test_function_calling_with_streaming_types(
329
client: openai.AsyncOpenAI, model_name: str, tool_choice
@@ -462,7 +462,7 @@ async def test_function_calling_with_streaming_types(
462
@pytest.mark.parametrize("model_name", [MODEL_NAME])
463
@pytest.mark.parametrize(
464
"tool_choice",
465
- ["required", "auto"],
465
+ ["required", "auto", {"type": "function", "name": "get_weather"}],
466
)
467
async def test_function_calling_with_streaming_forced_tool_choice(
468
client: openai.AsyncOpenAI, model_name: str, tool_choice: str
vllm/entrypoints/openai/chat_completion/serving.py
+20
-36
@@ -70,7 +70,10 @@ from vllm.reasoning import ReasoningParser
70
from vllm.renderers import ChatParams
71
from vllm.sampling_params import BeamSearchParams, SamplingParams
72
from vllm.tokenizers import TokenizerLike
73
-from vllm.tool_parsers.streaming import extract_required_tool_call_streaming
73
+from vllm.tool_parsers.streaming import (
74
+ extract_named_tool_call_streaming,
75
+ extract_required_tool_call_streaming,
76
+)
77
from vllm.utils.collection_utils import as_list
78
from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser
79
@@ -773,43 +776,24 @@ class OpenAIServingChat(OpenAIServing):
776
delta_text = previous_text + delta_text
777
current_text = ""
778
776
- if function_name_returned[i]:
777
- delta_tool_call = DeltaToolCall(
778
- function=DeltaFunctionCall(arguments=delta_text),
779
- index=i,
780
- )
781
- else:
782
- # Generate ID based on tokenizer type
783
- if is_mistral_tokenizer(tokenizer):
784
- from vllm.tool_parsers.mistral_tool_parser import (
785
- MistralToolCall,
786
- )
787
-
788
- tool_call_id = MistralToolCall.generate_random_id()
789
- else:
790
- tool_call_id = make_tool_call_id(
791
- id_type=self.tool_call_id_type,
792
- func_name=tool_choice_function_name,
793
- idx=history_tool_call_cnt,
794
- )
795
- delta_tool_call = DeltaToolCall(
796
- id=tool_call_id,
797
- type="function",
798
- function=DeltaFunctionCall(
799
- name=tool_choice_function_name,
800
- arguments=delta_text,
801
- ),
802
- index=i,
779
+ delta_message, function_name_returned[i] = (
780
+ extract_named_tool_call_streaming(
781
+ delta_text=delta_text,
782
+ function_name=tool_choice_function_name,
783
+ function_name_returned=function_name_returned[i],
784
+ tool_call_idx=history_tool_call_cnt,
785
+ tool_call_id_type=self.tool_call_id_type,
786
+ tokenizer=tokenizer,
787
+ tool_call_array_index=i,
788
)
804
- function_name_returned[i] = True
805
- history_tool_call_cnt += 1
806
-
807
- delta_message = DeltaMessage(
808
- tool_calls=[
809
- delta_tool_call,
810
- ]
789
)
812
- tools_streamed[i] = True
790
+ if (
791
+ delta_message
792
+ and delta_message.tool_calls
793
+ and delta_message.tool_calls[0].id is not None
794
+ ):
795
+ history_tool_call_cnt += 1
796
+ tools_streamed[i] = True
797
798
# Skip when tool_choice_uses_parser so it falls through
799
# to the auto tool_parser branches below.
vllm/parser/abstract_parser.py
+33
-16
@@ -38,7 +38,10 @@ from vllm.logger import init_logger
38
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
39
from vllm.tokenizers import TokenizerLike
40
from vllm.tool_parsers.abstract_tool_parser import ToolParser
41
-from vllm.tool_parsers.streaming import extract_required_tool_call_streaming
41
+from vllm.tool_parsers.streaming import (
42
+ extract_named_tool_call_streaming,
43
+ extract_required_tool_call_streaming,
44
+)
45
from vllm.tool_parsers.utils import Tool
46
from vllm.utils import random_uuid
47
@@ -423,6 +426,17 @@ class DelegatingParser(Parser):
426
427
return outputs
428
429
+ def _get_function_name(
430
+ self, request: ChatCompletionRequest | ResponsesRequest
431
+ ) -> str:
432
+ if request.tool_choice and isinstance(request.tool_choice, ToolChoiceFunction):
433
+ return request.tool_choice.name
434
+ if request.tool_choice and isinstance(
435
+ request.tool_choice, ChatCompletionNamedToolChoiceParam
436
+ ):
437
+ return request.tool_choice.function.name
438
+ raise ValueError("Invalid tool_choice for function name extraction.")
439
+
440
def _parse_tool_calls(
441
self,
442
request: ResponsesRequest,
@@ -440,21 +454,14 @@ class DelegatingParser(Parser):
454
"""
455
function_calls: list[FunctionCall] = []
456
443
- if request.tool_choice and isinstance(request.tool_choice, ToolChoiceFunction):
444
- # Forced Function Call (Responses API style)
445
- assert content is not None
446
- function_calls.append(
447
- FunctionCall(name=request.tool_choice.name, arguments=content)
448
- )
449
- return function_calls, None # Clear content since tool is called.
450
-
457
if request.tool_choice and isinstance(
452
- request.tool_choice, ChatCompletionNamedToolChoiceParam
458
+ request.tool_choice,
459
+ (ToolChoiceFunction, ChatCompletionNamedToolChoiceParam),
460
):
454
- # Forced Function Call (Chat Completion API style)
461
+ # Forced Function Call
462
assert content is not None
463
function_calls.append(
457
- FunctionCall(name=request.tool_choice.function.name, arguments=content)
464
+ FunctionCall(name=self._get_function_name(request), arguments=content)
465
)
466
return function_calls, None # Clear content since tool is called.
467
@@ -572,7 +579,7 @@ class DelegatingParser(Parser):
579
previous_token_ids: Sequence[int],
580
current_token_ids: Sequence[int],
581
delta_token_ids: Sequence[int],
575
- request: ChatCompletionRequest,
582
+ request: ChatCompletionRequest | ResponsesRequest,
583
# The following parameters are used for "required" tool choice parsing and are
584
# tracked in StreamState for streaming parsing.
585
tool_call_idx: int | None = None,
@@ -580,9 +587,19 @@ class DelegatingParser(Parser):
587
function_name_returned: bool = False,
588
) -> tuple[DeltaMessage | None, bool]:
589
if request.tool_choice and isinstance(
583
- request.tool_choice, ChatCompletionNamedToolChoiceParam
590
+ request.tool_choice,
591
+ (ToolChoiceFunction, ChatCompletionNamedToolChoiceParam),
592
):
585
- return None, False
593
+ delta_message, function_name_returned = extract_named_tool_call_streaming(
594
+ delta_text=delta_text,
595
+ function_name=self._get_function_name(request),
596
+ function_name_returned=function_name_returned,
597
+ tool_call_idx=tool_call_idx,
598
+ tool_call_id_type=tool_call_id_type,
599
+ tokenizer=self.model_tokenizer,
600
+ )
601
+ return delta_message, function_name_returned
602
+
603
if request.tool_choice == "required":
604
delta_message, function_name_returned = (
605
extract_required_tool_call_streaming(
@@ -602,7 +619,7 @@ class DelegatingParser(Parser):
619
previous_token_ids,
620
current_token_ids,
621
delta_token_ids,
605
- request,
622
+ request, # type: ignore[arg-type]
623
), False
624
625
def is_reasoning_end(self, input_ids: list[int]) -> bool:
vllm/tool_parsers/streaming.py
+2
-6
@@ -67,10 +67,9 @@ def extract_named_tool_call_streaming(
67
tool_call_idx: int | None,
68
tool_call_id_type: str,
69
tokenizer: "TokenizerLike",
70
- tool_call_array_index: int,
71
-) -> tuple[DeltaMessage, bool, bool]:
70
+ tool_call_array_index: int = 0,
71
+) -> tuple[DeltaMessage | None, bool]:
72
"""Build a streaming tool-call delta for forced named tool choice."""
73
- created_new_tool_call = False
73
if function_name_returned:
74
delta_tool_call = DeltaToolCall(
75
function=DeltaFunctionCall(arguments=delta_text),
@@ -95,12 +94,9 @@ def extract_named_tool_call_streaming(
94
index=tool_call_array_index,
95
)
96
function_name_returned = True
98
- created_new_tool_call = True
99
-
97
return (
98
DeltaMessage(tool_calls=[delta_tool_call]),
99
function_name_returned,
103
- created_new_tool_call,
100
)
101
102