@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 59a50afa08
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 13, 17:00 UTC

[Frontend] OpenAI Responses API supports Tool/Function calling - non-harmony (#26874)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>

Chauncey committed Nov 6, 2025 at 18:40 UTC 59a50afa084dbd26b8a4f58b960ce337af6a4667
12 files changed +404 -30
examples/online_serving/openai_responses_client_with_tools.py new
+83
@@ -0,0 +1,83 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""
4 +Set up this example by starting a vLLM OpenAI-compatible server with tool call
5 +options enabled.
6 +Reasoning models can be used through the Responses API as seen here
7 +https://platform.openai.com/docs/api-reference/responses
8 +For example:
9 +vllm serve Qwen/Qwen3-1.7B --reasoning-parser qwen3 \
10 + --structured-outputs-config.backend xgrammar \
11 + --enable-auto-tool-choice --tool-call-parser hermes
12 +"""
13 +
14 +import json
15 +
16 +from openai import OpenAI
17 +from utils import get_first_model
18 +
19 +
20 +def get_weather(latitude: float, longitude: float) -> str:
21 + """
22 + Mock function to simulate getting weather data.
23 + In a real application, this would call an external weather API.
24 + """
25 + return f"Current temperature at ({latitude}, {longitude}) is 20°C."
26 +
27 +
28 +tools = [
29 + {
30 + "type": "function",
31 + "name": "get_weather",
32 + "description": "Get current temperature for provided coordinates in celsius.",
33 + "parameters": {
34 + "type": "object",
35 + "properties": {
36 + "latitude": {"type": "number"},
37 + "longitude": {"type": "number"},
38 + },
39 + "required": ["latitude", "longitude"],
40 + "additionalProperties": False,
41 + },
42 + "strict": True,
43 + }
44 +]
45 +
46 +input_messages = [
47 + {"role": "user", "content": "What's the weather like in Paris today?"}
48 +]
49 +
50 +
51 +def main():
52 + base_url = "http://0.0.0.0:8000/v1"
53 + client = OpenAI(base_url=base_url, api_key="empty")
54 + model = get_first_model(client)
55 + response = client.responses.create(
56 + model=model, input=input_messages, tools=tools, tool_choice="required"
57 + )
58 +
59 + for out in response.output:
60 + if out.type == "function_call":
61 + print("Function call:", out.name, out.arguments)
62 + tool_call = out
63 + args = json.loads(tool_call.arguments)
64 + result = get_weather(args["latitude"], args["longitude"])
65 +
66 + input_messages.append(tool_call) # append model's function call message
67 + input_messages.append(
68 + { # append result message
69 + "type": "function_call_output",
70 + "call_id": tool_call.call_id,
71 + "output": str(result),
72 + }
73 + )
74 + response_2 = client.responses.create(
75 + model=model,
76 + input=input_messages,
77 + tools=tools,
78 + )
79 + print(response_2.output_text)
80 +
81 +
82 +if __name__ == "__main__":
83 + main()
tests/v1/entrypoints/openai/serving_responses/__init__.py renamed
tests/v1/entrypoints/openai/serving_responses/conftest.py renamed
+6 -1
@@ -15,8 +15,13 @@ def default_server_args():
15 "--max-model-len",
16 "8192",
17 "--enforce-eager", # For faster startup.
18 + "--enable-auto-tool-choice",
19 + "--structured-outputs-config.backend",
20 + "xgrammar",
21 + "--tool-call-parser",
22 + "hermes",
23 "--reasoning-parser",
19 - "deepseek_r1",
24 + "qwen3",
25 ]
26
27
tests/v1/entrypoints/openai/serving_responses/test_basic.py renamed
tests/v1/entrypoints/openai/serving_responses/test_function_call.py new
+198
@@ -0,0 +1,198 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import json
5 +
6 +import openai # use the official client for correctness check
7 +import pytest
8 +
9 +MODEL_NAME = "Qwen/Qwen3-1.7B"
10 +tools = [
11 + {
12 + "type": "function",
13 + "name": "get_current_weather",
14 + "description": "Get the current weather in a given location",
15 + "parameters": {
16 + "type": "object",
17 + "properties": {
18 + "city": {
19 + "type": "string",
20 + "description": "The city to find the weather for, e.g. 'Vienna'",
21 + "default": "Vienna",
22 + },
23 + "country": {
24 + "type": "string",
25 + "description": "The country that the city is in, e.g. 'Austria'",
26 + },
27 + "unit": {
28 + "type": "string",
29 + "description": "The unit to fetch the temperature in",
30 + "enum": ["celsius", "fahrenheit"],
31 + },
32 + "options": {
33 + "$ref": "#/$defs/WeatherOptions",
34 + "description": "Optional parameters for weather query",
35 + },
36 + },
37 + "required": ["country", "unit"],
38 + "$defs": {
39 + "WeatherOptions": {
40 + "title": "WeatherOptions",
41 + "type": "object",
42 + "additionalProperties": False,
43 + "properties": {
44 + "unit": {
45 + "type": "string",
46 + "enum": ["celsius", "fahrenheit"],
47 + "default": "celsius",
48 + "description": "Temperature unit",
49 + "title": "Temperature Unit",
50 + },
51 + "include_forecast": {
52 + "type": "boolean",
53 + "default": False,
54 + "description": "Whether to include a 24-hour forecast",
55 + "title": "Include Forecast",
56 + },
57 + "language": {
58 + "type": "string",
59 + "default": "zh-CN",
60 + "description": "Language of the response",
61 + "title": "Language",
62 + "enum": ["zh-CN", "en-US", "ja-JP"],
63 + },
64 + },
65 + },
66 + },
67 + },
68 + },
69 + {
70 + "type": "function",
71 + "name": "get_forecast",
72 + "description": "Get the weather forecast for a given location",
73 + "parameters": {
74 + "type": "object",
75 + "properties": {
76 + "city": {
77 + "type": "string",
78 + "description": "The city to get the forecast for, e.g. 'Vienna'",
79 + "default": "Vienna",
80 + },
81 + "country": {
82 + "type": "string",
83 + "description": "The country that the city is in, e.g. 'Austria'",
84 + },
85 + "days": {
86 + "type": "integer",
87 + "description": "Number of days to get the forecast for (1-7)",
88 + },
89 + "unit": {
90 + "type": "string",
91 + "description": "The unit to fetch the temperature in",
92 + "enum": ["celsius", "fahrenheit"],
93 + },
94 + },
95 + "required": ["country", "days", "unit"],
96 + },
97 + },
98 +]
99 +
100 +
101 +@pytest.mark.asyncio
102 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
103 +@pytest.mark.parametrize("tool_choice", ["auto", "required"])
104 +async def test_function_tool_use(
105 + client: openai.AsyncOpenAI, model_name: str, tool_choice: str
106 +):
107 + prompt = [
108 + {
109 + "role": "user",
110 + "content": "Can you tell me what the current weather is in Berlin and the "
111 + "forecast for the next 5 days, in fahrenheit?",
112 + },
113 + ]
114 + response = await client.responses.create(
115 + model=model_name,
116 + input=prompt,
117 + tools=tools,
118 + tool_choice=tool_choice,
119 + )
120 +
121 + assert len(response.output) >= 1
122 + tool_call = None
123 + reasoning = None
124 + for out in response.output:
125 + if out.type == "function_call":
126 + tool_call = out
127 + if out.type == "reasoning":
128 + reasoning = out
129 + assert tool_call is not None
130 + assert tool_call.type == "function_call"
131 + assert json.loads(tool_call.arguments) is not None
132 + assert reasoning is not None
133 + assert reasoning.type == "reasoning"
134 +
135 +
136 +@pytest.mark.asyncio
137 +async def test_named_tool_use(client: openai.AsyncOpenAI):
138 + def get_weather(latitude: float, longitude: float) -> str:
139 + """
140 + Mock function to simulate getting weather data.
141 + In a real application, this would call an external weather API.
142 + """
143 + return f"Current temperature at ({latitude}, {longitude}) is 20°C."
144 +
145 + tools = [
146 + {
147 + "type": "function",
148 + "name": "get_weather",
149 + "description": (
150 + "Get current temperature for provided coordinates in celsius."
151 + ),
152 + "parameters": {
153 + "type": "object",
154 + "properties": {
155 + "latitude": {"type": "number"},
156 + "longitude": {"type": "number"},
157 + },
158 + "required": ["latitude", "longitude"],
159 + "additionalProperties": False,
160 + },
161 + "strict": True,
162 + }
163 + ]
164 +
165 + input_messages = [
166 + {"role": "user", "content": "What's the weather like in Paris today?"}
167 + ]
168 +
169 + response = await client.responses.create(
170 + model=MODEL_NAME,
171 + input=input_messages,
172 + tools=tools,
173 + tool_choice={"type": "function", "name": "get_weather"},
174 + )
175 + assert len(response.output) >= 1
176 + for out in response.output:
177 + if out.type == "function_call":
178 + tool_call = out
179 + assert tool_call is not None
180 + assert tool_call.type == "function_call"
181 + assert tool_call.name == "get_weather"
182 + args = json.loads(tool_call.arguments)
183 + assert args["latitude"] is not None
184 + assert args["longitude"] is not None
185 + # call the tool
186 + result = get_weather(args["latitude"], args["longitude"])
187 + input_messages.append(tool_call) # append model's function call message
188 + input_messages.append(
189 + { # append result message
190 + "type": "function_call_output",
191 + "call_id": tool_call.call_id,
192 + "output": str(result),
193 + }
194 + )
195 + # create a new response with the tool call result
196 + response_2 = await client.responses.create(model=MODEL_NAME, input=input_messages)
197 + # check the output
198 + assert len(response_2.output_text) > 0
tests/v1/entrypoints/openai/serving_responses/test_image.py renamed
tests/v1/entrypoints/openai/serving_responses/test_stateful.py renamed
tests/v1/entrypoints/openai/serving_responses/test_structured_output.py renamed
vllm/entrypoints/openai/serving_engine.py
+6 -6
@@ -1098,13 +1098,13 @@ class OpenAIServing:
1098 )
1099
1100 if should_parse_tools:
1101 - if not isinstance(request, ChatCompletionRequest):
1102 - msg = "Tool usage is only supported for Chat Completions API"
1101 + if not isinstance(request, ChatCompletionRequest | ResponsesRequest):
1102 + msg = (
1103 + "Tool usage is only supported for Chat Completions API "
1104 + "or Responses API requests."
1105 + )
1106 raise NotImplementedError(msg)
1104 -
1105 - request = tool_parser(tokenizer).adjust_request( # type: ignore
1106 - request=request
1107 - )
1107 + request = tool_parser(tokenizer).adjust_request(request=request) # type: ignore
1108
1109 if tokenizer is None:
1110 assert isinstance(request_prompt, str), (
vllm/entrypoints/openai/serving_responses.py
+46 -19
@@ -94,6 +94,7 @@ from vllm.entrypoints.openai.protocol import (
94 )
95 from vllm.entrypoints.openai.serving_engine import OpenAIServing
96 from vllm.entrypoints.openai.serving_models import OpenAIServingModels
97 +from vllm.entrypoints.responses_utils import construct_chat_message_with_tool_call
98 from vllm.entrypoints.tool_server import ToolServer
99 from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
100 from vllm.logger import init_logger
@@ -196,16 +197,12 @@ class OpenAIServingResponses(OpenAIServing):
197 self.default_sampling_params["stop_token_ids"].extend(
198 get_stop_tokens_for_assistant_actions()
199 )
199 -
200 + self.enable_auto_tools = enable_auto_tools
201 # set up tool use
201 - self.enable_auto_tools: bool = enable_auto_tools
202 - if self.enable_auto_tools:
203 - logger.info(
204 - '"auto" tool choice has been enabled please note that while'
205 - " the parallel_tool_calls client option is preset for "
206 - "compatibility reasons, it will be ignored."
207 - )
208 -
202 + self.tool_parser = self._get_tool_parser(
203 + tool_parser_name=tool_parser, enable_auto_tools=enable_auto_tools
204 + )
205 + self.exclude_tools_when_tool_choice_none = False
206 # HACK(woosuk): This is a hack. We should use a better store.
207 # FIXME: If enable_store=True, this may cause a memory leak since we
208 # never remove responses from the store.
@@ -511,16 +508,20 @@ class OpenAIServingResponses(OpenAIServing):
508 prev_response: ResponsesResponse | None,
509 tokenizer: AnyTokenizer,
510 ):
514 - if len(request.tools) > 0:
515 - raise NotImplementedError(
516 - "Tool use is not supported in Responses API without Harmony"
517 - )
511 + if request.tools is None or (
512 + request.tool_choice == "none" and self.exclude_tools_when_tool_choice_none
513 + ):
514 + tool_dicts = None
515 + else:
516 + tool_dicts = [tool.model_dump() for tool in request.tools]
517 # Construct the input messages.
518 messages = self._construct_input_messages(request, prev_response)
519 _, request_prompts, engine_prompts = await self._preprocess_chat(
520 request,
521 tokenizer,
522 messages,
523 + tool_dicts=tool_dicts,
524 + tool_parser=self.tool_parser,
525 chat_template=self.chat_template,
526 chat_template_content_format=self.chat_template_content_format,
527 )
@@ -802,7 +803,8 @@ class OpenAIServingResponses(OpenAIServing):
803 delta=False,
804 )
805
805 - output = []
806 + reasoning_item = None
807 + message_item = None
808 if reasoning_content:
809 reasoning_item = ResponseReasoningItem(
810 id=f"rs_{random_uuid()}",
@@ -815,7 +817,13 @@ class OpenAIServingResponses(OpenAIServing):
817 ],
818 status=None, # NOTE: Only the last output item has status.
819 )
818 - output.append(reasoning_item)
820 + tool_calls, content = self._parse_tool_calls_from_content(
821 + request=request,
822 + tokenizer=tokenizer,
823 + content=content,
824 + enable_auto_tools=self.enable_auto_tools,
825 + tool_parser_cls=self.tool_parser,
826 + )
827 if content:
828 output_text = ResponseOutputText(
829 text=content,
@@ -832,15 +840,33 @@ class OpenAIServingResponses(OpenAIServing):
840 else None
841 ),
842 )
835 - message = ResponseOutputMessage(
843 + message_item = ResponseOutputMessage(
844 id=f"msg_{random_uuid()}",
845 content=[output_text],
846 role="assistant",
847 status="completed",
848 type="message",
849 )
842 - output.append(message)
843 - return output
850 + outputs = []
851 +
852 + if reasoning_item:
853 + outputs.append(reasoning_item)
854 + if message_item:
855 + outputs.append(message_item)
856 + if tool_calls:
857 + tool_call_items = [
858 + ResponseFunctionToolCall(
859 + id=f"fc_{random_uuid()}",
860 + call_id=f"call_{random_uuid()}",
861 + type="function_call",
862 + status="completed",
863 + name=tool_call.name,
864 + arguments=tool_call.arguments,
865 + )
866 + for tool_call in tool_calls
867 + ]
868 + outputs.extend(tool_call_items)
869 + return outputs
870
871 def _make_response_output_items_with_harmony(
872 self,
@@ -893,7 +919,8 @@ class OpenAIServingResponses(OpenAIServing):
919 if isinstance(request.input, str):
920 messages.append({"role": "user", "content": request.input})
921 else:
896 - messages.extend(request.input) # type: ignore
922 + for item in request.input:
923 + messages.append(construct_chat_message_with_tool_call(item))
924 return messages
925
926 def _construct_harmony_system_input_message(
vllm/entrypoints/openai/tool_parsers/abstract_tool_parser.py
+20 -4
@@ -6,10 +6,16 @@ import os
6 from collections.abc import Callable, Sequence
7 from functools import cached_property
8
9 +from openai.types.responses.response_format_text_json_schema_config import (
10 + ResponseFormatTextJSONSchemaConfig,
11 +)
12 +
13 from vllm.entrypoints.openai.protocol import (
14 ChatCompletionRequest,
15 DeltaMessage,
16 ExtractedToolCallInformation,
17 + ResponsesRequest,
18 + ResponseTextConfig,
19 )
20 from vllm.entrypoints.openai.tool_parsers.utils import get_json_schema_from_tools
21 from vllm.logger import init_logger
@@ -56,11 +62,21 @@ class ToolParser:
62 )
63 # Set structured output params for tool calling
64 if json_schema_from_tool is not None:
59 - if request.structured_outputs is None:
65 + if isinstance(request, ChatCompletionRequest):
66 request.structured_outputs = StructuredOutputsParams()
61 - # tool_choice: "Forced Function" or "required" will override
62 - # structured output json settings to make tool calling work correctly
63 - request.structured_outputs.json = json_schema_from_tool
67 + # tool_choice: "Forced Function" or "required" will override
68 + # structured output json settings to make tool calling work correctly
69 + request.structured_outputs.json = json_schema_from_tool
70 + if isinstance(request, ResponsesRequest):
71 + request.text = ResponseTextConfig()
72 + request.text.format = ResponseFormatTextJSONSchemaConfig(
73 + name="tool_calling_response",
74 + schema=json_schema_from_tool,
75 + type="json_schema",
76 + description="Response format for tool calling",
77 + strict=True,
78 + )
79 +
80 return request
81
82 def extract_tool_calls(
vllm/entrypoints/responses_utils.py new
+45
@@ -0,0 +1,45 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from openai.types.chat import (
5 + ChatCompletionAssistantMessageParam,
6 + ChatCompletionMessageToolCallParam,
7 + ChatCompletionToolMessageParam,
8 +)
9 +from openai.types.chat.chat_completion_message_tool_call_param import (
10 + Function as FunctionCallTool,
11 +)
12 +from openai.types.responses import ResponseFunctionToolCall
13 +
14 +from vllm.entrypoints.openai.protocol import (
15 + ChatCompletionMessageParam,
16 + ResponseInputOutputItem,
17 +)
18 +
19 +
20 +def construct_chat_message_with_tool_call(
21 + item: ResponseInputOutputItem,
22 +) -> ChatCompletionMessageParam:
23 + if isinstance(item, ResponseFunctionToolCall):
24 + # Append the function call as a tool call.
25 + return ChatCompletionAssistantMessageParam(
26 + role="assistant",
27 + tool_calls=[
28 + ChatCompletionMessageToolCallParam(
29 + id=item.call_id,
30 + function=FunctionCallTool(
31 + name=item.name,
32 + arguments=item.arguments,
33 + ),
34 + type="function",
35 + )
36 + ],
37 + )
38 + elif item.get("type") == "function_call_output":
39 + # Append the function call output as a tool message.
40 + return ChatCompletionToolMessageParam(
41 + role="tool",
42 + content=item.get("output"),
43 + tool_call_id=item.get("call_id"),
44 + )
45 + return item # type: ignore