A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 02:15 UTC
[Frontend] Support Tool Calling with both `tool_choice='required'` and `$defs`. (#20629)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Jul 10, 2025 at 13:56 UTC
8f2720def9c32163822d9ccf9a31019f7a892e12
2 files changed
+56
tests/entrypoints/openai/test_completion_with_function_calling.py
+35
@@ -72,8 +72,43 @@ async def test_function_tool_use(client: openai.AsyncOpenAI, model_name: str,
72
"The unit to fetch the temperature in",
73
"enum": ["celsius", "fahrenheit"],
74
},
75
+ "options": {
76
+ "$ref": "#/$defs/WeatherOptions",
77
+ "description":
78
+ "Optional parameters for weather query",
79
+ },
80
},
81
"required": ["country", "unit"],
82
+ "$defs": {
83
+ "WeatherOptions": {
84
+ "title": "WeatherOptions",
85
+ "type": "object",
86
+ "additionalProperties": False,
87
+ "properties": {
88
+ "unit": {
89
+ "type": "string",
90
+ "enum": ["celsius", "fahrenheit"],
91
+ "default": "celsius",
92
+ "description": "Temperature unit",
93
+ "title": "Temperature Unit",
94
+ },
95
+ "include_forecast": {
96
+ "type": "boolean",
97
+ "default": False,
98
+ "description":
99
+ "Whether to include a 24-hour forecast",
100
+ "title": "Include Forecast",
101
+ },
102
+ "language": {
103
+ "type": "string",
104
+ "default": "zh-CN",
105
+ "description": "Language of the response",
106
+ "title": "Language",
107
+ "enum": ["zh-CN", "en-US", "ja-JP"],
108
+ },
109
+ },
110
+ },
111
+ },
112
},
113
},
114
},
vllm/entrypoints/openai/protocol.py
+21
@@ -707,6 +707,24 @@ class ChatCompletionRequest(OpenAIBaseModel):
707
"required": ["name", "parameters"]
708
}
709
710
+ def get_tool_schema_defs(
711
+ tools: list[ChatCompletionToolsParam]) -> dict:
712
+ all_defs = dict[str, dict[str, Any]]()
713
+ for tool in tools:
714
+ if tool.function.parameters is None:
715
+ continue
716
+ defs = tool.function.parameters.pop("$defs", {})
717
+ for def_name, def_schema in defs.items():
718
+ if def_name in all_defs and all_defs[
719
+ def_name] != def_schema:
720
+ raise ValueError(
721
+ f"Tool definition '{def_name}' has "
722
+ "multiple schemas, which is not "
723
+ "supported.")
724
+ else:
725
+ all_defs[def_name] = def_schema
726
+ return all_defs
727
+
728
json_schema = {
729
"type": "array",
730
"minItems": 1,
@@ -715,6 +733,9 @@ class ChatCompletionRequest(OpenAIBaseModel):
733
"anyOf": [get_tool_schema(tool) for tool in self.tools]
734
}
735
}
736
+ json_schema_defs = get_tool_schema_defs(self.tools)
737
+ if json_schema_defs:
738
+ json_schema["$defs"] = json_schema_defs
739
return json_schema
740
741
return None