A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 22:45 UTC
[Frontend] add FunctionGemma tool parser support (#31218)
Signed-off-by: gateremark <gateremg@gmail.com>
Mark Gatere committed
Dec 25, 2025 at 10:29 UTC
ba25a6599212f01e6b877e22549612333e95b2c1
5 files changed
+557
docs/features/tool_calling.md
+24
@@ -372,6 +372,30 @@ Supported models:
372
373
Flags: `--tool-call-parser glm47`
374
375
+### FunctionGemma Models (`functiongemma`)
376
+
377
+Google's FunctionGemma is a lightweight (270M parameter) model specifically designed for function calling.
378
+It's built on Gemma 3 and optimized for edge deployment on devices like laptops and phones.
379
+
380
+Supported models:
381
+
382
+* `google/functiongemma-270m-it`
383
+
384
+FunctionGemma uses a unique output format with `<start_function_call>` and `<end_function_call>` tags:
385
+
386
+```text
387
+<start_function_call>call:get_weather{location:<escape>London<escape>}<end_function_call>
388
+```
389
+
390
+The model is designed to be fine-tuned for specific function-calling tasks for best results.
391
+
392
+Flags: `--tool-call-parser functiongemma --chat-template examples/tool_chat_template_functiongemma.jinja`
393
+
394
+!!! note
395
+ FunctionGemma is intended to be fine-tuned for your specific function-calling task.
396
+ The base model provides general function calling capabilities, but best results
397
+ are achieved with task-specific fine-tuning. See Google's [FunctionGemma documentation](https://ai.google.dev/gemma/docs/functiongemma) for fine-tuning guides.
398
+
399
### Qwen3-Coder Models (`qwen3_xml`)
400
401
Supported models:
examples/tool_chat_template_functiongemma.jinja
new
+54
@@ -0,0 +1,54 @@
1
+{%- set ns = namespace(developer_content='', has_tools=false) -%}
2
+
3
+{%- if tools is defined and tools | length > 0 -%}
4
+ {%- set ns.has_tools = true -%}
5
+{%- endif -%}
6
+
7
+{%- for message in messages -%}
8
+ {%- if message.role == 'developer' or message.role == 'system' -%}
9
+<start_of_turn>user
10
+{{ message.content }}
11
+{%- if ns.has_tools %}
12
+
13
+Available functions:
14
+{%- for tool in tools %}
15
+{%- if tool.type == 'function' %}
16
+
17
+Function: {{ tool.function.name }}
18
+Description: {{ tool.function.description | default('No description provided') }}
19
+Parameters: {{ tool.function.parameters | tojson }}
20
+{%- endif %}
21
+{%- endfor %}
22
+{%- endif %}
23
+<end_of_turn>
24
+ {%- elif message.role == 'user' -%}
25
+<start_of_turn>user
26
+{{ message.content }}<end_of_turn>
27
+ {%- elif message.role == 'assistant' -%}
28
+ {%- if message.tool_calls is defined and message.tool_calls | length > 0 -%}
29
+<start_of_turn>model
30
+{%- for tool_call in message.tool_calls %}
31
+<start_function_call>call:{{ tool_call.function.name }}{
32
+{%- set args = tool_call.function.arguments -%}
33
+{%- if args is string -%}
34
+{%- set args = args | fromjson -%}
35
+{%- endif -%}
36
+{%- for key, value in args.items() -%}
37
+{{ key }}:<escape>{{ value }}<escape>{% if not loop.last %},{% endif %}
38
+{%- endfor -%}
39
+}<end_function_call>
40
+{%- endfor %}
41
+<end_of_turn>
42
+ {%- else -%}
43
+<start_of_turn>model
44
+{{ message.content }}<end_of_turn>
45
+ {%- endif -%}
46
+ {%- elif message.role == 'tool' -%}
47
+<start_of_turn>user
48
+Function result for {{ message.name | default('function') }}: {{ message.content }}<end_of_turn>
49
+ {%- endif -%}
50
+{%- endfor -%}
51
+
52
+{%- if add_generation_prompt -%}
53
+<start_of_turn>model
54
+{%- endif -%}
tests/tool_parsers/test_functiongemma_tool_parser.py
new
+154
@@ -0,0 +1,154 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from unittest.mock import MagicMock
5
+
6
+import pytest
7
+
8
+from vllm.entrypoints.openai.protocol import ChatCompletionRequest
9
+from vllm.tool_parsers.functiongemma_tool_parser import FunctionGemmaToolParser
10
+
11
+
12
+@pytest.fixture
13
+def mock_tokenizer():
14
+ tokenizer = MagicMock()
15
+ tokenizer.encode.return_value = [1, 2, 3]
16
+ tokenizer.get_vocab.return_value = {}
17
+ return tokenizer
18
+
19
+
20
+@pytest.fixture
21
+def parser(mock_tokenizer):
22
+ return FunctionGemmaToolParser(mock_tokenizer)
23
+
24
+
25
+@pytest.fixture
26
+def mock_request():
27
+ request = MagicMock(spec=ChatCompletionRequest)
28
+ request.tools = []
29
+ request.tool_choice = "auto"
30
+ return request
31
+
32
+
33
+class TestExtractToolCalls:
34
+ def test_no_tool_calls(self, parser, mock_request):
35
+ model_output = "Hello, how can I help you today?"
36
+ result = parser.extract_tool_calls(model_output, mock_request)
37
+
38
+ assert result.tools_called is False
39
+ assert result.tool_calls == []
40
+ assert result.content == model_output
41
+
42
+ def test_single_tool_call(self, parser, mock_request):
43
+ model_output = (
44
+ "<start_function_call>call:get_weather{location:<escape>London<escape>}"
45
+ "<end_function_call>"
46
+ )
47
+ result = parser.extract_tool_calls(model_output, mock_request)
48
+
49
+ assert result.tools_called is True
50
+ assert len(result.tool_calls) == 1
51
+ assert result.tool_calls[0].function.name == "get_weather"
52
+ assert '"location": "London"' in result.tool_calls[0].function.arguments
53
+
54
+ def test_multiple_arguments(self, parser, mock_request):
55
+ model_output = (
56
+ "<start_function_call>call:get_weather{"
57
+ "location:<escape>San Francisco<escape>,"
58
+ "unit:<escape>celsius<escape>}"
59
+ "<end_function_call>"
60
+ )
61
+ result = parser.extract_tool_calls(model_output, mock_request)
62
+
63
+ assert result.tools_called is True
64
+ assert len(result.tool_calls) == 1
65
+ assert result.tool_calls[0].function.name == "get_weather"
66
+ args = result.tool_calls[0].function.arguments
67
+ assert "San Francisco" in args
68
+ assert "celsius" in args
69
+
70
+ def test_text_before_tool_call(self, parser, mock_request):
71
+ model_output = (
72
+ "Let me check the weather for you. "
73
+ "<start_function_call>call:get_weather{location:<escape>Paris<escape>}"
74
+ "<end_function_call>"
75
+ )
76
+ result = parser.extract_tool_calls(model_output, mock_request)
77
+
78
+ assert result.tools_called is True
79
+ assert result.content == "Let me check the weather for you."
80
+
81
+ def test_multiple_tool_calls(self, parser, mock_request):
82
+ model_output = (
83
+ "<start_function_call>call:get_weather{location:<escape>London<escape>}"
84
+ "<end_function_call>"
85
+ "<start_function_call>call:get_time{timezone:<escape>UTC<escape>}"
86
+ "<end_function_call>"
87
+ )
88
+ result = parser.extract_tool_calls(model_output, mock_request)
89
+
90
+ assert result.tools_called is True
91
+ assert len(result.tool_calls) == 2
92
+ assert result.tool_calls[0].function.name == "get_weather"
93
+ assert result.tool_calls[1].function.name == "get_time"
94
+
95
+
96
+class TestParseArguments:
97
+ def test_empty_arguments(self, parser):
98
+ result = parser._parse_arguments("")
99
+ assert result == {}
100
+
101
+ def test_single_string_argument(self, parser):
102
+ result = parser._parse_arguments("city:<escape>Tokyo<escape>")
103
+ assert result == {"city": "Tokyo"}
104
+
105
+ def test_multiple_arguments(self, parser):
106
+ args_str = "city:<escape>Tokyo<escape>,country:<escape>Japan<escape>"
107
+ result = parser._parse_arguments(args_str)
108
+ assert result == {"city": "Tokyo", "country": "Japan"}
109
+
110
+ def test_numeric_argument(self, parser):
111
+ result = parser._parse_arguments("count:<escape>42<escape>")
112
+ assert result == {"count": 42}
113
+
114
+ def test_boolean_argument(self, parser):
115
+ result = parser._parse_arguments("enabled:<escape>true<escape>")
116
+ assert result == {"enabled": True}
117
+
118
+ def test_argument_with_spaces(self, parser):
119
+ result = parser._parse_arguments("message:<escape>Hello World<escape>")
120
+ assert result == {"message": "Hello World"}
121
+
122
+
123
+class TestAdjustRequest:
124
+ def test_skip_special_tokens_disabled(self, parser, mock_request):
125
+ mock_request.tools = [{"type": "function", "function": {"name": "test"}}]
126
+ mock_request.tool_choice = "auto"
127
+ mock_request.skip_special_tokens = True
128
+
129
+ result = parser.adjust_request(mock_request)
130
+ assert result.skip_special_tokens is False
131
+
132
+ def test_skip_special_tokens_when_tool_choice_none(self, parser, mock_request):
133
+ mock_request.tools = [{"type": "function", "function": {"name": "test"}}]
134
+ mock_request.tool_choice = "none"
135
+ mock_request.skip_special_tokens = True
136
+
137
+ result = parser.adjust_request(mock_request)
138
+ assert result.skip_special_tokens is True
139
+
140
+
141
+class TestBufferDeltaText:
142
+ def test_regular_text_not_buffered(self, parser):
143
+ result = parser._buffer_delta_text("hello")
144
+ assert result == "hello"
145
+ assert parser.buffered_delta_text == ""
146
+
147
+ def test_complete_tag_flushed(self, parser):
148
+ parser.buffered_delta_text = "<start_function_"
149
+ result = parser._buffer_delta_text("call>")
150
+ assert "<start_function_call>" in result
151
+
152
+
153
+if __name__ == "__main__":
154
+ pytest.main([__file__, "-v"])
vllm/tool_parsers/__init__.py
+4
@@ -142,6 +142,10 @@ _TOOL_PARSERS_TO_REGISTER = {
142
"gigachat3_tool_parser",
143
"GigaChat3ToolParser",
144
),
145
+ "functiongemma": (
146
+ "functiongemma_tool_parser",
147
+ "FunctionGemmaToolParser",
148
+ ),
149
}
150
151
vllm/tool_parsers/functiongemma_tool_parser.py
new
+321
@@ -0,0 +1,321 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import json
5
+from collections.abc import Sequence
6
+
7
+import regex as re
8
+
9
+from vllm.entrypoints.chat_utils import make_tool_call_id
10
+from vllm.entrypoints.openai.protocol import (
11
+ ChatCompletionRequest,
12
+ DeltaFunctionCall,
13
+ DeltaMessage,
14
+ DeltaToolCall,
15
+ ExtractedToolCallInformation,
16
+ FunctionCall,
17
+ ToolCall,
18
+)
19
+from vllm.logger import init_logger
20
+from vllm.tokenizers import TokenizerLike
21
+from vllm.tool_parsers.abstract_tool_parser import ToolParser
22
+
23
+logger = init_logger(__name__)
24
+
25
+
26
+class FunctionGemmaToolParser(ToolParser):
27
+ """
28
+ Tool parser for Google's FunctionGemma model (google/functiongemma-270m-it).
29
+
30
+ Handles the FunctionGemma function call format:
31
+ <start_function_call>call:func_name{param:<escape>value<escape>}<end_function_call>
32
+ """
33
+
34
+ def __init__(self, tokenizer: TokenizerLike):
35
+ super().__init__(tokenizer)
36
+
37
+ # Streaming state
38
+ self.current_tool_name_sent: bool = False
39
+ self.prev_tool_call_arr: list[dict] = []
40
+ self.current_tool_id: int = -1
41
+ self.streamed_args_for_tool: list[str] = []
42
+
43
+ # FunctionGemma tokens
44
+ self.tool_call_start_token: str = "<start_function_call>"
45
+ self.tool_call_end_token: str = "<end_function_call>"
46
+
47
+ # Regex patterns
48
+ self.tool_call_regex = re.compile(
49
+ r"<start_function_call>call:(\w+)\{(.*?)\}<end_function_call>"
50
+ r"|<start_function_call>call:(\w+)\{(.*)",
51
+ re.DOTALL,
52
+ )
53
+ self.arg_regex = re.compile(
54
+ r"(\w+):<escape>(.*?)<escape>",
55
+ re.DOTALL,
56
+ )
57
+
58
+ if self.model_tokenizer:
59
+ self.tool_call_start_token_ids = self.model_tokenizer.encode(
60
+ self.tool_call_start_token, add_special_tokens=False
61
+ )
62
+ self.tool_call_end_token_ids = self.model_tokenizer.encode(
63
+ self.tool_call_end_token, add_special_tokens=False
64
+ )
65
+ else:
66
+ self.tool_call_start_token_ids = []
67
+ self.tool_call_end_token_ids = []
68
+
69
+ self.buffered_delta_text = ""
70
+
71
+ def _parse_arguments(self, args_str: str) -> dict:
72
+ """Parse FunctionGemma argument string into a dictionary."""
73
+ arguments = {}
74
+ if not args_str:
75
+ return arguments
76
+
77
+ matches = self.arg_regex.findall(args_str)
78
+ for key, value in matches:
79
+ try:
80
+ parsed_value = json.loads(value)
81
+ arguments[key] = parsed_value
82
+ except json.JSONDecodeError:
83
+ arguments[key] = value
84
+
85
+ return arguments
86
+
87
+ def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest:
88
+ request = super().adjust_request(request)
89
+ if request.tools and request.tool_choice != "none":
90
+ request.skip_special_tokens = False
91
+ return request
92
+
93
+ def extract_tool_calls(
94
+ self,
95
+ model_output: str,
96
+ request: ChatCompletionRequest,
97
+ ) -> ExtractedToolCallInformation:
98
+ if self.tool_call_start_token not in model_output:
99
+ return ExtractedToolCallInformation(
100
+ tools_called=False, tool_calls=[], content=model_output
101
+ )
102
+
103
+ try:
104
+ matches = self.tool_call_regex.findall(model_output)
105
+
106
+ if not matches:
107
+ return ExtractedToolCallInformation(
108
+ tools_called=False, tool_calls=[], content=model_output
109
+ )
110
+
111
+ tool_calls: list[ToolCall] = []
112
+
113
+ for match in matches:
114
+ func_name = match[0] if match[0] else match[2]
115
+ args_str = match[1] if match[1] else match[3]
116
+
117
+ if not func_name:
118
+ continue
119
+
120
+ arguments = self._parse_arguments(args_str)
121
+
122
+ tool_calls.append(
123
+ ToolCall(
124
+ type="function",
125
+ function=FunctionCall(
126
+ name=func_name,
127
+ arguments=json.dumps(arguments, ensure_ascii=False),
128
+ ),
129
+ )
130
+ )
131
+
132
+ if tool_calls:
133
+ content_end = model_output.find(self.tool_call_start_token)
134
+ content = (
135
+ model_output[:content_end].strip() if content_end > 0 else None
136
+ )
137
+
138
+ return ExtractedToolCallInformation(
139
+ tools_called=True,
140
+ tool_calls=tool_calls,
141
+ content=content if content else None,
142
+ )
143
+
144
+ return ExtractedToolCallInformation(
145
+ tools_called=False, tool_calls=[], content=model_output
146
+ )
147
+
148
+ except Exception:
149
+ logger.exception("Error extracting tool calls from FunctionGemma response")
150
+ return ExtractedToolCallInformation(
151
+ tools_called=False, tool_calls=[], content=model_output
152
+ )
153
+
154
+ def _buffer_delta_text(self, delta_text: str) -> str:
155
+ """Buffer incoming delta text to handle multi-token special sequences."""
156
+ potential_start = "<start_function_call>"
157
+ potential_end = "<end_function_call>"
158
+
159
+ combined = self.buffered_delta_text + delta_text
160
+
161
+ if combined.endswith(potential_start) or combined.endswith(potential_end):
162
+ self.buffered_delta_text = ""
163
+ return combined
164
+
165
+ for tag in [potential_start, potential_end]:
166
+ for i in range(1, len(tag)):
167
+ if combined.endswith(tag[:i]):
168
+ self.buffered_delta_text = combined[-(i):]
169
+ return combined[:-i]
170
+
171
+ self.buffered_delta_text = ""
172
+ return combined
173
+
174
+ def extract_tool_calls_streaming(
175
+ self,
176
+ previous_text: str,
177
+ current_text: str,
178
+ delta_text: str,
179
+ previous_token_ids: Sequence[int],
180
+ current_token_ids: Sequence[int],
181
+ delta_token_ids: Sequence[int],
182
+ request: ChatCompletionRequest,
183
+ ) -> DeltaMessage | None:
184
+ delta_text = self._buffer_delta_text(delta_text)
185
+ current_text = previous_text + delta_text
186
+
187
+ if self.tool_call_start_token not in current_text:
188
+ if delta_text:
189
+ return DeltaMessage(content=delta_text)
190
+ return None
191
+
192
+ try:
193
+ start_count = current_text.count(self.tool_call_start_token)
194
+ end_count = current_text.count(self.tool_call_end_token)
195
+ prev_start_count = previous_text.count(self.tool_call_start_token)
196
+ prev_end_count = previous_text.count(self.tool_call_end_token)
197
+
198
+ if self.tool_call_start_token not in current_text:
199
+ return DeltaMessage(content=delta_text)
200
+
201
+ # Starting a new function call
202
+ if start_count > prev_start_count and start_count > end_count:
203
+ self.current_tool_id += 1
204
+ self.current_tool_name_sent = False
205
+ self.streamed_args_for_tool.append("")
206
+ self.prev_tool_call_arr.append({})
207
+ logger.debug("Starting new tool call %d", self.current_tool_id)
208
+ return None
209
+
210
+ # In the middle of a function call
211
+ if start_count > end_count:
212
+ last_start = current_text.rfind(self.tool_call_start_token)
213
+ partial_call = current_text[
214
+ last_start + len(self.tool_call_start_token) :
215
+ ]
216
+
217
+ if partial_call.startswith("call:"):
218
+ func_part = partial_call[5:]
219
+
220
+ if "{" in func_part:
221
+ func_name = func_part.split("{")[0]
222
+ args_part = (
223
+ func_part.split("{", 1)[1] if "{" in func_part else ""
224
+ )
225
+
226
+ if not self.current_tool_name_sent and func_name:
227
+ self.current_tool_name_sent = True
228
+ self.prev_tool_call_arr[self.current_tool_id] = {
229
+ "name": func_name,
230
+ "arguments": {},
231
+ }
232
+ return DeltaMessage(
233
+ tool_calls=[
234
+ DeltaToolCall(
235
+ index=self.current_tool_id,
236
+ type="function",
237
+ id=make_tool_call_id(),
238
+ function=DeltaFunctionCall(
239
+ name=func_name
240
+ ).model_dump(exclude_none=True),
241
+ )
242
+ ]
243
+ )
244
+
245
+ if self.current_tool_name_sent and args_part:
246
+ current_args = self._parse_arguments(args_part)
247
+ if current_args:
248
+ current_args_json = json.dumps(
249
+ current_args, ensure_ascii=False
250
+ )
251
+ prev_streamed = self.streamed_args_for_tool[
252
+ self.current_tool_id
253
+ ]
254
+
255
+ if len(current_args_json) > len(prev_streamed):
256
+ diff = current_args_json[len(prev_streamed) :]
257
+ self.streamed_args_for_tool[
258
+ self.current_tool_id
259
+ ] = current_args_json
260
+ self.prev_tool_call_arr[self.current_tool_id][
261
+ "arguments"
262
+ ] = current_args
263
+
264
+ return DeltaMessage(
265
+ tool_calls=[
266
+ DeltaToolCall(
267
+ index=self.current_tool_id,
268
+ function=DeltaFunctionCall(
269
+ arguments=diff
270
+ ).model_dump(exclude_none=True),
271
+ )
272
+ ]
273
+ )
274
+
275
+ return None
276
+
277
+ # Function call just ended
278
+ if end_count > prev_end_count:
279
+ if self.current_tool_id >= 0 and self.current_tool_id < len(
280
+ self.prev_tool_call_arr
281
+ ):
282
+ all_calls = self.tool_call_regex.findall(current_text)
283
+ args = {}
284
+ if self.current_tool_id < len(all_calls):
285
+ match = all_calls[self.current_tool_id]
286
+ if match[0]:
287
+ args_str = match[1]
288
+ args = self._parse_arguments(args_str)
289
+ self.prev_tool_call_arr[self.current_tool_id][
290
+ "arguments"
291
+ ] = args
292
+
293
+ if args:
294
+ args_json = json.dumps(args, ensure_ascii=False)
295
+ prev_streamed = self.streamed_args_for_tool[
296
+ self.current_tool_id
297
+ ]
298
+ if len(args_json) > len(prev_streamed):
299
+ diff = args_json[len(prev_streamed) :]
300
+ self.streamed_args_for_tool[self.current_tool_id] = (
301
+ args_json
302
+ )
303
+ return DeltaMessage(
304
+ tool_calls=[
305
+ DeltaToolCall(
306
+ index=self.current_tool_id,
307
+ function=DeltaFunctionCall(
308
+ arguments=diff
309
+ ).model_dump(exclude_none=True),
310
+ )
311
+ ]
312
+ )
313
+ return None
314
+
315
+ if delta_text:
316
+ return DeltaMessage(content=delta_text)
317
+ return None
318
+
319
+ except Exception:
320
+ logger.exception("Error in streaming tool call extraction")
321
+ return None