@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9f9ecff4cd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 6, 15:45 UTC

Add simple granite4 tool parser (#36827)

Signed-off-by: Max de Bayser <maxdebayser@gmail.com> Signed-off-by: Max de Bayser <mbayser@br.ibm.com>

Max de Bayser committed Mar 16, 2026 at 14:49 UTC 9f9ecff4cdff5b8847f541b896c0ca081397cc51
5 files changed +850 -168
docs/features/tool_calling.md
+1 -1
@@ -219,7 +219,7 @@ Supported models:
219
220 * `ibm-granite/granite-4.0-h-small` and other Granite 4.0 models
221
222 - Recommended flags: `--tool-call-parser hermes`
222 + Recommended flags: `--tool-call-parser granite4`
223
224 * `ibm-granite/granite-3.0-8b-instruct`
225
tests/entrypoints/openai/tool_parsers/test_granite4_tool_parser.py new
+360
@@ -0,0 +1,360 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import json
4 +import random
5 +from typing import Any
6 +
7 +import openai
8 +import pytest
9 +from transformers import AutoTokenizer
10 +
11 +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
12 +from vllm.entrypoints.openai.engine.protocol import (
13 + DeltaMessage,
14 +)
15 +from vllm.tool_parsers.granite4_tool_parser import Granite4ToolParser
16 +
17 +from ....utils import RemoteOpenAIServer
18 +
19 +MODEL = "ibm-granite/granite-4.0-h-tiny"
20 +
21 +
22 +@pytest.fixture(scope="module")
23 +def server():
24 + model = MODEL
25 + args_for_model = [
26 + "--enforce-eager",
27 + "--enable-auto-tool-choice",
28 + "--tool-call-parser",
29 + "granite4",
30 + "--tokenizer",
31 + "ibm-granite/granite-4.0-h-tiny",
32 + "--max-model-len",
33 + "4096",
34 + "--max-num-seqs",
35 + "2",
36 + ]
37 + with RemoteOpenAIServer(model, args_for_model, max_wait_seconds=480) as server:
38 + yield server
39 +
40 +
41 +def create_complex_input(create_string_args: bool):
42 + coord_arg: dict | str = {
43 + "coordinates": [[23.54, 43.1], [-12.2, 54.3], [4, 5]],
44 + "coordinate_type": "latlong",
45 + }
46 + if create_string_args:
47 + # test granite behavior
48 + coord_arg = json.dumps(coord_arg)
49 + return [
50 + {"name": "find_bbox", "arguments": coord_arg},
51 + {
52 + "name": "get_stock_price",
53 + "arguments": {
54 + "symbol": "AAPL",
55 + "start_date": "2021-01-01",
56 + "end_date": "2021-12-31",
57 + },
58 + },
59 + {"name": "find_bbox", "arguments": coord_arg},
60 + ]
61 +
62 +
63 +def random_chunks(s: str, min_len: int, max_len: int):
64 + chunks = []
65 + i = 0
66 + n = len(s)
67 +
68 + while i < n:
69 + size = random.randint(min_len, max_len)
70 + chunks.append(s[i : i + size])
71 + i += size
72 +
73 + return chunks
74 +
75 +
76 +@pytest.fixture(scope="module")
77 +def tokenizer():
78 + return AutoTokenizer.from_pretrained(MODEL)
79 +
80 +
81 +# create a variety of input chunk sizes
82 +@pytest.mark.parametrize(
83 + "min_chunk, max_chunk",
84 + [
85 + (1, 1),
86 + (1, 2),
87 + (5, 7),
88 + (6, 20),
89 + ],
90 +)
91 +def test_tool_call_parser_complex(min_chunk: int, max_chunk: int, tokenizer):
92 + input_dicts = create_complex_input(True)
93 +
94 + formatted_tcs = [
95 + "<tool_call> " + json.dumps(call) + " </tool_call>" for call in input_dicts
96 + ]
97 +
98 + text_messages = [
99 + "Here goes the bbox call: \n",
100 + " Now the stock price call: \n ",
101 + " Now another bbox call: \n ",
102 + " See? I'm a helpful assistant.",
103 + ]
104 +
105 + test_input = (
106 + text_messages[0]
107 + + formatted_tcs[0]
108 + + text_messages[1]
109 + + formatted_tcs[1]
110 + + text_messages[2]
111 + + formatted_tcs[2]
112 + + text_messages[3]
113 + )
114 +
115 + any_chat_request = ChatCompletionRequest(
116 + seed=42,
117 + model=MODEL,
118 + messages=[],
119 + )
120 +
121 + parser = Granite4ToolParser(tokenizer=tokenizer)
122 +
123 + delta_messages = list[DeltaMessage]()
124 + for text in random_chunks(test_input, min_chunk, max_chunk):
125 + delta = parser.extract_tool_calls_streaming(
126 + previous_text="",
127 + current_text="",
128 + delta_text=text,
129 + previous_token_ids=[],
130 + current_token_ids=[],
131 + delta_token_ids=[],
132 + request=any_chat_request,
133 + )
134 + if delta is not None:
135 + delta_messages.append(delta)
136 +
137 + content = ""
138 + tool_calls = list[dict[str, Any]]()
139 +
140 + current_name = "__start__"
141 + current_args = ""
142 +
143 + for msg in delta_messages:
144 + if msg.content:
145 + content += msg.content
146 + for tool_call in msg.tool_calls:
147 + if delta_func := tool_call.function:
148 + if delta_func.name is not None:
149 + if current_name == "__start__":
150 + current_name = delta_func.name
151 +
152 + if delta_func.name != current_name:
153 + tool_calls.append(
154 + {
155 + "name": current_name,
156 + "arguments": json.loads(current_args),
157 + }
158 + )
159 + current_name = delta_func.name
160 + current_args = ""
161 +
162 + if delta_func.arguments:
163 + current_args += delta_func.arguments
164 +
165 + if current_name != "__start__":
166 + tool_calls.append({"name": current_name, "arguments": json.loads(current_args)})
167 +
168 + assert content == "".join(text_messages)
169 + assert tool_calls == create_complex_input(False)
170 +
171 +
172 +tools = [
173 + {
174 + "type": "function",
175 + "function": {
176 + "name": "get_acme_region_name_for_transaction_id",
177 + "description": "Returns ACME transaction/transaction ID information"
178 + " including ACME regions\n\nArgs:\n start_time "
179 + "(str): Start date and time in datetime format "
180 + '"%Y-%m-%dT%H:%M:%S.%f"\n end_time (str): End '
181 + "date and time in datetime format "
182 + '"%Y-%m-%dT%H:%M:%S.%f"\n size (int, optional): '
183 + "Number of ACME Transaction IDs to return\n "
184 + "order (str, optional): Sort by most run "
185 + "transaction IDs. The value can be 'asc' for "
186 + "ascending or 'desc' for descending\n "
187 + "transaction_id (str, optional): ACME Transaction "
188 + "ID to filter on\n acme_region (str, optional): "
189 + "ACME Region to filter on\nReturns:\n - A "
190 + "dictionary containing a list of ACME transaction "
191 + "ids and the ACME regions they run in:\n {\n"
192 + ' "Number of transaction IDs" : int,\n'
193 + ' "Total transaction IDs available": int'
194 + ',\n "ACME Transaction IDs": [\n '
195 + ' {\n "Transaction ID": '
196 + 'str,\n "Number of runs": int,\n'
197 + ' "ACME Regions": [str],\n '
198 + " },\n ...\n ],"
199 + '\n "Start time" : datetime,\n '
200 + ' "End time" : datetime,\n '
201 + ' "Order" : str\n }\n '
202 + " - If no ACME region found for transaction id, "
203 + 'returns:\n {"Success": "No ACME region '
204 + 'found for transaction id."}\n - If an error '
205 + 'occurs, returns:\n {"Error": "{exception'
206 + ' message}"}',
207 + "parameters": {
208 + "properties": {
209 + "start_time": {},
210 + "end_time": {},
211 + "size": {"default": 500},
212 + "order": {"default": "desc"},
213 + "transaction_id": {"default": None},
214 + "acme_region": {"default": None},
215 + },
216 + "required": ["start_time", "end_time"],
217 + "type": "object",
218 + },
219 + },
220 + }
221 +]
222 +
223 +tools2 = [
224 + {
225 + "type": "function",
226 + "function": {
227 + "name": "get_current_weather",
228 + "description": "Get the current weather",
229 + "parameters": {
230 + "type": "object",
231 + "properties": {
232 + "location": {
233 + "description": "The city and state, e.g. San Francisco, CA",
234 + "type": "string",
235 + }
236 + },
237 + "required": ["location"],
238 + },
239 + },
240 + },
241 + {
242 + "type": "function",
243 + "function": {
244 + "name": "get_stock_price",
245 + "description": "Retrieves the current stock price for a given "
246 + "ticker symbol. The ticker symbol must be a valid "
247 + "symbol for a publicly traded company on a major US"
248 + " stock exchange like NYSE or NASDAQ. The tool will"
249 + " return the latest trade price in USD. It should "
250 + "be used when the user asks about the current or "
251 + "most recent price of a specific stock. It will not"
252 + " provide any other information about the stock or"
253 + " company.",
254 + "parameters": {
255 + "type": "object",
256 + "properties": {
257 + "ticker": {
258 + "description": "The stock ticker symbol, e.g."
259 + " AAPL for Apple Inc.",
260 + "type": "string",
261 + }
262 + },
263 + },
264 + },
265 + },
266 +]
267 +
268 +messages = [
269 + {
270 + "content": "\n\nSystem: You are a helpful, precise, and methodical AI"
271 + " assistant that uses tool outputs provided inline.\nAlways"
272 + " assume the current datetime is 2026-01-29T13:59:09.238901"
273 + "+00:00.\n\nIf you receive a ToolMessage with `tool_call_id"
274 + '` equal to "get_time_range" (or "time_range_tool"), you '
275 + "MUST:\n 1. Parse that JSON and use the values `start` and"
276 + " `end` directly when calling other tools.\n 2. Do not "
277 + "re-call or re-compute the time range.\n 3. Pass resolved "
278 + "values (ISO strings) as arguments to any subsequent tool "
279 + "(do not pass function metadata or placeholders).\n 4. If "
280 + "a tool requires datetime objects rather than strings, "
281 + "convert the ISO strings into language-native datetime "
282 + "objects before invoking.\n\nAlways return fully resolved "
283 + "arguments in correct types (e.g., ISO datetime strings or"
284 + " datetime objects) and never include placeholders like "
285 + '"<start>".\n\n',
286 + "role": "system",
287 + },
288 + {
289 + "content": "What are the transaction IDs that ran in the"
290 + " ACME region A9345 over the last two months?",
291 + "role": "user",
292 + },
293 + {
294 + "content": '["2026-01-26T09: 51: 55.467722Z", "2026-01-27T09: 51: 55.467722Z"]',
295 + "role": "tool",
296 + "tool_call_id": "time_range_tool",
297 + },
298 +]
299 +messages2 = [{"role": "user", "content": "What's stock price for IBM?"}]
300 +
301 +messages3 = [{"role": "user", "content": "What's the current weather in New York?"}]
302 +
303 +
304 +def get_args(client: openai.OpenAI, _tools, _messages, _stop):
305 + response = client.chat.completions.create(
306 + model=MODEL,
307 + messages=_messages,
308 + temperature=0,
309 + tools=_tools,
310 + max_tokens=200,
311 + stop=_stop,
312 + tool_choice="auto",
313 + )
314 +
315 + return response.choices[0].message.tool_calls[0].function.arguments
316 +
317 +
318 +async def get_args_streaming(
319 + async_client: openai.AsyncOpenAI, _tools, _messages, _stop
320 +):
321 + stream = await async_client.chat.completions.create(
322 + model=MODEL,
323 + messages=_messages,
324 + temperature=0,
325 + tools=_tools,
326 + max_tokens=200,
327 + stop=_stop,
328 + tool_choice="auto",
329 + stream=True,
330 + )
331 + full_call = []
332 + async for chunk in stream:
333 + tc = chunk.choices[0].delta.tool_calls
334 + if tc and tc[0].function.arguments:
335 + full_call.append(tc[0].function.arguments)
336 + return "".join(full_call)
337 +
338 +
339 +async def run_scenario(server: RemoteOpenAIServer, _tools, _messages, _stop):
340 + non_streaming = get_args(server.get_client(), _tools, _messages, _stop)
341 + json.loads(non_streaming) # verify that it is json loadable
342 + streaming = await get_args_streaming(
343 + server.get_async_client(), _tools, _messages, _stop
344 + )
345 + json.loads(streaming)
346 + assert non_streaming == streaming, f"{non_streaming=}, {streaming=}"
347 +
348 +
349 +@pytest.mark.asyncio
350 +async def test_stop_sequence_interference(server: RemoteOpenAIServer):
351 + print("Testing scenario 1")
352 + await run_scenario(server, tools, messages, "veroniqueprattyushveroniqueprattyush")
353 +
354 + print("Testing scenario 2")
355 + await run_scenario(
356 + server, tools2, messages2, "veroniqueprattyushveroniqueprattyush"
357 + )
358 +
359 + print("Testing scenario 3")
360 + await run_scenario(server, tools2, messages3, "prattyush")
tests/entrypoints/openai/tool_parsers/test_hermes_tool_parser.py
+233 -167
@@ -3,29 +3,22 @@
3
4 import json
5
6 +import openai
7 import pytest
8 +import pytest_asyncio
9 +from huggingface_hub import snapshot_download
10 +from typing_extensions import TypedDict
11
12 from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
13 from vllm.tokenizers import TokenizerLike
14 +from vllm.tool_parsers.abstract_tool_parser import ToolParser
15 +from vllm.tool_parsers.granite4_tool_parser import Granite4ToolParser
16 from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
17
18 from ....utils import RemoteOpenAIServer
19
14 -MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct"
20 LORA_MODEL = "minpeter/LoRA-Llama-3.2-1B-tool-vllm-ci"
21
17 -SERVER_ARGS = [
18 - "--enforce-eager",
19 - "--enable-auto-tool-choice",
20 - "--tool-call-parser",
21 - "hermes",
22 - "--enable-lora",
23 - "--lora-modules",
24 - f"{LORA_MODEL}={LORA_MODEL}",
25 - "--tokenizer",
26 - f"{LORA_MODEL}",
27 -]
28 -
22 TOOLS = [
23 {
24 "type": "function",
@@ -50,6 +43,75 @@ TOOLS = [
43 }
44 ]
45
46 +
47 +class ServerConfig(TypedDict, total=False):
48 + model: str
49 + arguments: list[str]
50 + model_arg: str
51 + tool_parser: ToolParser
52 +
53 +
54 +CONFIGS: dict[str, ServerConfig] = {
55 + "llama": {
56 + "model": "meta-llama/Llama-3.2-1B-Instruct",
57 + "arguments": [
58 + "--enforce-eager",
59 + "--enable-auto-tool-choice",
60 + "--tool-call-parser",
61 + "hermes",
62 + "--enable-lora",
63 + "--lora-modules",
64 + f"{LORA_MODEL}={LORA_MODEL}",
65 + "--tokenizer",
66 + f"{LORA_MODEL}",
67 + ],
68 + "model_arg": LORA_MODEL,
69 + "tool_parser": Hermes2ProToolParser,
70 + },
71 + "granite4": {
72 + "model": "ibm-granite/granite-4.0-h-tiny",
73 + "arguments": [
74 + "--enforce-eager",
75 + "--enable-auto-tool-choice",
76 + "--tool-call-parser",
77 + "granite4",
78 + "--tokenizer",
79 + "ibm-granite/granite-4.0-h-tiny",
80 + "--max-model-len",
81 + "4096",
82 + "--max-num-seqs",
83 + "2",
84 + ],
85 + "model_arg": "ibm-granite/granite-4.0-h-tiny",
86 + "tool_parser": Granite4ToolParser,
87 + },
88 +}
89 +
90 +
91 +# for each server config, download the model and return the config
92 +@pytest.fixture(scope="session", params=CONFIGS.keys())
93 +def server_config(request):
94 + config = CONFIGS[request.param]
95 +
96 + # download model and tokenizer using transformers
97 + snapshot_download(config["model"])
98 + yield CONFIGS[request.param]
99 +
100 +
101 +@pytest.fixture(scope="module")
102 +def server(request, server_config: ServerConfig):
103 + model = server_config["model"]
104 + args_for_model = server_config["arguments"]
105 + with RemoteOpenAIServer(model, args_for_model, max_wait_seconds=480) as server:
106 + yield server
107 +
108 +
109 +@pytest_asyncio.fixture
110 +async def client(server: RemoteOpenAIServer):
111 + async with server.get_async_client() as async_client:
112 + yield async_client
113 +
114 +
115 PRODUCT_TOOLS = [
116 {
117 "type": "function",
@@ -87,186 +149,182 @@ PRODUCT_MESSAGES = [
149
150
151 @pytest.mark.asyncio
90 -async def test_non_streaming_tool_call():
152 +async def test_non_streaming_tool_call(
153 + client: openai.AsyncOpenAI, server_config: ServerConfig
154 +):
155 """Test tool call in non-streaming mode."""
92 - with RemoteOpenAIServer(MODEL_NAME, SERVER_ARGS) as server:
93 - client = server.get_async_client()
94 -
95 - response = await client.chat.completions.create(
96 - model=LORA_MODEL,
97 - messages=MESSAGES,
98 - tools=TOOLS,
99 - tool_choice="auto",
100 - temperature=0.0,
101 - )
156
103 - assert response.choices
104 - choice = response.choices[0]
105 - message = choice.message
157 + response = await client.chat.completions.create(
158 + model=server_config["model_arg"],
159 + messages=MESSAGES,
160 + tools=TOOLS,
161 + tool_choice="auto",
162 + temperature=0.0,
163 + )
164 +
165 + assert response.choices
166 + choice = response.choices[0]
167 + message = choice.message
168
107 - assert choice.finish_reason == "tool_calls"
108 - assert message.tool_calls is not None
169 + assert choice.finish_reason == "tool_calls"
170 + assert message.tool_calls is not None
171
110 - tool_call = message.tool_calls[0]
111 - assert tool_call.type == "function"
112 - assert tool_call.function.name == "get_current_weather"
172 + tool_call = message.tool_calls[0]
173 + assert tool_call.type == "function"
174 + assert tool_call.function.name == "get_current_weather"
175
114 - arguments = json.loads(tool_call.function.arguments)
115 - assert "location" in arguments
116 - assert "Boston" in arguments["location"]
117 - print("\n[Non-Streaming Test Passed]")
118 - print(f"Tool Call: {tool_call.function.name}")
119 - print(f"Arguments: {arguments}")
176 + arguments = json.loads(tool_call.function.arguments)
177 + assert "location" in arguments
178 + assert "Boston" in arguments["location"]
179 + print("\n[Non-Streaming Test Passed]")
180 + print(f"Tool Call: {tool_call.function.name}")
181 + print(f"Arguments: {arguments}")
182
183
184 @pytest.mark.asyncio
123 -async def test_streaming_tool_call():
185 +async def test_streaming_tool_call(
186 + client: openai.AsyncOpenAI, server_config: ServerConfig
187 +):
188 """Test tool call in streaming mode."""
125 - with RemoteOpenAIServer(MODEL_NAME, SERVER_ARGS) as server:
126 - client = server.get_async_client()
127 -
128 - stream = await client.chat.completions.create(
129 - model=LORA_MODEL,
130 - messages=MESSAGES,
131 - tools=TOOLS,
132 - tool_choice="auto",
133 - temperature=0.0,
134 - stream=True,
135 - )
189
137 - tool_call_chunks = {}
138 - async for chunk in stream:
139 - if not chunk.choices:
140 - continue
190 + stream = await client.chat.completions.create(
191 + model=server_config["model_arg"],
192 + messages=MESSAGES,
193 + tools=TOOLS,
194 + tool_choice="auto",
195 + temperature=0.0,
196 + stream=True,
197 + )
198 +
199 + tool_call_chunks = {}
200 + async for chunk in stream:
201 + if not chunk.choices:
202 + continue
203
142 - delta = chunk.choices[0].delta
143 - if not delta or not delta.tool_calls:
144 - continue
204 + delta = chunk.choices[0].delta
205 + if not delta or not delta.tool_calls:
206 + continue
207
146 - for tool_chunk in delta.tool_calls:
147 - index = tool_chunk.index
148 - if index not in tool_call_chunks:
149 - tool_call_chunks[index] = {"name": "", "arguments": ""}
208 + for tool_chunk in delta.tool_calls:
209 + index = tool_chunk.index
210 + if index not in tool_call_chunks:
211 + tool_call_chunks[index] = {"name": "", "arguments": ""}
212
151 - if tool_chunk.function.name:
152 - tool_call_chunks[index]["name"] += tool_chunk.function.name
153 - if tool_chunk.function.arguments:
154 - tool_call_chunks[index]["arguments"] += (
155 - tool_chunk.function.arguments
156 - )
213 + if tool_chunk.function.name:
214 + tool_call_chunks[index]["name"] += tool_chunk.function.name
215 + if tool_chunk.function.arguments:
216 + tool_call_chunks[index]["arguments"] += tool_chunk.function.arguments
217
158 - assert len(tool_call_chunks) == 1
159 - reconstructed_tool_call = tool_call_chunks[0]
218 + assert len(tool_call_chunks) == 1
219 + reconstructed_tool_call = tool_call_chunks[0]
220
161 - assert reconstructed_tool_call["name"] == "get_current_weather"
221 + assert reconstructed_tool_call["name"] == "get_current_weather"
222
163 - arguments = json.loads(reconstructed_tool_call["arguments"])
164 - assert "location" in arguments
165 - assert "Boston" in arguments["location"]
166 - print("\n[Streaming Test Passed]")
167 - print(f"Reconstructed Tool Call: {reconstructed_tool_call['name']}")
168 - print(f"Reconstructed Arguments: {arguments}")
223 + arguments = json.loads(reconstructed_tool_call["arguments"])
224 + assert "location" in arguments
225 + assert "Boston" in arguments["location"]
226 + print("\n[Streaming Test Passed]")
227 + print(f"Reconstructed Tool Call: {reconstructed_tool_call['name']}")
228 + print(f"Reconstructed Arguments: {arguments}")
229
230
231 @pytest.mark.asyncio
172 -async def test_non_streaming_product_tool_call():
232 +async def test_non_streaming_product_tool_call(
233 + client: openai.AsyncOpenAI, server_config: ServerConfig
234 +):
235 """Test tool call integer and boolean parameters in non-streaming mode."""
174 - with RemoteOpenAIServer(MODEL_NAME, SERVER_ARGS) as server:
175 - client = server.get_async_client()
176 -
177 - response = await client.chat.completions.create(
178 - model=LORA_MODEL,
179 - messages=PRODUCT_MESSAGES,
180 - tools=PRODUCT_TOOLS,
181 - tool_choice="auto",
182 - temperature=0.66,
183 - )
236
185 - assert response.choices
186 - choice = response.choices[0]
187 - message = choice.message
237 + response = await client.chat.completions.create(
238 + model=server_config["model_arg"],
239 + messages=PRODUCT_MESSAGES,
240 + tools=PRODUCT_TOOLS,
241 + tool_choice="auto",
242 + temperature=0.66,
243 + )
244 +
245 + assert response.choices
246 + choice = response.choices[0]
247 + message = choice.message
248
189 - assert choice.finish_reason == "tool_calls"
190 - assert message.tool_calls is not None
249 + assert choice.finish_reason == "tool_calls"
250 + assert message.tool_calls is not None
251
192 - tool_call = message.tool_calls[0]
193 - assert tool_call.type == "function"
194 - assert tool_call.function.name == "get_product_info"
252 + tool_call = message.tool_calls[0]
253 + assert tool_call.type == "function"
254 + assert tool_call.function.name == "get_product_info"
255
196 - arguments = json.loads(tool_call.function.arguments)
197 - assert "product_id" in arguments
198 - assert "inserted" in arguments
256 + arguments = json.loads(tool_call.function.arguments)
257 + assert "product_id" in arguments
258 + assert "inserted" in arguments
259
200 - product_id = arguments.get("product_id")
201 - inserted = arguments.get("inserted")
260 + product_id = arguments.get("product_id")
261 + inserted = arguments.get("inserted")
262
203 - assert isinstance(product_id, int)
204 - assert product_id == 7355608
205 - assert isinstance(inserted, bool)
206 - assert inserted is True
263 + assert isinstance(product_id, int)
264 + assert product_id == 7355608
265 + assert isinstance(inserted, bool)
266 + assert inserted is True
267
208 - print("\n[Non-Streaming Product Test Passed]")
209 - print(f"Tool Call: {tool_call.function.name}")
210 - print(f"Arguments: {arguments}")
268 + print("\n[Non-Streaming Product Test Passed]")
269 + print(f"Tool Call: {tool_call.function.name}")
270 + print(f"Arguments: {arguments}")
271
272
273 @pytest.mark.asyncio
214 -async def test_streaming_product_tool_call():
274 +async def test_streaming_product_tool_call(
275 + client: openai.AsyncOpenAI, server_config: ServerConfig
276 +):
277 """Test tool call integer and boolean parameters in streaming mode."""
216 - with RemoteOpenAIServer(MODEL_NAME, SERVER_ARGS) as server:
217 - client = server.get_async_client()
218 -
219 - stream = await client.chat.completions.create(
220 - model=LORA_MODEL,
221 - messages=PRODUCT_MESSAGES,
222 - tools=PRODUCT_TOOLS,
223 - tool_choice="auto",
224 - temperature=0.66,
225 - stream=True,
226 - )
278
228 - tool_call_chunks = {}
229 - async for chunk in stream:
230 - if not chunk.choices:
231 - continue
279 + stream = await client.chat.completions.create(
280 + model=server_config["model_arg"],
281 + messages=PRODUCT_MESSAGES,
282 + tools=PRODUCT_TOOLS,
283 + tool_choice="auto",
284 + temperature=0.66,
285 + stream=True,
286 + )
287 +
288 + tool_call_chunks = {}
289 + async for chunk in stream:
290 + if not chunk.choices:
291 + continue
292
233 - delta = chunk.choices[0].delta
234 - if not delta or not delta.tool_calls:
235 - continue
293 + delta = chunk.choices[0].delta
294 + if not delta or not delta.tool_calls:
295 + continue
296
237 - for tool_chunk in delta.tool_calls:
238 - index = tool_chunk.index
239 - if index not in tool_call_chunks:
240 - tool_call_chunks[index] = {"name": "", "arguments": ""}
297 + for tool_chunk in delta.tool_calls:
298 + index = tool_chunk.index
299 + if index not in tool_call_chunks:
300 + tool_call_chunks[index] = {"name": "", "arguments": ""}
301
242 - if tool_chunk.function.name:
243 - tool_call_chunks[index]["name"] += tool_chunk.function.name
244 - if tool_chunk.function.arguments:
245 - tool_call_chunks[index]["arguments"] += (
246 - tool_chunk.function.arguments
247 - )
302 + if tool_chunk.function.name:
303 + tool_call_chunks[index]["name"] += tool_chunk.function.name
304 + if tool_chunk.function.arguments:
305 + tool_call_chunks[index]["arguments"] += tool_chunk.function.arguments
306
249 - assert len(tool_call_chunks) == 1
250 - reconstructed_tool_call = tool_call_chunks[0]
307 + assert len(tool_call_chunks) == 1
308 + reconstructed_tool_call = tool_call_chunks[0]
309
252 - assert reconstructed_tool_call["name"] == "get_product_info"
310 + assert reconstructed_tool_call["name"] == "get_product_info"
311
254 - arguments = json.loads(reconstructed_tool_call["arguments"])
255 - assert "product_id" in arguments
256 - assert "inserted" in arguments
312 + arguments = json.loads(reconstructed_tool_call["arguments"])
313 + assert "product_id" in arguments
314 + assert "inserted" in arguments
315
258 - # Handle type coercion for streaming test as well
259 - product_id = arguments.get("product_id")
260 - inserted = arguments.get("inserted")
316 + # Handle type coercion for streaming test as well
317 + product_id = arguments.get("product_id")
318 + inserted = arguments.get("inserted")
319
262 - assert isinstance(product_id, int)
263 - assert product_id == 7355608
264 - assert isinstance(inserted, bool)
265 - assert inserted is True
320 + assert isinstance(product_id, int)
321 + assert product_id == 7355608
322 + assert isinstance(inserted, bool)
323 + assert inserted is True
324
267 - print("\n[Streaming Product Test Passed]")
268 - print(f"Reconstructed Tool Call: {reconstructed_tool_call['name']}")
269 - print(f"Reconstructed Arguments: {arguments}")
325 + print("\n[Streaming Product Test Passed]")
326 + print(f"Reconstructed Tool Call: {reconstructed_tool_call['name']}")
327 + print(f"Reconstructed Arguments: {arguments}")
328
329
330 @pytest.fixture
@@ -276,9 +334,10 @@ def qwen_tokenizer() -> TokenizerLike:
334 return get_tokenizer("Qwen/Qwen3-32B")
335
336
279 -@pytest.fixture
280 -def hermes_parser(qwen_tokenizer: TokenizerLike) -> Hermes2ProToolParser:
281 - return Hermes2ProToolParser(qwen_tokenizer)
337 +@pytest.fixture(params=CONFIGS.keys())
338 +def hermes_parser(request, qwen_tokenizer: TokenizerLike) -> ToolParser:
339 + config = CONFIGS[request.param]
340 + return config["tool_parser"](qwen_tokenizer)
341
342
343 @pytest.fixture
@@ -292,7 +351,7 @@ def any_chat_request() -> ChatCompletionRequest:
351
352 def test_hermes_parser_streaming_just_forward_text(
353 qwen_tokenizer: TokenizerLike,
295 - hermes_parser: Hermes2ProToolParser,
354 + hermes_parser: ToolParser,
355 any_chat_request: ChatCompletionRequest,
356 ) -> None:
357 text = """This is some prior text that has nothing to do with tool calling."""
@@ -324,7 +383,7 @@ def test_hermes_parser_streaming_just_forward_text(
383
384 def test_hermes_parser_streaming_failure_case_bug_19056(
385 qwen_tokenizer: TokenizerLike,
327 - hermes_parser: Hermes2ProToolParser,
386 + hermes_parser: ToolParser,
387 any_chat_request: ChatCompletionRequest,
388 ) -> None:
389 text = """<tool_call>
@@ -358,7 +417,7 @@ def test_hermes_parser_streaming_failure_case_bug_19056(
417
418 def test_hermes_parser_streaming(
419 qwen_tokenizer: TokenizerLike,
361 - hermes_parser: Hermes2ProToolParser,
420 + hermes_parser: ToolParser,
421 any_chat_request: ChatCompletionRequest,
422 ) -> None:
423 text = '<tool_call>\
@@ -387,16 +446,20 @@ def test_hermes_parser_streaming(
446 delta_messages.append(delta)
447 print(delta_messages)
448 assert delta_messages[0].tool_calls[0].function.name == "get_current_temperature"
390 - tool_call_args = "".join(
391 - delta.tool_calls[0].function.arguments or "" for delta in delta_messages
392 - )
393 - assert tool_call_args == (
394 - '{"location":"San Francisco, California, United States", "unit": "celsius"}'
449 + # load to normalize whitespace
450 + tool_call_args = json.loads(
451 + "".join(
452 + delta.tool_calls[0].function.arguments or "" for delta in delta_messages
453 + )
454 )
455 + assert tool_call_args == {
456 + "location": "San Francisco, California, United States",
457 + "unit": "celsius",
458 + }
459
460
461 def test_hermes_parser_non_streaming_no_tool_call(
399 - hermes_parser: Hermes2ProToolParser,
462 + hermes_parser: ToolParser,
463 any_chat_request: ChatCompletionRequest,
464 ) -> None:
465 text = """This is not a tool call."""
@@ -410,7 +473,7 @@ def test_hermes_parser_non_streaming_no_tool_call(
473
474
475 def test_hermes_parser_non_streaming_tool_call_between_tags(
413 - hermes_parser: Hermes2ProToolParser,
476 + hermes_parser: ToolParser,
477 any_chat_request: ChatCompletionRequest,
478 ) -> None:
479 text = """<tool_call>
@@ -428,9 +491,12 @@ def test_hermes_parser_non_streaming_tool_call_between_tags(
491
492
493 def test_hermes_parser_non_streaming_tool_call_until_eos(
431 - hermes_parser: Hermes2ProToolParser,
494 + hermes_parser: ToolParser,
495 any_chat_request: ChatCompletionRequest,
496 ) -> None:
497 + if isinstance(hermes_parser, Granite4ToolParser):
498 + pytest.skip(reason="The Granite4 tool parser enforces a complete response")
499 +
500 text = """<tool_call>
501 {"name": "final_answer", "arguments": {"trigger": true}}"""
502 tool_call = hermes_parser.extract_tool_calls(
@@ -445,7 +511,7 @@ def test_hermes_parser_non_streaming_tool_call_until_eos(
511
512
513 def test_hermes_parser_non_streaming_tool_call_invalid_json(
448 - hermes_parser: Hermes2ProToolParser,
514 + hermes_parser: ToolParser,
515 any_chat_request: ChatCompletionRequest,
516 ) -> None:
517 # Missing closing brace to trigger exception
vllm/tool_parsers/__init__.py
+4
@@ -54,6 +54,10 @@ _TOOL_PARSERS_TO_REGISTER = {
54 "granite_tool_parser",
55 "GraniteToolParser",
56 ),
57 + "granite4": (
58 + "granite4_tool_parser",
59 + "Granite4ToolParser",
60 + ),
61 "hermes": (
62 "hermes_tool_parser",
63 "Hermes2ProToolParser",
vllm/tool_parsers/granite4_tool_parser.py new
+252
@@ -0,0 +1,252 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import json
5 +from collections.abc import Sequence
6 +from typing import Any, Protocol, TypeVar
7 +
8 +import regex as re
9 +
10 +from vllm.entrypoints.chat_utils import make_tool_call_id
11 +from vllm.entrypoints.openai.chat_completion.protocol import (
12 + ChatCompletionRequest,
13 +)
14 +from vllm.entrypoints.openai.engine.protocol import (
15 + DeltaFunctionCall,
16 + DeltaMessage,
17 + DeltaToolCall,
18 + ExtractedToolCallInformation,
19 + FunctionCall,
20 + ToolCall,
21 +)
22 +from vllm.logger import init_logger
23 +from vllm.tokenizers import TokenizerLike
24 +from vllm.tool_parsers.abstract_tool_parser import (
25 + ToolParser,
26 +)
27 +
28 +logger = init_logger(__name__)
29 +
30 +
31 +def dump_args(args: None | dict[str, Any] | str) -> str | None:
32 + if args is None or isinstance(args, str):
33 + return args
34 + else:
35 + return json.dumps(args, ensure_ascii=False)
36 +
37 +
38 +class _FunctionCallCtor(Protocol):
39 + def __init__(self, *, name: str, arguments: str | None): ...
40 +
41 +
42 +FuncT = TypeVar("FuncT", bound=_FunctionCallCtor)
43 +
44 +
45 +class Granite4ToolParser(ToolParser):
46 + def __init__(self, tokenizer: TokenizerLike):
47 + super().__init__(tokenizer)
48 +
49 + self.prev_tool_call_arr: list[dict] = []
50 + self.current_tool_id: int = -1
51 + self.streamed_args_for_tool = list[str]()
52 +
53 + self.look_ahead = ""
54 + self.in_tc = False
55 +
56 + self.tc_start = "<tool_call>"
57 + self.tc_end = "</tool_call>"
58 + self.start_regex = re.compile(self.tc_start)
59 + self.end_regex = re.compile(self.tc_end)
60 +
61 + def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest:
62 + request = super().adjust_request(request)
63 + if request.tools and request.tool_choice != "none":
64 + # do not skip special tokens because the tool_call tokens are
65 + # marked "special" in some models. Since they are skipped
66 + # prior to the call to the tool parser, it breaks tool calling.
67 + request.skip_special_tokens = False
68 + return request
69 +
70 + def _collect_results(
71 + self, text_segments: list[str], tc_segments: list[str], cls: type[FuncT]
72 + ) -> tuple[str, list[FuncT]]:
73 + tool_calls_json: list[dict[str, Any]] = [
74 + json.loads(tc_text) for tc_text in tc_segments
75 + ]
76 + tool_calls = []
77 + for tc in tool_calls_json:
78 + assert isinstance(tc, dict)
79 + self.prev_tool_call_arr.append(tc)
80 + tool_calls.append(
81 + cls(
82 + name=tc["name"],
83 + arguments=dump_args(tc["arguments"]),
84 + )
85 + )
86 + return "".join(text_segments), tool_calls
87 +
88 + def extract_tool_calls(
89 + self,
90 + model_output: str,
91 + request: ChatCompletionRequest,
92 + ) -> ExtractedToolCallInformation:
93 + msg = ExtractedToolCallInformation(
94 + tools_called=False, tool_calls=[], content=model_output
95 + )
96 + try:
97 + delimiters = [("TC_START", self.tc_start), ("TC_END", self.tc_end)]
98 + pattern = "|".join(f"(?P<{name}>{pattern})" for name, pattern in delimiters)
99 + regex = re.compile(pattern)
100 +
101 + text_segments = list[str]()
102 + tc_segments = list[str]()
103 + last_cut_loc = 0
104 +
105 + for match in regex.finditer(model_output):
106 + match_type = match.lastgroup
107 + if match_type == "TC_START":
108 + assert not self.in_tc, "Two tool call start tokens found in a row"
109 + if preceding_text := model_output[last_cut_loc : match.start()]:
110 + text_segments.append(preceding_text)
111 + self.in_tc = True
112 + elif match_type == "TC_END":
113 + assert self.in_tc, (
114 + "Tool call end token found without corresponding start token"
115 + )
116 + tool_text = model_output[last_cut_loc : match.start()]
117 + assert tool_text, (
118 + "Expected the model to generate text between tool call tokens"
119 + )
120 + tc_segments.append(tool_text)
121 + self.in_tc = False
122 + else:
123 + raise ValueError("Unexpected match")
124 + last_cut_loc = match.end()
125 + assert not self.in_tc, "The model generated an incomplete tool call"
126 + if final_text := model_output[last_cut_loc:]:
127 + text_segments.append(final_text)
128 +
129 + content, tool_call_funcs = self._collect_results(
130 + text_segments, tc_segments, FunctionCall
131 + )
132 + tool_calls = [
133 + ToolCall(
134 + type="function",
135 + function=func,
136 + )
137 + for func in tool_call_funcs
138 + ]
139 + msg.tools_called = bool(tool_calls)
140 + msg.tool_calls = tool_calls
141 + msg.content = content or None
142 + except Exception:
143 + logger.exception("Error in extracting tool call from response.")
144 + return msg
145 +
146 + def _tool_extraction_step(
147 + self,
148 + delta_text: str,
149 + ) -> tuple[bool, str, str]:
150 + start_token_pos = start_token_end = end_token_pos = end_token_end = -1
151 +
152 + if start_match := self.start_regex.search(delta_text, partial=True):
153 + if not start_match.partial:
154 + start_token_pos, start_token_end = start_match.span()
155 + elif start_match.end() > start_match.start():
156 + start_token_pos = -2
157 +
158 + if end_match := self.end_regex.search(delta_text):
159 + end_token_pos, end_token_end = end_match.span()
160 +
161 + # Done means that we've exhausted the current buffer
162 + # and need more output from the model
163 + done = True
164 + content = tc_text = ""
165 +
166 + if start_token_pos < 0:
167 + # just streaming text so far
168 + if start_token_pos == -2:
169 + # There is a partial match
170 + content = delta_text[: start_match.start()]
171 + self.look_ahead = delta_text[start_match.start() :]
172 + else:
173 + content = delta_text
174 +
175 + elif not self.in_tc:
176 + # we're entering a new tool call
177 + self.in_tc = True
178 +
179 + content = delta_text[:start_token_pos]
180 + if end_token_pos > 0:
181 + self.start_in_tc = False
182 + tc_text = delta_text[start_token_end:end_token_pos]
183 + self.look_ahead = delta_text[end_token_end:]
184 + done = False # There could be more content already buffered
185 + else:
186 + self.look_ahead = delta_text[start_token_pos:]
187 +
188 + elif end_token_pos < 0:
189 + # we're in between the start and the end token
190 + assert self.in_tc
191 + self.look_ahead = delta_text
192 + else:
193 + # We have found the end
194 + assert self.in_tc
195 + tc_text = delta_text[start_token_end:end_token_pos]
196 + self.in_tc = False
197 + self.look_ahead = delta_text[end_token_end:]
198 + done = False # There could be more content already buffered
199 + return done, content, tc_text
200 +
201 + def extract_tool_calls_streaming(
202 + self,
203 + previous_text: str,
204 + current_text: str,
205 + delta_text: str,
206 + previous_token_ids: Sequence[int],
207 + current_token_ids: Sequence[int],
208 + delta_token_ids: Sequence[int],
209 + request: ChatCompletionRequest,
210 + ) -> DeltaMessage | None:
211 + try:
212 + done = False
213 + text_segments = list[str]()
214 + tc_segments = list[str]()
215 +
216 + while not done:
217 + delta_text = self.look_ahead + delta_text
218 + self.look_ahead = ""
219 + done, content, tc_text = self._tool_extraction_step(delta_text)
220 + if content:
221 + text_segments.append(content)
222 + if tc_text:
223 + tc_segments.append(tc_text)
224 + delta_text = ""
225 +
226 + content, tool_call_funcs = self._collect_results(
227 + text_segments, tc_segments, DeltaFunctionCall
228 + )
229 +
230 + delta_tool_calls = list[DeltaToolCall]()
231 + for function in tool_call_funcs:
232 + self.current_tool_id += 1
233 + delta_tool_calls.append(
234 + DeltaToolCall(
235 + id=make_tool_call_id(),
236 + type="function",
237 + index=self.current_tool_id,
238 + function=function.model_dump(exclude_none=True),
239 + )
240 + )
241 + self.streamed_args_for_tool.append(function.arguments or "")
242 +
243 + assert self.current_tool_id + 1 == len(self.prev_tool_call_arr)
244 + assert self.current_tool_id + 1 == len(self.streamed_args_for_tool)
245 +
246 + msg = DeltaMessage(content=content or None, tool_calls=delta_tool_calls)
247 + if msg.content or msg.tool_calls:
248 + return msg
249 +
250 + except Exception:
251 + logger.exception("Error trying to handle streaming tool call.")
252 + return None