A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 22:15 UTC
[Bugfix] Fix Hermes tool parser when stream interval > 1 (#38168)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Flora Feng committed
Mar 27, 2026 at 02:42 UTC
aee4c14689c59e6e5f288fc8e53cf57dc170bcda
3 files changed
+349
-382
tests/tool_parsers/test_hermes_tool_parser.py
+194
@@ -152,6 +152,175 @@ def test_hermes_parser_streaming(
152
}
153
154
155
+def _simulate_streaming(
156
+ tokenizer: TokenizerLike,
157
+ parser: ToolParser,
158
+ request: ChatCompletionRequest,
159
+ text: str,
160
+ stream_interval: int = 1,
161
+) -> list:
162
+ """Simulate streaming with a given stream_interval.
163
+
164
+ Tokens are batched into chunks of `stream_interval` tokens,
165
+ mimicking how the output processor delivers them.
166
+ Returns a list of non-None DeltaMessages.
167
+ """
168
+ tokens = tokenizer.encode(text)
169
+ previous_text = ""
170
+ delta_messages = []
171
+ for i in range(0, len(tokens), stream_interval):
172
+ chunk_ids = tokens[i : i + stream_interval]
173
+ delta_text = tokenizer.decode(chunk_ids)
174
+ current_text = previous_text + delta_text
175
+ delta = parser.extract_tool_calls_streaming(
176
+ previous_text=previous_text,
177
+ current_text=current_text,
178
+ delta_text=delta_text,
179
+ previous_token_ids=[],
180
+ current_token_ids=[],
181
+ delta_token_ids=chunk_ids,
182
+ request=request,
183
+ )
184
+ previous_text = current_text
185
+ if delta is not None:
186
+ delta_messages.append(delta)
187
+ return delta_messages
188
+
189
+
190
+@pytest.mark.parametrize("stream_interval", [2, 3, 5, 8])
191
+def test_hermes_streaming_tool_call_with_stream_interval(
192
+ qwen_tokenizer: TokenizerLike,
193
+ any_chat_request: ChatCompletionRequest,
194
+ stream_interval: int,
195
+) -> None:
196
+ """Tool call streaming must produce correct name + args at any interval."""
197
+ text = (
198
+ '<tool_call>{"name": "get_current_temperature", '
199
+ '"arguments": {"location": "San Francisco", "unit": "celsius"}}'
200
+ "</tool_call>"
201
+ )
202
+ parser = Hermes2ProToolParser(qwen_tokenizer)
203
+ deltas = _simulate_streaming(
204
+ qwen_tokenizer, parser, any_chat_request, text, stream_interval
205
+ )
206
+
207
+ # Flatten all DeltaToolCalls across all deltas.
208
+ tool_deltas = [tc for d in deltas if d.tool_calls for tc in d.tool_calls]
209
+ assert tool_deltas, "Expected at least one tool call delta"
210
+ assert tool_deltas[0].function.name == "get_current_temperature"
211
+
212
+ # Concatenated arguments must be valid JSON matching the original.
213
+ args_str = "".join(tc.function.arguments or "" for tc in tool_deltas)
214
+ assert json.loads(args_str) == {
215
+ "location": "San Francisco",
216
+ "unit": "celsius",
217
+ }
218
+
219
+
220
+@pytest.mark.parametrize("stream_interval", [2, 3, 5, 8])
221
+def test_hermes_streaming_content_then_tool_call_with_stream_interval(
222
+ qwen_tokenizer: TokenizerLike,
223
+ any_chat_request: ChatCompletionRequest,
224
+ stream_interval: int,
225
+) -> None:
226
+ """Content before a tool call must be fully streamed, then tool call."""
227
+ text = (
228
+ "Sure, let me check the weather."
229
+ '<tool_call>{"name": "get_weather", '
230
+ '"arguments": {"city": "NYC"}}</tool_call>'
231
+ )
232
+ parser = Hermes2ProToolParser(qwen_tokenizer)
233
+ deltas = _simulate_streaming(
234
+ qwen_tokenizer, parser, any_chat_request, text, stream_interval
235
+ )
236
+
237
+ content_deltas = [d for d in deltas if d.content]
238
+ tool_deltas = [d for d in deltas if d.tool_calls]
239
+
240
+ # Content must reconstruct the prefix.
241
+ content_str = "".join(d.content for d in content_deltas)
242
+ assert content_str == "Sure, let me check the weather."
243
+
244
+ # Tool call must be correct.
245
+ tool_calls = [tc for d in tool_deltas for tc in d.tool_calls]
246
+ assert tool_calls[0].function.name == "get_weather"
247
+ args_str = "".join(tc.function.arguments or "" for tc in tool_calls)
248
+ assert json.loads(args_str) == {"city": "NYC"}
249
+
250
+
251
+@pytest.mark.parametrize("stream_interval", [1, 2, 4])
252
+def test_hermes_streaming_multiple_tool_calls_with_stream_interval(
253
+ qwen_tokenizer: TokenizerLike,
254
+ any_chat_request: ChatCompletionRequest,
255
+ stream_interval: int,
256
+) -> None:
257
+ """Multiple sequential tool calls must each be streamed correctly."""
258
+ text = (
259
+ '<tool_call>{"name": "search", "arguments": {"q": "cats"}}</tool_call>'
260
+ '<tool_call>{"name": "search", "arguments": {"q": "dogs"}}</tool_call>'
261
+ )
262
+ parser = Hermes2ProToolParser(qwen_tokenizer)
263
+ deltas = _simulate_streaming(
264
+ qwen_tokenizer, parser, any_chat_request, text, stream_interval
265
+ )
266
+
267
+ # Flatten all DeltaToolCalls across all deltas.
268
+ all_tool_calls = [tc for d in deltas if d.tool_calls for tc in d.tool_calls]
269
+
270
+ # Separate by tool index.
271
+ tool0 = [tc for tc in all_tool_calls if tc.index == 0]
272
+ tool1 = [tc for tc in all_tool_calls if tc.index == 1]
273
+
274
+ assert tool0[0].function.name == "search"
275
+ args0 = "".join(tc.function.arguments or "" for tc in tool0)
276
+ assert json.loads(args0) == {"q": "cats"}
277
+
278
+ assert tool1[0].function.name == "search"
279
+ args1 = "".join(tc.function.arguments or "" for tc in tool1)
280
+ assert json.loads(args1) == {"q": "dogs"}
281
+
282
+
283
+@pytest.mark.parametrize("stream_interval", [2, 5])
284
+def test_hermes_streaming_boolean_args_with_stream_interval(
285
+ qwen_tokenizer: TokenizerLike,
286
+ any_chat_request: ChatCompletionRequest,
287
+ stream_interval: int,
288
+) -> None:
289
+ """Regression test for bug #19056 with stream_interval > 1."""
290
+ text = (
291
+ "<tool_call>\n"
292
+ '{"name": "final_answer", "arguments": {"trigger": true}}\n'
293
+ "</tool_call>"
294
+ )
295
+ parser = Hermes2ProToolParser(qwen_tokenizer)
296
+ deltas = _simulate_streaming(
297
+ qwen_tokenizer, parser, any_chat_request, text, stream_interval
298
+ )
299
+
300
+ tool_calls = [tc for d in deltas if d.tool_calls for tc in d.tool_calls]
301
+ assert tool_calls[0].function.name == "final_answer"
302
+ args_str = "".join(tc.function.arguments or "" for tc in tool_calls)
303
+ assert json.loads(args_str) == {"trigger": True}
304
+
305
+
306
+@pytest.mark.parametrize("stream_interval", [2, 3, 5])
307
+def test_hermes_streaming_just_forward_text_with_stream_interval(
308
+ qwen_tokenizer: TokenizerLike,
309
+ any_chat_request: ChatCompletionRequest,
310
+ stream_interval: int,
311
+) -> None:
312
+ """Plain text with no tool calls must be fully forwarded."""
313
+ text = "This is plain text with no tool calling involved."
314
+ parser = Hermes2ProToolParser(qwen_tokenizer)
315
+ deltas = _simulate_streaming(
316
+ qwen_tokenizer, parser, any_chat_request, text, stream_interval
317
+ )
318
+
319
+ for d in deltas:
320
+ assert not d.tool_calls
321
+ assert "".join(d.content for d in deltas) == text
322
+
323
+
324
def test_hermes_parser_non_streaming_no_tool_call(
325
hermes_parser: ToolParser,
326
any_chat_request: ChatCompletionRequest,
@@ -218,3 +387,28 @@ def test_hermes_parser_non_streaming_tool_call_invalid_json(
387
388
assert tool_call is not None
389
assert not tool_call.tools_called
390
+
391
+
392
+def test_hermes_streaming_content_and_tool_call_in_single_chunk(
393
+ qwen_tokenizer: TokenizerLike,
394
+ any_chat_request: ChatCompletionRequest,
395
+) -> None:
396
+ """Content + complete tool call in one chunk must both be emitted."""
397
+ text = 'Hi!<tool_call>{"name": "f", "arguments": {"x": 1}}</tool_call>'
398
+ # Use a stream_interval large enough to guarantee a single chunk.
399
+ parser = Hermes2ProToolParser(qwen_tokenizer)
400
+ deltas = _simulate_streaming(
401
+ qwen_tokenizer,
402
+ parser,
403
+ any_chat_request,
404
+ text,
405
+ stream_interval=9999,
406
+ )
407
+
408
+ content_parts = [d.content for d in deltas if d.content]
409
+ tool_parts = [tc for d in deltas if d.tool_calls for tc in d.tool_calls]
410
+
411
+ assert "".join(content_parts) == "Hi!"
412
+ assert tool_parts[0].function.name == "f"
413
+ args_str = "".join(tc.function.arguments or "" for tc in tool_parts)
414
+ assert json.loads(args_str) == {"x": 1}
vllm/tool_parsers/hermes_tool_parser.py
+153
-364
@@ -4,9 +4,7 @@
4
import json
5
from collections.abc import Sequence
6
7
-import partial_json_parser
7
import regex as re
9
-from partial_json_parser.core.options import Allow
8
9
from vllm.entrypoints.chat_utils import make_tool_call_id
10
from vllm.entrypoints.openai.chat_completion.protocol import (
@@ -31,6 +29,27 @@ from vllm.utils.mistral import is_mistral_tokenizer
29
logger = init_logger(__name__)
30
31
32
+def _partial_tag_overlap(text: str, tag: str) -> int:
33
+ """Length of the longest prefix of `tag` that matches a suffix of `text`.
34
+
35
+ E.g. text ending in "<tool_" returns 6 when tag is "<tool_call>".
36
+ Returns 0 if there is no overlap.
37
+ """
38
+ max_check = min(len(tag) - 1, len(text))
39
+ for k in range(max_check, 0, -1):
40
+ if text.endswith(tag[:k]):
41
+ return k
42
+ return 0
43
+
44
+
45
+def _is_valid_json(text: str) -> bool:
46
+ try:
47
+ json.loads(text)
48
+ return True
49
+ except (json.JSONDecodeError, ValueError):
50
+ return False
51
+
52
+
53
class Hermes2ProToolParser(ToolParser):
54
def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
55
super().__init__(tokenizer, tools)
@@ -39,13 +58,6 @@ class Hermes2ProToolParser(ToolParser):
58
logger.error("Detected Mistral tokenizer when using a Hermes model")
59
self.model_tokenizer = tokenizer.tokenizer
60
42
- self.current_tool_name_sent: bool = False
43
- self.prev_tool_call_arr: list[dict] = []
44
- self.current_tool_id: int = -1
45
- self.streamed_args_for_tool: list[
46
- str
47
- ] = [] # map what has been streamed for each tool so far to a list
48
-
61
self.tool_call_start_token: str = "<tool_call>"
62
self.tool_call_end_token: str = "</tool_call>"
63
@@ -61,57 +73,9 @@ class Hermes2ProToolParser(ToolParser):
73
"The model tokenizer must be passed to the ToolParser "
74
"constructor during construction."
75
)
64
- self.tool_call_start_token_ids = self.model_tokenizer.encode(
65
- self.tool_call_start_token, add_special_tokens=False
66
- )
67
- self.tool_call_end_token_ids = self.model_tokenizer.encode(
68
- self.tool_call_end_token, add_special_tokens=False
69
- )
76
71
- self.tool_call_start_token_array = [
72
- self.model_tokenizer.decode([token_id])
73
- for token_id in self.tool_call_start_token_ids
74
- ]
75
-
76
- self.tool_call_end_token_array = [
77
- self.model_tokenizer.decode([token_id])
78
- for token_id in self.tool_call_end_token_ids
79
- ]
80
-
81
- self.buffered_delta_text = ""
82
-
83
- # Very simple idea: when encountering tokens like <, tool, _call, >,
84
- # <, /, tool, _call, >, store them in a buffer.
85
- # When the last token is encountered, empty the buffer and return it.
86
- # If a token appears in an incorrect sequence while storing in the buffer,
87
- # return the preceding buffer along with the token.
88
- def tool_call_delta_buffer(self, delta_text: str):
89
- # If the sequence of tool_call_start or tool_call_end tokens is not yet
90
- # complete, fill the buffer with the token and return "".
91
- if (
92
- delta_text in self.tool_call_start_token_array
93
- or delta_text in self.tool_call_end_token_array
94
- ):
95
- # If delta_text is the last token of tool_call_start_token or
96
- # tool_call_end_token, empty the buffer and return
97
- # the buffered text + delta_text.
98
- if (
99
- delta_text == self.tool_call_start_token_array[-1]
100
- or delta_text == self.tool_call_end_token_array[-1]
101
- ):
102
- buffered_text = self.buffered_delta_text
103
- self.buffered_delta_text = ""
104
- return buffered_text + delta_text
105
- else:
106
- self.buffered_delta_text = self.buffered_delta_text + delta_text
107
- return ""
108
- else:
109
- if self.buffered_delta_text:
110
- buffered_text = self.buffered_delta_text
111
- self.buffered_delta_text = ""
112
- return buffered_text + delta_text
113
- else:
114
- return delta_text
77
+ # Streaming state: what has been sent to the client.
78
+ self._sent_content_idx: int = 0
79
80
def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest:
81
request = super().adjust_request(request)
@@ -174,6 +138,88 @@ class Hermes2ProToolParser(ToolParser):
138
tools_called=False, tool_calls=[], content=model_output
139
)
140
141
+ def _extract_content(self, current_text: str) -> str | None:
142
+ """Return unsent non-tool-call text, or None.
143
+
144
+ Holds back any suffix that could be a partial <tool_call> tag.
145
+ """
146
+ if self.tool_call_start_token not in current_text:
147
+ overlap_length = _partial_tag_overlap(
148
+ current_text, self.tool_call_start_token
149
+ )
150
+ sendable_idx = len(current_text) - overlap_length
151
+ else:
152
+ sendable_idx = current_text.index(self.tool_call_start_token)
153
+
154
+ if sendable_idx > self._sent_content_idx:
155
+ content = current_text[self._sent_content_idx : sendable_idx]
156
+ self._sent_content_idx = sendable_idx
157
+ return content
158
+ return None
159
+
160
+ def _extract_tool_call_jsons(self, text: str) -> list[tuple[str, bool]]:
161
+ """Extract (json_text, is_complete) for each <tool_call> region."""
162
+ results: list[tuple[str, bool]] = []
163
+ pos = 0
164
+ while True:
165
+ start = text.find(self.tool_call_start_token, pos)
166
+ if start == -1:
167
+ break
168
+ json_start = start + len(self.tool_call_start_token)
169
+ json_end = text.find(self.tool_call_end_token, json_start)
170
+ if json_end != -1:
171
+ results.append((text[json_start:json_end].strip(), True))
172
+ pos = json_end + len(self.tool_call_end_token)
173
+ else:
174
+ raw = text[json_start:]
175
+ # Strip partial </tool_call> suffix if present.
176
+ overlap = _partial_tag_overlap(raw, self.tool_call_end_token)
177
+ if overlap:
178
+ raw = raw[:-overlap]
179
+ tc_json = raw.strip()
180
+ # Valid JSON without closing tag = complete body,
181
+ # tag tokens just haven't arrived yet.
182
+ is_complete = _is_valid_json(tc_json) if tc_json else False
183
+ results.append((tc_json, is_complete))
184
+ break
185
+ return results
186
+
187
+ @staticmethod
188
+ def _extract_tool_name(tc_json: str) -> str | None:
189
+ """Extract tool name, or None if the name isn't complete yet."""
190
+ match = re.search(r'"name"\s*:\s*"([^"]+)"', tc_json)
191
+ return match.group(1) if match else None
192
+
193
+ @staticmethod
194
+ def _extract_tool_args(tc_json: str, is_complete: bool) -> str | None:
195
+ """Extract tool arguments from the tool call JSON.
196
+
197
+ Given {"name": "f", "arguments": {"x": 1}}, returns '{"x": 1}'.
198
+ When is_complete, strips the trailing '}' that closes the outer
199
+ object (not the arguments). For partial JSON, returns as-is.
200
+ """
201
+ match = re.search(r'"arguments"\s*:\s*', tc_json)
202
+ if not match:
203
+ return None
204
+ raw = tc_json[match.end() :]
205
+ if is_complete:
206
+ raw = raw.rstrip()
207
+ if raw.endswith("}"):
208
+ raw = raw[:-1].rstrip()
209
+ return raw
210
+
211
+ def _compute_args_diff(
212
+ self, index: int, tc_json: str, is_complete: bool
213
+ ) -> str | None:
214
+ """Return new argument text not yet sent for tool `index`, or None."""
215
+ args = self._extract_tool_args(tc_json, is_complete)
216
+ if args is None or len(args) <= len(self.streamed_args_for_tool[index]):
217
+ return None
218
+ diff = args[len(self.streamed_args_for_tool[index]) :]
219
+ self.streamed_args_for_tool[index] = args
220
+ self.prev_tool_call_arr[index]["arguments"] = args
221
+ return diff
222
+
223
def extract_tool_calls_streaming(
224
self,
225
previous_text: str,
@@ -184,321 +230,64 @@ class Hermes2ProToolParser(ToolParser):
230
delta_token_ids: Sequence[int],
231
request: ChatCompletionRequest,
232
) -> DeltaMessage | None:
187
- # 1. All tokens are parsed based on _text, not token_ids.
188
- # 2. All incoming text data is processed by the tool_call_delta_buffer
189
- # function for buffering before being used for parsing.
190
-
191
- delta_text = self.tool_call_delta_buffer(delta_text)
192
- # If the last characters of previous_text
193
- # match self.buffered_delta_text, remove only the matching part.
194
- if (
195
- len(previous_text) >= len(self.buffered_delta_text)
196
- and previous_text[-len(self.buffered_delta_text) :]
197
- == self.buffered_delta_text
198
- ):
199
- previous_text = previous_text[: -len(self.buffered_delta_text)]
200
- current_text = previous_text + delta_text
201
-
202
- logger.debug("delta_text: %s", delta_text)
203
- logger.debug("delta_token_ids: %s", delta_token_ids)
204
- # check to see if we should be streaming a tool call - is there a
205
- if self.tool_call_start_token not in current_text:
206
- logger.debug("No tool call tokens found!")
207
- return DeltaMessage(content=delta_text)
233
+ """Incrementally stream tool call deltas from accumulated output.
234
209
- try:
210
- # figure out where we are in the parsing by counting tool call
211
- # start & end tags
212
- prev_tool_start_count = previous_text.count(self.tool_call_start_token)
213
- prev_tool_end_count = previous_text.count(self.tool_call_end_token)
214
- cur_tool_start_count = current_text.count(self.tool_call_start_token)
215
- cur_tool_end_count = current_text.count(self.tool_call_end_token)
216
- tool_call_portion = None
217
- text_portion = None
218
-
219
- # case: if we're generating text, OR rounding out a tool call
220
- if (
221
- cur_tool_start_count == cur_tool_end_count
222
- and prev_tool_end_count == cur_tool_end_count
223
- and self.tool_call_end_token not in delta_text
224
- ):
225
- logger.debug("Generating text content! skipping tool parsing.")
226
- return DeltaMessage(content=delta_text)
227
-
228
- if self.tool_call_end_token in delta_text:
229
- logger.debug("tool_call_end_token in delta_text")
230
- full_text = current_text + delta_text
231
- tool_call_portion = (
232
- full_text.split(self.tool_call_start_token)[-1]
233
- .split(self.tool_call_end_token)[0]
234
- .rstrip()
235
- )
236
- delta_text = delta_text.split(self.tool_call_end_token)[0].rstrip()
237
- text_portion = delta_text.split(self.tool_call_end_token)[-1].lstrip()
238
-
239
- # case: if tool open & close tag counts don't match, we're doing
240
- # imaginary "else" block here
241
- # something with tools with this diff.
242
- # flags for partial JSON parting. exported constants from
243
- # "Allow" are handled via BIT MASK
244
- flags = Allow.ALL if self.current_tool_name_sent else Allow.ALL & ~Allow.STR
245
-
246
- # case -- we're starting a new tool call
247
- if (
248
- cur_tool_start_count > cur_tool_end_count
249
- and cur_tool_start_count > prev_tool_start_count
250
- ):
251
- if len(delta_token_ids) > 1:
252
- tool_call_portion = current_text.split(self.tool_call_start_token)[
253
- -1
254
- ]
255
- else:
256
- tool_call_portion = None
257
- delta = None
258
-
259
- text_portion = None
260
-
261
- # set cursors and state appropriately
262
- self.current_tool_id += 1
263
- self.current_tool_name_sent = False
264
- self.streamed_args_for_tool.append("")
265
- logger.debug("Starting on a new tool %s", self.current_tool_id)
266
-
267
- # case -- we're updating an existing tool call
268
- elif (
269
- cur_tool_start_count > cur_tool_end_count
270
- and cur_tool_start_count == prev_tool_start_count
271
- ):
272
- # get the portion of the text that's the tool call
273
- tool_call_portion = current_text.split(self.tool_call_start_token)[-1]
274
- text_portion = None
275
-
276
- # case -- the current tool call is being closed.
277
- elif (
278
- cur_tool_start_count == cur_tool_end_count
279
- and cur_tool_end_count >= prev_tool_end_count
280
- ):
281
- if self.prev_tool_call_arr is None or len(self.prev_tool_call_arr) == 0:
282
- logger.debug("attempting to close tool call, but no tool call")
283
- return None
284
- diff = self.prev_tool_call_arr[self.current_tool_id].get("arguments")
285
- if diff:
286
- diff = (
287
- diff.encode("utf-8").decode("unicode_escape")
288
- if diff is str
289
- else diff
290
- )
291
- if '"}' not in delta_text:
292
- return None
293
- end_loc = delta_text.rindex('"}')
294
- diff = delta_text[:end_loc] + '"}'
295
- logger.debug(
296
- "Finishing tool and found diff that had not "
297
- "been streamed yet: %s",
298
- diff,
299
- )
300
- self.streamed_args_for_tool[self.current_tool_id] += diff
301
- return DeltaMessage(
302
- tool_calls=[
303
- DeltaToolCall(
304
- index=self.current_tool_id,
305
- function=DeltaFunctionCall(arguments=diff).model_dump(
306
- exclude_none=True
307
- ),
308
- )
309
- ]
310
- )
235
+ On each invocation, re-parses the full ``current_text`` to find
236
+ ``<tool_call>`` regions, then diffs against previously sent state
237
+ to emit only new content, tool names, or argument fragments.
238
312
- # case -- otherwise we're just generating text
313
- else:
314
- text = delta_text.replace(self.tool_call_start_token, "")
315
- text = text.replace(self.tool_call_end_token, "")
316
- delta = DeltaMessage(tool_calls=[], content=text)
317
- return delta
318
-
319
- try:
320
- current_tool_call = (
321
- partial_json_parser.loads(tool_call_portion or "{}", flags)
322
- if tool_call_portion
323
- else None
324
- )
325
- logger.debug("Parsed tool call %s", current_tool_call)
326
- except partial_json_parser.core.exceptions.MalformedJSON:
327
- logger.debug("not enough tokens to parse into JSON yet")
328
- return None
329
- except json.decoder.JSONDecodeError:
330
- logger.debug("unable to parse JSON")
331
- return None
332
-
333
- if current_tool_call is None:
334
- return None
335
-
336
- # case - we haven't sent the tool name yet. If it's available, send
337
- # it. otherwise, wait until it's available.
338
- if not self.current_tool_name_sent:
339
- function_name: str | None = current_tool_call.get("name")
340
- if function_name:
341
- self.current_tool_name_sent = True
342
- return DeltaMessage(
343
- tool_calls=[
344
- DeltaToolCall(
345
- index=self.current_tool_id,
346
- type="function",
347
- id=make_tool_call_id(),
348
- function=DeltaFunctionCall(
349
- name=function_name
350
- ).model_dump(exclude_none=True),
351
- )
352
- ]
353
- )
354
- else:
355
- return None
356
- # case -- otherwise, send the tool call delta
357
-
358
- # if the tool call portion is None, send the delta as text
359
- if tool_call_portion is None:
360
- # if there's text but not tool calls, send that -
361
- # otherwise None to skip chunk
362
- delta = (
363
- DeltaMessage(content=delta_text)
364
- if text_portion is not None
365
- else None
366
- )
367
- return delta
368
-
369
- # now, the nitty-gritty of tool calls
370
- # now we have the portion to parse as tool call.
371
-
372
- if current_tool_call is None:
373
- return None
374
-
375
- logger.debug(
376
- "Trying to parse current tool call with ID %s", self.current_tool_id
377
- )
378
-
379
- # if we're starting a new tool call, push an empty object in as
380
- # a placeholder for the arguments
381
- if len(self.prev_tool_call_arr) <= self.current_tool_id:
382
- self.prev_tool_call_arr.append({})
383
-
384
- # main logic for tool parsing here - compare prev. partially-parsed
385
- # JSON to the current partially-parsed JSON
386
- prev_arguments = self.prev_tool_call_arr[self.current_tool_id].get(
387
- "arguments"
388
- )
389
- assert current_tool_call is not None
390
- cur_arguments = current_tool_call.get("arguments")
391
-
392
- logger.debug("diffing old arguments: %s", prev_arguments)
393
- logger.debug("against new ones: %s", cur_arguments)
394
-
395
- # case -- no arguments have been created yet. skip sending a delta.
396
- if not cur_arguments and not prev_arguments:
397
- logger.debug("Skipping text %s - no arguments", delta_text)
398
- delta = None
399
-
400
- # case -- prev arguments are defined, but non are now.
401
- # probably impossible, but not a fatal error - just keep going
402
- elif not cur_arguments and prev_arguments:
403
- logger.error(
404
- "should be impossible to have arguments reset "
405
- "mid-call. skipping streaming anything."
406
- )
407
- delta = None
408
-
409
- # case -- we now have the first info about arguments available from
410
- # autocompleting the JSON
411
- elif cur_arguments and not prev_arguments:
412
- # extract the content after {"name": ..., "arguments":
413
- # directly from tool_call_portion as cur_arguments_json,
414
- # since cur_arguments may differ from the original text
415
- # due to partial JSON parsing
416
- # for example, tool_call_portion =
417
- # {"name": "search", "arguments": {"search_request": {"
418
- # but cur_arguments =
419
- # {"search_request": {}}
420
- function_name = current_tool_call.get("name")
421
- match = re.search(
422
- r'\{"name":\s*"'
423
- + re.escape(function_name)
424
- + r'"\s*,\s*"arguments":\s*(.*)',
425
- tool_call_portion.strip(),
426
- re.DOTALL,
427
- )
428
- if match:
429
- cur_arguments_json = match.group(1)
430
- else:
431
- cur_arguments_json = json.dumps(cur_arguments, ensure_ascii=False)
432
-
433
- logger.debug("finding %s in %s", delta_text, cur_arguments_json)
434
-
435
- # get the location where previous args differ from current.
436
- if delta_text not in cur_arguments_json:
437
- return None
438
- args_delta_start_loc = cur_arguments_json.rindex(delta_text) + len(
439
- delta_text
440
- )
441
-
442
- # use that to find the actual delta
443
- arguments_delta = cur_arguments_json[:args_delta_start_loc]
444
- logger.debug("First tokens in arguments received: %s", arguments_delta)
445
-
446
- delta = DeltaMessage(
447
- tool_calls=[
239
+ Returns a ``DeltaMessage`` containing either plain content (for
240
+ text preceding any tool call) or one or more ``DeltaToolCall``
241
+ entries, or ``None`` if there is nothing new to send yet."""
242
+ try:
243
+ # Extract any content before tool calls.
244
+ content = self._extract_content(current_text)
245
+ tool_call_jsons = self._extract_tool_call_jsons(current_text)
246
+ tool_call_deltas: list[DeltaToolCall] = []
247
+
248
+ for i, (tc_json, is_complete) in enumerate(tool_call_jsons):
249
+ if i >= len(self.prev_tool_call_arr):
250
+ self.prev_tool_call_arr.append({})
251
+ self.streamed_args_for_tool.append("")
252
+
253
+ # Stream back tool name.
254
+ if "name" not in self.prev_tool_call_arr[i]:
255
+ name = self._extract_tool_name(tc_json)
256
+ if not name:
257
+ # Can't skip to tool i+1 if i isn't ready
258
+ break
259
+ self.prev_tool_call_arr[i]["name"] = name
260
+ tool_call_deltas.append(
261
DeltaToolCall(
449
- index=self.current_tool_id,
450
- function=DeltaFunctionCall(
451
- arguments=arguments_delta
452
- ).model_dump(exclude_none=True),
262
+ index=i,
263
+ type="function",
264
+ id=make_tool_call_id(),
265
+ function=DeltaFunctionCall(name=name).model_dump(
266
+ exclude_none=True
267
+ ),
268
)
454
- ]
455
- )
456
- self.streamed_args_for_tool[self.current_tool_id] += arguments_delta
457
-
458
- # last case -- we have an update to existing arguments.
459
- elif cur_arguments and prev_arguments:
460
- # judge whether the tool_call_portion is a complete JSON
461
- try:
462
- json.loads(tool_call_portion)
463
- is_complete_json = True
464
- except Exception:
465
- is_complete_json = False
466
-
467
- # if the delta_text ends with a '}' and tool_call_portion is a
468
- # complete JSON, then the last '}' does not belong to the
469
- # arguments, so we should trim it off
470
- if (
471
- isinstance(delta_text, str)
472
- and len(delta_text.rstrip()) >= 1
473
- and delta_text.rstrip()[-1] == "}"
474
- and is_complete_json
475
- ):
476
- delta_text = delta_text.rstrip()[:-1]
477
-
478
- logger.debug("got diff %s", delta_text)
479
-
480
- delta = DeltaMessage(
481
- tool_calls=[
269
+ )
270
+
271
+ # Stream back new tool args by diffing against what was sent.
272
+ args_diff = self._compute_args_diff(i, tc_json, is_complete)
273
+ if args_diff:
274
+ tool_call_deltas.append(
275
DeltaToolCall(
483
- index=self.current_tool_id,
484
- function=DeltaFunctionCall(arguments=delta_text).model_dump(
276
+ index=i,
277
+ function=DeltaFunctionCall(arguments=args_diff).model_dump(
278
exclude_none=True
279
),
280
)
488
- ]
489
- )
490
- self.streamed_args_for_tool[self.current_tool_id] += delta_text
281
+ )
282
492
- # handle saving the state for the current tool into
493
- # the "prev" list for use in diffing for the next iteration
494
- assert isinstance(current_tool_call, dict)
495
- if self.current_tool_id == len(self.prev_tool_call_arr) - 1:
496
- self.prev_tool_call_arr[self.current_tool_id] = current_tool_call
497
- else:
498
- self.prev_tool_call_arr.append(current_tool_call)
283
+ if content or tool_call_deltas:
284
+ return DeltaMessage(
285
+ content=content,
286
+ tool_calls=tool_call_deltas,
287
+ )
288
500
- return delta
289
+ return None
290
291
except Exception:
292
logger.exception("Error trying to handle streaming tool call.")
504
- return None # do not stream a delta. skip this token ID.
293
+ return None
vllm/tool_parsers/longcat_tool_parser.py
+2
-18
@@ -16,23 +16,7 @@ class LongcatFlashToolParser(Hermes2ProToolParser):
16
self.tool_call_end_token: str = "</longcat_tool_call>"
17
18
self.tool_call_regex = re.compile(
19
- r"<longcat_tool_call>(.*?)</longcat_tool_call>|<longcat_tool_call>(.*)",
19
+ r"<longcat_tool_call>(.*?)</longcat_tool_call>"
20
+ r"|<longcat_tool_call>(.*)",
21
re.DOTALL,
22
)
22
-
23
- self.tool_call_start_token_ids = self.model_tokenizer.encode(
24
- self.tool_call_start_token, add_special_tokens=False
25
- )
26
- self.tool_call_end_token_ids = self.model_tokenizer.encode(
27
- self.tool_call_end_token, add_special_tokens=False
28
- )
29
-
30
- self.tool_call_start_token_array = [
31
- self.model_tokenizer.decode([token_id])
32
- for token_id in self.tool_call_start_token_ids
33
- ]
34
-
35
- self.tool_call_end_token_array = [
36
- self.model_tokenizer.decode([token_id])
37
- for token_id in self.tool_call_end_token_ids
38
- ]