@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 50f2db2555
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 4, 03:45 UTC

add: LFM2/2.5 Tool Parser (#39243)

Signed-off-by: Jonathan Buchanan <jonathan.buchanan@liquid.ai> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>

Jonathan committed May 7, 2026 at 18:58 UTC 50f2db25554a3a4b9aa4151ec8f6bc0b65bc878d
4 files changed +864 -5
tests/tool_parsers/test_lfm2_tool_parser.py new
+468
@@ -0,0 +1,468 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from unittest.mock import MagicMock, patch
5 +
6 +import pytest
7 +from transformers import AutoTokenizer
8 +
9 +from tests.tool_parsers.utils import (
10 + run_tool_extraction,
11 + run_tool_extraction_streaming,
12 +)
13 +from vllm.entrypoints.openai.engine.protocol import FunctionCall
14 +from vllm.tokenizers import TokenizerLike
15 +from vllm.tool_parsers import ToolParser, ToolParserManager
16 +
17 +TOOL_CALL_START = "<|tool_call_start|>"
18 +TOOL_CALL_END = "<|tool_call_end|>"
19 +
20 +SIMPLE_FUNCTION_OUTPUT = "get_candidate_status(candidate_id='12345')"
21 +SIMPLE_FUNCTION_CALL = FunctionCall(
22 + name="get_candidate_status",
23 + arguments='{"candidate_id": "12345"}',
24 +)
25 +MORE_TYPES_FUNCTION_OUTPUT = (
26 + "register_user(name='John Doe', "
27 + "age=37, "
28 + "address={'city': 'San Francisco', 'state': 'CA'}, "
29 + "role=None, "
30 + "passed_test=True, "
31 + "aliases=['John', 'Johnny'])"
32 +)
33 +MORE_TYPES_FUNCTION_CALL = FunctionCall(
34 + name="register_user",
35 + arguments='{"name": "John Doe", '
36 + '"age": 37, '
37 + '"address": {"city": "San Francisco", "state": "CA"}, '
38 + '"role": null, '
39 + '"passed_test": true, '
40 + '"aliases": ["John", "Johnny"]}',
41 +)
42 +PARAMETERLESS_FUNCTION_OUTPUT = "get_weather()"
43 +PARAMETERLESS_FUNCTION_CALL = FunctionCall(
44 + name="get_weather",
45 + arguments="{}",
46 +)
47 +EMPTY_DICT_FUNCTION_OUTPUT = "do_something_cool(additional_data={})"
48 +EMPTY_DICT_FUNCTION_CALL = FunctionCall(
49 + name="do_something_cool",
50 + arguments='{"additional_data": {}}',
51 +)
52 +EMPTY_LIST_FUNCTION_OUTPUT = "do_something_cool(steps=[])"
53 +EMPTY_LIST_FUNCTION_CALL = FunctionCall(
54 + name="do_something_cool",
55 + arguments='{"steps": []}',
56 +)
57 +ESCAPED_STRING_FUNCTION_OUTPUT = (
58 + r"get_weather(city='Martha\'s Vineyard', metric='\"cool units\"')"
59 +)
60 +ESCAPED_STRING_FUNCTION_CALL = FunctionCall(
61 + name="get_weather",
62 + arguments='{"city": "Martha\'s Vineyard", "metric": "\\"cool units\\""}',
63 +)
64 +DOTTED_NAME_FUNCTION_OUTPUT = (
65 + "grocery.orderIngredients("
66 + "ingredientList=[{'name': 'Lasagna noodles', 'amount': 250, 'unit': 'g'}], "
67 + "deliveryAddress='845 Willow Lane, Springfield, IL 62704')"
68 +)
69 +DOTTED_NAME_FUNCTION_CALL = FunctionCall(
70 + name="grocery.orderIngredients",
71 + arguments=(
72 + '{"ingredientList": ['
73 + '{"name": "Lasagna noodles", "amount": 250, "unit": "g"}], '
74 + '"deliveryAddress": "845 Willow Lane, Springfield, IL 62704"}'
75 + ),
76 +)
77 +
78 +
79 +@pytest.fixture(scope="module")
80 +def lfm2_tokenizer() -> TokenizerLike:
81 + return AutoTokenizer.from_pretrained("LiquidAI/LFM2.5-1.2B-Instruct")
82 +
83 +
84 +def _wrap(tool_text: str, content_after: str = "") -> str:
85 + """Wrap pythonic tool call in LFM2.5 sentinel tokens."""
86 + result = f"{TOOL_CALL_START}[{tool_text}]{TOOL_CALL_END}"
87 + if content_after:
88 + result += f"\n{content_after}"
89 + return result
90 +
91 +
92 +@pytest.mark.parametrize("streaming", [True, False])
93 +def test_no_tool_call(streaming: bool, lfm2_tokenizer: TokenizerLike):
94 + tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
95 + model_output = "How can I help you today?"
96 +
97 + content, tool_calls = run_tool_extraction(
98 + tool_parser, model_output, streaming=streaming
99 + )
100 +
101 + assert content == model_output
102 + assert len(tool_calls) == 0
103 +
104 +
105 +TEST_CASES = [
106 + pytest.param(
107 + True,
108 + _wrap(SIMPLE_FUNCTION_OUTPUT),
109 + [SIMPLE_FUNCTION_CALL],
110 + None,
111 + id="simple_streaming",
112 + ),
113 + pytest.param(
114 + False,
115 + _wrap(SIMPLE_FUNCTION_OUTPUT),
116 + [SIMPLE_FUNCTION_CALL],
117 + None,
118 + id="simple_nonstreaming",
119 + ),
120 + pytest.param(
121 + True,
122 + _wrap(MORE_TYPES_FUNCTION_OUTPUT),
123 + [MORE_TYPES_FUNCTION_CALL],
124 + None,
125 + id="more_types_streaming",
126 + ),
127 + pytest.param(
128 + False,
129 + _wrap(MORE_TYPES_FUNCTION_OUTPUT),
130 + [MORE_TYPES_FUNCTION_CALL],
131 + None,
132 + id="more_types_nonstreaming",
133 + ),
134 + pytest.param(
135 + True,
136 + _wrap(PARAMETERLESS_FUNCTION_OUTPUT),
137 + [PARAMETERLESS_FUNCTION_CALL],
138 + None,
139 + id="parameterless_streaming",
140 + ),
141 + pytest.param(
142 + False,
143 + _wrap(PARAMETERLESS_FUNCTION_OUTPUT),
144 + [PARAMETERLESS_FUNCTION_CALL],
145 + None,
146 + id="parameterless_nonstreaming",
147 + ),
148 + pytest.param(
149 + True,
150 + _wrap(EMPTY_DICT_FUNCTION_OUTPUT),
151 + [EMPTY_DICT_FUNCTION_CALL],
152 + None,
153 + id="empty_dict_streaming",
154 + ),
155 + pytest.param(
156 + False,
157 + _wrap(EMPTY_DICT_FUNCTION_OUTPUT),
158 + [EMPTY_DICT_FUNCTION_CALL],
159 + None,
160 + id="empty_dict_nonstreaming",
161 + ),
162 + pytest.param(
163 + True,
164 + _wrap(EMPTY_LIST_FUNCTION_OUTPUT),
165 + [EMPTY_LIST_FUNCTION_CALL],
166 + None,
167 + id="empty_list_streaming",
168 + ),
169 + pytest.param(
170 + False,
171 + _wrap(EMPTY_LIST_FUNCTION_OUTPUT),
172 + [EMPTY_LIST_FUNCTION_CALL],
173 + None,
174 + id="empty_list_nonstreaming",
175 + ),
176 + pytest.param(
177 + True,
178 + _wrap(ESCAPED_STRING_FUNCTION_OUTPUT),
179 + [ESCAPED_STRING_FUNCTION_CALL],
180 + None,
181 + id="escaped_string_streaming",
182 + ),
183 + pytest.param(
184 + False,
185 + _wrap(ESCAPED_STRING_FUNCTION_OUTPUT),
186 + [ESCAPED_STRING_FUNCTION_CALL],
187 + None,
188 + id="escaped_string_nonstreaming",
189 + ),
190 + pytest.param(
191 + True,
192 + _wrap(f"{SIMPLE_FUNCTION_OUTPUT}, {MORE_TYPES_FUNCTION_OUTPUT}"),
193 + [SIMPLE_FUNCTION_CALL, MORE_TYPES_FUNCTION_CALL],
194 + None,
195 + id="parallel_calls_streaming",
196 + ),
197 + pytest.param(
198 + False,
199 + _wrap(f"{SIMPLE_FUNCTION_OUTPUT}, {MORE_TYPES_FUNCTION_OUTPUT}"),
200 + [SIMPLE_FUNCTION_CALL, MORE_TYPES_FUNCTION_CALL],
201 + None,
202 + id="parallel_calls_nonstreaming",
203 + ),
204 + # LFM2.5 specific: content AFTER tool call
205 + pytest.param(
206 + False,
207 + _wrap(
208 + SIMPLE_FUNCTION_OUTPUT,
209 + content_after="Checking the current status of candidate ID 12345.",
210 + ),
211 + [SIMPLE_FUNCTION_CALL],
212 + "Checking the current status of candidate ID 12345.",
213 + id="content_after_tool_call_nonstreaming",
214 + ),
215 + # Dotted / class-method function names: grocery.orderIngredients(...)
216 + pytest.param(
217 + True,
218 + _wrap(DOTTED_NAME_FUNCTION_OUTPUT),
219 + [DOTTED_NAME_FUNCTION_CALL],
220 + None,
221 + id="dotted_name_streaming",
222 + ),
223 + pytest.param(
224 + False,
225 + _wrap(DOTTED_NAME_FUNCTION_OUTPUT),
226 + [DOTTED_NAME_FUNCTION_CALL],
227 + None,
228 + id="dotted_name_nonstreaming",
229 + ),
230 +]
231 +
232 +
233 +@pytest.mark.parametrize(
234 + "streaming, model_output, expected_tool_calls, expected_content",
235 + TEST_CASES,
236 +)
237 +def test_tool_call(
238 + streaming: bool,
239 + model_output: str,
240 + expected_tool_calls: list[FunctionCall],
241 + expected_content: str | None,
242 + lfm2_tokenizer: TokenizerLike,
243 +):
244 + tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
245 +
246 + content, tool_calls = run_tool_extraction(
247 + tool_parser, model_output, streaming=streaming
248 + )
249 +
250 + if expected_content and not streaming:
251 + assert content == expected_content
252 + assert len(tool_calls) == len(expected_tool_calls)
253 + for actual, expected in zip(tool_calls, expected_tool_calls):
254 + assert actual.type == "function"
255 + assert actual.function == expected
256 +
257 +
258 +def test_streaming_tool_call_with_large_steps(lfm2_tokenizer: TokenizerLike):
259 + tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
260 + model_output_deltas = [
261 + f"{TOOL_CALL_START}[get_candidate_status(candidate_id='12345'), "
262 + f"{PARAMETERLESS_FUNCTION_OUTPUT}, "
263 + f"{EMPTY_LIST_FUNCTION_OUTPUT}]{TOOL_CALL_END}",
264 + ]
265 +
266 + reconstructor = run_tool_extraction_streaming(
267 + tool_parser, model_output_deltas, assert_one_tool_per_delta=False
268 + )
269 +
270 + assert len(reconstructor.tool_calls) == 3
271 + assert reconstructor.tool_calls[0].function == SIMPLE_FUNCTION_CALL
272 + assert reconstructor.tool_calls[1].function == PARAMETERLESS_FUNCTION_CALL
273 + assert reconstructor.tool_calls[2].function == EMPTY_LIST_FUNCTION_CALL
274 +
275 +
276 +def test_streaming_full_block_and_trailing_in_single_delta(
277 + lfm2_tokenizer: TokenizerLike,
278 +):
279 + """The entire <|tool_call_start|>[...]<|tool_call_end|> block plus
280 + trailing assistant text arrive in one delta. Trailing content must
281 + still be emitted — not silently dropped."""
282 + tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
283 + full_text = f"{TOOL_CALL_START}[{SIMPLE_FUNCTION_OUTPUT}]{TOOL_CALL_END}\nDone."
284 +
285 + reconstructor = run_tool_extraction_streaming(tool_parser, [full_text])
286 +
287 + assert len(reconstructor.tool_calls) == 1
288 + assert reconstructor.tool_calls[0].function == SIMPLE_FUNCTION_CALL
289 + assert "Done." in reconstructor.other_content
290 +
291 +
292 +def test_streaming_leading_content_and_full_block_in_single_delta(
293 + lfm2_tokenizer: TokenizerLike,
294 +):
295 + """Leading assistant text plus the entire tool block arrive in one
296 + delta. Leading content must be emitted — not silently dropped."""
297 + tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
298 + full_text = (
299 + f"Let me check. {TOOL_CALL_START}[{SIMPLE_FUNCTION_OUTPUT}]{TOOL_CALL_END}"
300 + )
301 +
302 + reconstructor = run_tool_extraction_streaming(tool_parser, [full_text])
303 +
304 + assert len(reconstructor.tool_calls) == 1
305 + assert reconstructor.tool_calls[0].function == SIMPLE_FUNCTION_CALL
306 + assert "Let me check." in reconstructor.other_content
307 +
308 +
309 +def test_streaming_leading_block_and_trailing_in_single_delta(
310 + lfm2_tokenizer: TokenizerLike,
311 +):
312 + """Leading text + complete tool block + trailing text in one delta.
313 + Both leading and trailing content must be preserved."""
314 + tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
315 + full_text = (
316 + "Let me check. "
317 + f"{TOOL_CALL_START}[{SIMPLE_FUNCTION_OUTPUT}]{TOOL_CALL_END}\nDone."
318 + )
319 +
320 + reconstructor = run_tool_extraction_streaming(tool_parser, [full_text])
321 +
322 + assert len(reconstructor.tool_calls) == 1
323 + assert reconstructor.tool_calls[0].function == SIMPLE_FUNCTION_CALL
324 + assert "Let me check." in reconstructor.other_content
325 + assert "Done." in reconstructor.other_content
326 +
327 +
328 +def test_echoed_tool_call_body_not_leaked_to_content(
329 + lfm2_tokenizer: TokenizerLike,
330 +):
331 + """LFM2 sometimes emits the tool call body again after the first
332 + <|tool_call_end|>, capped with a second <|tool_call_end|>. The
333 + echoed body must not surface as assistant content — neither in
334 + streaming nor non-streaming paths."""
335 + tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
336 + body = (
337 + "[grocery.orderIngredients("
338 + "ingredientList=[{'name': 'apple', 'quantity': '2'}], "
339 + "deliveryAddress='123 Main St')]"
340 + )
341 + model_output = f"{TOOL_CALL_START}{body}{TOOL_CALL_END}{body}{TOOL_CALL_END}"
342 +
343 + # Non-streaming
344 + content_ns, tool_calls_ns = run_tool_extraction(
345 + tool_parser, model_output, streaming=False
346 + )
347 + assert len(tool_calls_ns) == 1
348 + assert tool_calls_ns[0].function.name == "grocery.orderIngredients"
349 + assert content_ns in (None, "")
350 +
351 + # Streaming: re-fetch a fresh parser since state was mutated above.
352 + tool_parser2: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
353 + content_s, tool_calls_s = run_tool_extraction(
354 + tool_parser2, model_output, streaming=True
355 + )
356 + assert len(tool_calls_s) == 1
357 + assert tool_calls_s[0].function.name == "grocery.orderIngredients"
358 + # Echoed body must not leak as content.
359 + assert content_s in (None, "")
360 + assert "grocery.orderIngredients" not in (content_s or "")
361 + assert TOOL_CALL_END not in (content_s or "")
362 +
363 +
364 +def test_streaming_char_by_char_multi_dict_list(lfm2_tokenizer: TokenizerLike):
365 + """Stream a tool call containing a list of multiple dicts one
366 + character at a time. Every prefix lands in some partial-parse state
367 + (mid-key, mid-value, open quote inside dict, empty dict, etc.). The
368 + parser must not raise — incomplete prefixes should silently wait for
369 + more text instead of logging exceptions."""
370 + tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
371 + full_text = (
372 + f"{TOOL_CALL_START}[grocery.orderIngredients("
373 + "ingredientList=["
374 + '{"name": "apple", "quantity": "2"}, '
375 + '{"name": "bread", "quantity": "1"}'
376 + f"])]{TOOL_CALL_END}"
377 + )
378 + deltas = [c for c in full_text]
379 +
380 + reconstructor = run_tool_extraction_streaming(
381 + tool_parser, deltas, assert_one_tool_per_delta=False
382 + )
383 +
384 + assert len(reconstructor.tool_calls) == 1
385 + assert reconstructor.tool_calls[0].function.name == "grocery.orderIngredients"
386 + import json
387 +
388 + args = json.loads(reconstructor.tool_calls[0].function.arguments)
389 + assert args == {
390 + "ingredientList": [
391 + {"name": "apple", "quantity": "2"},
392 + {"name": "bread", "quantity": "1"},
393 + ]
394 + }
395 +
396 +
397 +def test_streaming_dotted_name_in_single_delta(lfm2_tokenizer: TokenizerLike):
398 + """A pythonic call with a dotted/attribute function name (e.g.
399 + ``domain.method(arg=...)``) must be parsed correctly in streaming mode
400 + just as in non-streaming mode."""
401 + tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
402 + full_text = f"{TOOL_CALL_START}[{DOTTED_NAME_FUNCTION_OUTPUT}]{TOOL_CALL_END}"
403 +
404 + reconstructor = run_tool_extraction_streaming(tool_parser, [full_text])
405 +
406 + assert len(reconstructor.tool_calls) == 1
407 + assert reconstructor.tool_calls[0].function == DOTTED_NAME_FUNCTION_CALL
408 +
409 +
410 +def test_adjust_request_disables_skip_special_tokens(
411 + lfm2_tokenizer: TokenizerLike,
412 +):
413 + """When tools are present, the parser must force
414 + ``skip_special_tokens=False`` so the engine does not strip the
415 + <|tool_call_start|>/<|tool_call_end|> sentinels before they reach the
416 + parser."""
417 + from vllm.entrypoints.openai.chat_completion.protocol import (
418 + ChatCompletionRequest,
419 + )
420 +
421 + tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
422 +
423 + request_with_tools = ChatCompletionRequest(
424 + messages=[],
425 + model="test-model",
426 + tools=[
427 + {
428 + "type": "function",
429 + "function": {
430 + "name": "get_weather",
431 + "parameters": {
432 + "type": "object",
433 + "properties": {"city": {"type": "string"}},
434 + },
435 + },
436 + }
437 + ],
438 + )
439 + assert request_with_tools.skip_special_tokens is True
440 + adjusted = tool_parser.adjust_request(request_with_tools)
441 + assert adjusted.skip_special_tokens is False
442 +
443 + # No tools → no override; default behaviour preserved.
444 + request_no_tools = ChatCompletionRequest(messages=[], model="test-model")
445 + assert request_no_tools.skip_special_tokens is True
446 + adjusted_no_tools = tool_parser.adjust_request(request_no_tools)
447 + assert adjusted_no_tools.skip_special_tokens is True
448 +
449 +
450 +@pytest.mark.parametrize("streaming", [False])
451 +def test_regex_timeout_handling(streaming: bool, lfm2_tokenizer: TokenizerLike):
452 + """Test regex timeout is handled gracefully."""
453 + tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
454 +
455 + fake_input = f"{TOOL_CALL_START}[A(A=" + "\t)A(A=,\t" * 2
456 + fake_input += f"]{TOOL_CALL_END}"
457 +
458 + mock_regex = MagicMock()
459 + mock_regex.match.side_effect = TimeoutError("Regex timeout")
460 +
461 + with patch.object(tool_parser, "TOOL_CALL_REGEX", mock_regex):
462 + content, tool_calls = run_tool_extraction(
463 + tool_parser, fake_input, streaming=streaming
464 + )
465 +
466 + assert content == fake_input
467 + assert len(tool_calls) == 0
468 + mock_regex.match.assert_called_once()
vllm/tool_parsers/__init__.py
+4
@@ -94,6 +94,10 @@ _TOOL_PARSERS_TO_REGISTER = {
94 "jamba_tool_parser",
95 "JambaToolParser",
96 ),
97 + "lfm2": (
98 + "lfm2_tool_parser",
99 + "Lfm2ToolParser",
100 + ),
101 "kimi_k2": (
102 "kimi_k2_tool_parser",
103 "KimiK2ToolParser",
vllm/tool_parsers/lfm2_tool_parser.py new
+343
@@ -0,0 +1,343 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import ast
5 +from collections.abc import Sequence
6 +
7 +import regex as re
8 +
9 +import vllm.envs as envs
10 +from vllm.entrypoints.openai.chat_completion.protocol import (
11 + ChatCompletionRequest,
12 +)
13 +from vllm.entrypoints.openai.engine.protocol import (
14 + DeltaMessage,
15 + ExtractedToolCallInformation,
16 +)
17 +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
18 +from vllm.logger import init_logger
19 +from vllm.tokenizers import TokenizerLike
20 +from vllm.tool_parsers.abstract_tool_parser import (
21 + Tool,
22 + ToolParser,
23 +)
24 +from vllm.tool_parsers.utils import (
25 + UnexpectedAstError,
26 + compute_tool_delta,
27 + handle_single_tool,
28 + make_valid_python,
29 +)
30 +
31 +logger = init_logger(__name__)
32 +
33 +TOOL_CALL_START = "<|tool_call_start|>"
34 +TOOL_CALL_END = "<|tool_call_end|>"
35 +
36 +
37 +class Lfm2ToolParser(ToolParser):
38 + """
39 + Tool call parser for LiquidAI LFM2/LFM2.5 models that produce pythonic
40 + tool calls wrapped in <|tool_call_start|> and <|tool_call_end|> tokens.
41 +
42 + Example model output:
43 + <|tool_call_start|>[get_weather(location="Paris")]<|tool_call_end|>
44 + The weather in Paris is sunny.
45 +
46 + Used when --enable-auto-tool-choice --tool-call-parser lfm2 are all set.
47 + """
48 +
49 + TOOL_CALL_REGEX = re.compile(r"\[.*\]$", re.DOTALL)
50 +
51 + def __init__(
52 + self,
53 + tokenizer: TokenizerLike,
54 + tools: list[Tool] | None = None,
55 + ):
56 + super().__init__(tokenizer, tools)
57 +
58 + self.tool_call_start_token_id = self.vocab.get(TOOL_CALL_START)
59 + self.tool_call_end_token_id = self.vocab.get(TOOL_CALL_END)
60 +
61 + if self.tool_call_start_token_id is None or self.tool_call_end_token_id is None:
62 + raise RuntimeError(
63 + "LFM2 tool parser could not locate "
64 + "<|tool_call_start|>/<|tool_call_end|> tokens in the "
65 + "tokenizer!"
66 + )
67 +
68 + # Trailing content already emitted to the client. Used by the
69 + # streaming path to suppress LFM2's frequent echo of the tool
70 + # call body after the first <|tool_call_end|> while still
71 + # allowing legitimate post-call prose through.
72 + self._trailing_emitted: str = ""
73 +
74 + def adjust_request(
75 + self, request: ChatCompletionRequest | ResponsesRequest
76 + ) -> ChatCompletionRequest | ResponsesRequest:
77 + request = super().adjust_request(request)
78 + if request.tools and request.tool_choice != "none":
79 + # The <|tool_call_start|>/<|tool_call_end|> sentinels are
80 + # registered as special tokens in the LFM2/LFM2.5 tokenizer.
81 + # With the default ``skip_special_tokens=True`` they are
82 + # stripped from the decoded text before reaching this parser,
83 + # so the tool block becomes invisible. Force the engine to
84 + # preserve them when tool calling is enabled.
85 + request.skip_special_tokens = False
86 + return request
87 +
88 + # Rename for readability. This is NOT a tool id.
89 + @property
90 + def current_tool_index(self) -> int:
91 + return self.current_tool_id
92 +
93 + @current_tool_index.setter
94 + def current_tool_index(self, value: int) -> None:
95 + self.current_tool_id = value
96 +
97 + @staticmethod
98 + def _strip_echo(raw_after: str) -> str:
99 + """Drop any orphan <|tool_call_end|> (and the preceding text) from
100 + trailing content. LFM2 occasionally echoes the call body after the
101 + first end token and caps it with a second end token; everything
102 + through the last such orphan is model garbage, not user content."""
103 + last_orphan = raw_after.rfind(TOOL_CALL_END)
104 + if last_orphan != -1:
105 + return raw_after[last_orphan + len(TOOL_CALL_END) :]
106 + return raw_after
107 +
108 + @classmethod
109 + def _extract_tool_call_text(
110 + cls, model_output: str
111 + ) -> tuple[str | None, str | None]:
112 + """Extract the pythonic call text and surrounding content.
113 +
114 + Returns (tool_text, content) where tool_text is the text between
115 + the sentinel tokens and content is everything outside them.
116 + """
117 + start_idx = model_output.find(TOOL_CALL_START)
118 + if start_idx == -1:
119 + return None, model_output
120 +
121 + end_idx = model_output.find(TOOL_CALL_END, start_idx)
122 + if end_idx == -1:
123 + # Incomplete — treat entire text after start as tool call
124 + tool_text = model_output[start_idx + len(TOOL_CALL_START) :]
125 + content_before = model_output[:start_idx].strip()
126 + content = content_before or None
127 + return tool_text, content
128 +
129 + tool_text = model_output[start_idx + len(TOOL_CALL_START) : end_idx]
130 + content_before = model_output[:start_idx].strip()
131 + content_after = cls._strip_echo(
132 + model_output[end_idx + len(TOOL_CALL_END) :]
133 + ).strip()
134 +
135 + content_parts = []
136 + if content_before:
137 + content_parts.append(content_before)
138 + if content_after:
139 + content_parts.append(content_after)
140 + content = "\n".join(content_parts) if content_parts else None
141 +
142 + return tool_text, content
143 +
144 + def extract_tool_calls(
145 + self, model_output: str, request: ChatCompletionRequest
146 + ) -> ExtractedToolCallInformation:
147 + tool_text, content = self._extract_tool_call_text(model_output)
148 +
149 + if tool_text is None:
150 + return ExtractedToolCallInformation(
151 + tools_called=False, tool_calls=[], content=model_output
152 + )
153 +
154 + tool_text = tool_text.strip()
155 +
156 + is_tool_call_pattern = False
157 + try:
158 + is_tool_call_pattern = (
159 + self.TOOL_CALL_REGEX.match(
160 + tool_text,
161 + timeout=envs.VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS,
162 + )
163 + is not None
164 + )
165 + except TimeoutError:
166 + logger.warning("Regex timeout occurred when matching tool call pattern.")
167 +
168 + if not is_tool_call_pattern:
169 + return ExtractedToolCallInformation(
170 + tools_called=False, tool_calls=[], content=model_output
171 + )
172 +
173 + try:
174 + module = ast.parse(tool_text)
175 + parsed = getattr(module.body[0], "value", None)
176 + if isinstance(parsed, ast.List) and all(
177 + isinstance(e, ast.Call) for e in parsed.elts
178 + ):
179 + return ExtractedToolCallInformation(
180 + tools_called=True,
181 + tool_calls=[
182 + handle_single_tool(e) # type: ignore
183 + for e in parsed.elts
184 + ],
185 + content=content,
186 + )
187 + else:
188 + raise UnexpectedAstError("Tool output must be a list of function calls")
189 + except Exception:
190 + logger.exception("Error in extracting tool call from response.")
191 + return ExtractedToolCallInformation(
192 + tools_called=False, tool_calls=[], content=model_output
193 + )
194 +
195 + def extract_tool_calls_streaming(
196 + self,
197 + previous_text: str,
198 + current_text: str,
199 + delta_text: str,
200 + previous_token_ids: Sequence[int],
201 + current_token_ids: Sequence[int],
202 + delta_token_ids: Sequence[int],
203 + request: ChatCompletionRequest,
204 + ) -> DeltaMessage | None:
205 + # If the tool call start token hasn't appeared yet, stream as content.
206 + if TOOL_CALL_START not in current_text:
207 + return DeltaMessage(content=delta_text)
208 +
209 + # Compute leading content (before <|tool_call_start|>) that arrived
210 + # in this delta and hasn't been streamed yet. Without this, when the
211 + # prefix and the start token land in the same delta the prefix is
212 + # silently dropped — token-by-token streaming masked the bug because
213 + # the prefix tokens always arrived in earlier deltas.
214 + leading_content = ""
215 + if TOOL_CALL_START not in previous_text:
216 + start_idx = current_text.find(TOOL_CALL_START)
217 + # previous_text contained no start token, so it has already been
218 + # streamed via the no-start-token branch above.
219 + leading_content = current_text[len(previous_text) : start_idx]
220 +
221 + has_end_in_current = TOOL_CALL_END in current_text
222 + has_end_in_previous = TOOL_CALL_END in previous_text
223 +
224 + # Compute trailing content (after <|tool_call_end|>) not yet
225 + # streamed. LFM2 frequently echoes the tool call body again
226 + # after the first end token, capped with a second end token.
227 + # Suppress that echo:
228 + # - If a second <|tool_call_end|> has appeared, treat
229 + # everything through the last one as garbage.
230 + # - If the trailing starts with `[` or `<` (potential echo
231 + # body or another sentinel) and no second end token has
232 + # arrived yet, buffer it instead of emitting.
233 + trailing_content = ""
234 + if has_end_in_current:
235 + end_idx = current_text.find(TOOL_CALL_END) + len(TOOL_CALL_END)
236 + full_trailing = current_text[end_idx:]
237 + stripped_trailing = self._strip_echo(full_trailing)
238 + if stripped_trailing == full_trailing:
239 + # No second end token yet — possibly mid-echo.
240 + lstripped = full_trailing.lstrip()
241 + if lstripped.startswith("[") or lstripped.startswith("<"):
242 + # Suspect echo; hold off until resolved.
243 + final_trailing = self._trailing_emitted
244 + else:
245 + final_trailing = full_trailing
246 + else:
247 + final_trailing = stripped_trailing
248 + if final_trailing.startswith(self._trailing_emitted):
249 + trailing_content = final_trailing[len(self._trailing_emitted) :]
250 + self._trailing_emitted = final_trailing
251 +
252 + # If tools were already parsed in a prior delta, just stream any
253 + # newly arrived trailing content.
254 + if has_end_in_current and self.prev_tool_call_arr and has_end_in_previous:
255 + if trailing_content:
256 + return DeltaMessage(content=trailing_content)
257 + return DeltaMessage(content="")
258 +
259 + # Extract the pythonic text between start and end tokens.
260 + tool_text = current_text.split(TOOL_CALL_START, 1)[1]
261 + # Strip the end token if present (entire call arrived at once).
262 + if TOOL_CALL_END in tool_text:
263 + tool_text = tool_text.split(TOOL_CALL_END, 1)[0]
264 +
265 + def _content_only_or_none() -> DeltaMessage | None:
266 + """Return a content-only delta if any content arrived in this
267 + chunk, otherwise None. Used on incremental-parse failure paths
268 + so leading/trailing content is never silently dropped.
269 + """
270 + combined = leading_content + trailing_content
271 + return DeltaMessage(content=combined) if combined else None
272 +
273 + try:
274 + valid_and_added_text = make_valid_python(tool_text)
275 + if valid_and_added_text is None:
276 + return _content_only_or_none()
277 + valid_text, added_text = valid_and_added_text
278 +
279 + module = ast.parse(valid_text)
280 + parsed = getattr(module.body[0], "value", None)
281 + if not isinstance(parsed, ast.List) or not all(
282 + isinstance(e, ast.Call) for e in parsed.elts
283 + ):
284 + raise UnexpectedAstError("Tool output must be a list of function calls")
285 + tool_calls = [
286 + handle_single_tool(e) # type: ignore
287 + for e in parsed.elts
288 + ]
289 +
290 + tool_deltas = []
291 + for index, new_call in enumerate(tool_calls):
292 + if index < self.current_tool_index:
293 + continue
294 +
295 + self.current_tool_index = index
296 + if len(self.streamed_args_for_tool) == index:
297 + self.streamed_args_for_tool.append("")
298 +
299 + new_call_complete = (
300 + index < len(tool_calls) - 1 or ")]" not in added_text
301 + )
302 + if new_call_complete:
303 + self.current_tool_index += 1
304 +
305 + withheld_suffix = added_text[:-2] if not new_call_complete else ""
306 + if not new_call_complete and added_text[-2] == ")":
307 + withheld_suffix = withheld_suffix + "}"
308 + withheld_suffix = withheld_suffix.replace("'", '"')
309 + delta = compute_tool_delta(
310 + self.streamed_args_for_tool[index],
311 + new_call,
312 + index,
313 + withheld_suffix,
314 + )
315 +
316 + if delta is not None:
317 + tool_deltas.append(delta)
318 + if (
319 + delta.function is not None
320 + and delta.function.arguments is not None
321 + ):
322 + self.streamed_args_for_tool[index] += delta.function.arguments
323 +
324 + if tool_deltas and not self.prev_tool_call_arr:
325 + self.prev_tool_call_arr = [{"arguments": {}}]
326 +
327 + combined_content = leading_content + trailing_content
328 +
329 + if tool_deltas or combined_content:
330 + return DeltaMessage(
331 + content=combined_content if combined_content else None,
332 + tool_calls=tool_deltas,
333 + )
334 + elif not added_text and self.current_tool_id > 0:
335 + return DeltaMessage(content="")
336 + else:
337 + return None
338 + except Exception:
339 + logger.exception("Error trying to handle streaming tool call.")
340 + logger.debug(
341 + "Skipping chunk as a result of tool streaming extraction error"
342 + )
343 + return _content_only_or_none()
vllm/tool_parsers/utils.py
+49 -5
@@ -308,20 +308,43 @@ def get_parameter_value(val: ast.expr) -> Any:
308 raise UnexpectedAstError("Tool call arguments must be literals")
309
310
311 +def _ast_callable_dotted_name(node: ast.expr) -> str:
312 + """Return the dotted name for a call target, walking ``ast.Attribute``
313 + chains so ``a.b.c(...)`` becomes ``"a.b.c"``.
314 +
315 + Raises:
316 + UnexpectedAstError: If the chain does not bottom out in an
317 + ``ast.Name`` (e.g. subscript or call expression as receiver).
318 + """
319 + parts: list[str] = []
320 + current: ast.expr = node
321 + while isinstance(current, ast.Attribute):
322 + parts.append(current.attr)
323 + current = current.value
324 + if not isinstance(current, ast.Name):
325 + raise UnexpectedAstError("Invalid tool call name")
326 + parts.append(current.id)
327 + return ".".join(reversed(parts))
328 +
329 +
330 def handle_single_tool(call: ast.Call) -> ToolCall:
331 """Convert a single AST function call node into a ToolCall object.
332
333 + Accepts both bare names (``foo(...)``) and dotted attribute chains
334 + (``a.b.c(...)``); the resulting tool call ``name`` field preserves the
335 + dotted form.
336 +
337 Raises:
315 - UnexpectedAstError: If the call node does not have a simple
316 - function name (e.g. it's an attribute access or subscript).
338 + UnexpectedAstError: If the call target is neither a simple name
339 + nor a chain of attribute accesses bottoming out in a name.
340 """
318 - if not isinstance(call.func, ast.Name):
341 + if not isinstance(call.func, (ast.Name, ast.Attribute)):
342 logger.warning(
343 "Tool call has non-simple function name: %s",
344 ast.dump(call.func),
345 )
346 raise UnexpectedAstError("Invalid tool call name")
324 - function_name = call.func.id
347 + function_name = _ast_callable_dotted_name(call.func)
348 arguments = {}
349 for keyword in call.keywords:
350 arguments[keyword.arg] = get_parameter_value(keyword.value)
@@ -403,7 +426,28 @@ def make_valid_python(text: str) -> tuple[str, str] | None:
426 for char in reversed(bracket_stack):
427 added_text += _CLOSING[char]
428
406 - return text + added_text, added_text
429 + candidate = text + added_text
430 +
431 + # Streaming partial text can land in shapes the bracket-counting
432 + # heuristics above don't catch. Two failure modes:
433 + # 1. Mid-key inside a dict (`..., "k`) closes to `..., "k"}` — a
434 + # syntactically invalid mixed dict/set.
435 + # 2. A bare string inside a dict (`{"k`) closes to `{"k"}` — valid
436 + # Python but a *set* literal, which downstream tool-call AST
437 + # handling rejects.
438 + # Validate the candidate parses, has a body, and contains no Set
439 + # nodes (pythonic tool calls always use dicts for `{...}`).
440 + try:
441 + module = ast.parse(candidate)
442 + except SyntaxError:
443 + return None
444 + if not module.body:
445 + return None
446 + for node in ast.walk(module):
447 + if isinstance(node, ast.Set):
448 + return None
449 +
450 + return candidate, added_text
451
452
453 def compute_tool_delta(