| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from unittest.mock import MagicMock, patch |
| 5 | |
| 6 | import pytest |
| 7 | from transformers import AutoTokenizer |
| 8 | |
| 9 | from tests.tool_parsers.utils import ( |
| 10 | run_tool_extraction, |
| 11 | run_tool_extraction_streaming, |
| 12 | ) |
| 13 | from vllm.entrypoints.generate.base.protocol import FunctionCall |
| 14 | from vllm.tokenizers import TokenizerLike |
| 15 | from vllm.tool_parsers import ToolParser, ToolParserManager |
| 16 | |
| 17 | TOOL_CALL_START = "<|tool_call_start|>" |
| 18 | TOOL_CALL_END = "<|tool_call_end|>" |
| 19 | |
| 20 | SIMPLE_FUNCTION_OUTPUT = "get_candidate_status(candidate_id='12345')" |
| 21 | SIMPLE_FUNCTION_CALL = FunctionCall( |
| 22 | name="get_candidate_status", |
| 23 | arguments='{"candidate_id": "12345"}', |
| 24 | ) |
| 25 | MORE_TYPES_FUNCTION_OUTPUT = ( |
| 26 | "register_user(name='John Doe', " |
| 27 | "age=37, " |
| 28 | "address={'city': 'San Francisco', 'state': 'CA'}, " |
| 29 | "role=None, " |
| 30 | "passed_test=True, " |
| 31 | "aliases=['John', 'Johnny'])" |
| 32 | ) |
| 33 | MORE_TYPES_FUNCTION_CALL = FunctionCall( |
| 34 | name="register_user", |
| 35 | arguments='{"name": "John Doe", ' |
| 36 | '"age": 37, ' |
| 37 | '"address": {"city": "San Francisco", "state": "CA"}, ' |
| 38 | '"role": null, ' |
| 39 | '"passed_test": true, ' |
| 40 | '"aliases": ["John", "Johnny"]}', |
| 41 | ) |
| 42 | PARAMETERLESS_FUNCTION_OUTPUT = "get_weather()" |
| 43 | PARAMETERLESS_FUNCTION_CALL = FunctionCall( |
| 44 | name="get_weather", |
| 45 | arguments="{}", |
| 46 | ) |
| 47 | EMPTY_DICT_FUNCTION_OUTPUT = "do_something_cool(additional_data={})" |
| 48 | EMPTY_DICT_FUNCTION_CALL = FunctionCall( |
| 49 | name="do_something_cool", |
| 50 | arguments='{"additional_data": {}}', |
| 51 | ) |
| 52 | EMPTY_LIST_FUNCTION_OUTPUT = "do_something_cool(steps=[])" |
| 53 | EMPTY_LIST_FUNCTION_CALL = FunctionCall( |
| 54 | name="do_something_cool", |
| 55 | arguments='{"steps": []}', |
| 56 | ) |
| 57 | SET_ARG_FUNCTION_OUTPUT = "label(tags={'urgent', 'bug'})" |
| 58 | SET_ARG_FUNCTION_CALL = FunctionCall( |
| 59 | name="label", |
| 60 | arguments='{"tags": ["urgent", "bug"]}', |
| 61 | ) |
| 62 | LEADING_ZERO_FUNCTION_OUTPUT = "set_date(month=07, day=05)" |
| 63 | LEADING_ZERO_FUNCTION_CALL = FunctionCall( |
| 64 | name="set_date", |
| 65 | arguments='{"month": 7, "day": 5}', |
| 66 | ) |
| 67 | NESTED_QUOTE_FUNCTION_OUTPUT = ( |
| 68 | "bash(command='sed -n '360,450p' /testbed/sympy/matrices/common.py')" |
| 69 | ) |
| 70 | NESTED_QUOTE_FUNCTION_CALL = FunctionCall( |
| 71 | name="bash", |
| 72 | arguments=('{"command": "sed -n \'360,450p\' /testbed/sympy/matrices/common.py"}'), |
| 73 | ) |
| 74 | NESTED_PYTHON_C_FUNCTION_OUTPUT = ( |
| 75 | "bash(command='cd /testbed && python3 -c \"from sympy import latex\n" |
| 76 | "print(latex(3*x, mul_symbol='\\,'))\"')" |
| 77 | ) |
| 78 | NESTED_PYTHON_C_FUNCTION_CALL = FunctionCall( |
| 79 | name="bash", |
| 80 | arguments=( |
| 81 | '{"command": "cd /testbed && python3 -c \\"from sympy import latex\\n' |
| 82 | "print(latex(3*x, mul_symbol='\\\\,'))\\\"\"}" |
| 83 | ), |
| 84 | ) |
| 85 | ESCAPED_STRING_FUNCTION_OUTPUT = ( |
| 86 | r"get_weather(city='Martha\'s Vineyard', metric='\"cool units\"')" |
| 87 | ) |
| 88 | ESCAPED_STRING_FUNCTION_CALL = FunctionCall( |
| 89 | name="get_weather", |
| 90 | arguments='{"city": "Martha\'s Vineyard", "metric": "\\"cool units\\""}', |
| 91 | ) |
| 92 | DOTTED_NAME_FUNCTION_OUTPUT = ( |
| 93 | "grocery.orderIngredients(" |
| 94 | "ingredientList=[{'name': 'Lasagna noodles', 'amount': 250, 'unit': 'g'}], " |
| 95 | "deliveryAddress='845 Willow Lane, Springfield, IL 62704')" |
| 96 | ) |
| 97 | DOTTED_NAME_FUNCTION_CALL = FunctionCall( |
| 98 | name="grocery.orderIngredients", |
| 99 | arguments=( |
| 100 | '{"ingredientList": [' |
| 101 | '{"name": "Lasagna noodles", "amount": 250, "unit": "g"}], ' |
| 102 | '"deliveryAddress": "845 Willow Lane, Springfield, IL 62704"}' |
| 103 | ), |
| 104 | ) |
| 105 | # Agentic shell commands: a bracket inside a string argument must not corrupt |
| 106 | # the streaming bracket tracker. |
| 107 | BRACKET_IN_STRING_FUNCTION_OUTPUT = "exec(command='grep -F \"]\" log.txt')" |
| 108 | BRACKET_IN_STRING_FUNCTION_CALL = FunctionCall( |
| 109 | name="exec", |
| 110 | arguments='{"command": "grep -F \\"]\\" log.txt"}', |
| 111 | ) |
| 112 | # A raw newline inside a string argument (multi-line shell command) is invalid |
| 113 | # Python; the parser must recover the full value instead of dropping the call. |
| 114 | MULTILINE_FUNCTION_OUTPUT = ( |
| 115 | "exec(command='cat > f.py << EOF\nimport csv\nprint(1)\nEOF')" |
| 116 | ) |
| 117 | MULTILINE_FUNCTION_CALL = FunctionCall( |
| 118 | name="exec", |
| 119 | arguments='{"command": "cat > f.py << EOF\\nimport csv\\nprint(1)\\nEOF"}', |
| 120 | ) |
| 121 | # A NUL byte inside a string argument makes ast.parse raise ValueError (not |
| 122 | # SyntaxError) for the whole source; the escape path must recover the call. |
| 123 | NUL_BYTE_FUNCTION_OUTPUT = "exec(command='printf a\x00b')" |
| 124 | NUL_BYTE_FUNCTION_CALL = FunctionCall( |
| 125 | name="exec", |
| 126 | arguments='{"command": "printf a\\u0000b"}', |
| 127 | ) |
| 128 | |
| 129 | |
| 130 | @pytest.fixture(scope="module") |
| 131 | def lfm2_tokenizer() -> TokenizerLike: |
| 132 | return AutoTokenizer.from_pretrained("LiquidAI/LFM2.5-1.2B-Instruct") |
| 133 | |
| 134 | |
| 135 | def _wrap(tool_text: str, content_after: str = "") -> str: |
| 136 | """Wrap pythonic tool call in LFM2.5 sentinel tokens.""" |
| 137 | result = f"{TOOL_CALL_START}[{tool_text}]{TOOL_CALL_END}" |
| 138 | if content_after: |
| 139 | result += f"\n{content_after}" |
| 140 | return result |
| 141 | |
| 142 | |
| 143 | @pytest.mark.parametrize("streaming", [True, False]) |
| 144 | def test_no_tool_call(streaming: bool, lfm2_tokenizer: TokenizerLike): |
| 145 | tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer) |
| 146 | model_output = "How can I help you today?" |
| 147 | |
| 148 | content, tool_calls = run_tool_extraction( |
| 149 | tool_parser, model_output, streaming=streaming |
| 150 | ) |
| 151 | |
| 152 | assert content == model_output |
| 153 | assert len(tool_calls) == 0 |
| 154 | |
| 155 | |
| 156 | TEST_CASES = [ |
| 157 | pytest.param( |
| 158 | True, |
| 159 | _wrap(SIMPLE_FUNCTION_OUTPUT), |
| 160 | [SIMPLE_FUNCTION_CALL], |
| 161 | None, |
| 162 | id="simple_streaming", |
| 163 | ), |
| 164 | pytest.param( |
| 165 | False, |
| 166 | _wrap(SIMPLE_FUNCTION_OUTPUT), |
| 167 | [SIMPLE_FUNCTION_CALL], |
| 168 | None, |
| 169 | id="simple_nonstreaming", |
| 170 | ), |
| 171 | pytest.param( |
| 172 | True, |
| 173 | _wrap(MORE_TYPES_FUNCTION_OUTPUT), |
| 174 | [MORE_TYPES_FUNCTION_CALL], |
| 175 | None, |
| 176 | id="more_types_streaming", |
| 177 | ), |
| 178 | pytest.param( |
| 179 | False, |
| 180 | _wrap(MORE_TYPES_FUNCTION_OUTPUT), |
| 181 | [MORE_TYPES_FUNCTION_CALL], |
| 182 | None, |
| 183 | id="more_types_nonstreaming", |
| 184 | ), |
| 185 | pytest.param( |
| 186 | True, |
| 187 | _wrap(PARAMETERLESS_FUNCTION_OUTPUT), |
| 188 | [PARAMETERLESS_FUNCTION_CALL], |
| 189 | None, |
| 190 | id="parameterless_streaming", |
| 191 | ), |
| 192 | pytest.param( |
| 193 | False, |
| 194 | _wrap(PARAMETERLESS_FUNCTION_OUTPUT), |
| 195 | [PARAMETERLESS_FUNCTION_CALL], |
| 196 | None, |
| 197 | id="parameterless_nonstreaming", |
| 198 | ), |
| 199 | pytest.param( |
| 200 | True, |
| 201 | _wrap(EMPTY_DICT_FUNCTION_OUTPUT), |
| 202 | [EMPTY_DICT_FUNCTION_CALL], |
| 203 | None, |
| 204 | id="empty_dict_streaming", |
| 205 | ), |
| 206 | pytest.param( |
| 207 | False, |
| 208 | _wrap(EMPTY_DICT_FUNCTION_OUTPUT), |
| 209 | [EMPTY_DICT_FUNCTION_CALL], |
| 210 | None, |
| 211 | id="empty_dict_nonstreaming", |
| 212 | ), |
| 213 | pytest.param( |
| 214 | True, |
| 215 | _wrap(EMPTY_LIST_FUNCTION_OUTPUT), |
| 216 | [EMPTY_LIST_FUNCTION_CALL], |
| 217 | None, |
| 218 | id="empty_list_streaming", |
| 219 | ), |
| 220 | pytest.param( |
| 221 | False, |
| 222 | _wrap(EMPTY_LIST_FUNCTION_OUTPUT), |
| 223 | [EMPTY_LIST_FUNCTION_CALL], |
| 224 | None, |
| 225 | id="empty_list_nonstreaming", |
| 226 | ), |
| 227 | pytest.param( |
| 228 | True, |
| 229 | _wrap(ESCAPED_STRING_FUNCTION_OUTPUT), |
| 230 | [ESCAPED_STRING_FUNCTION_CALL], |
| 231 | None, |
| 232 | id="escaped_string_streaming", |
| 233 | ), |
| 234 | pytest.param( |
| 235 | False, |
| 236 | _wrap(ESCAPED_STRING_FUNCTION_OUTPUT), |
| 237 | [ESCAPED_STRING_FUNCTION_CALL], |
| 238 | None, |
| 239 | id="escaped_string_nonstreaming", |
| 240 | ), |
| 241 | pytest.param( |
| 242 | True, |
| 243 | _wrap(f"{SIMPLE_FUNCTION_OUTPUT}, {MORE_TYPES_FUNCTION_OUTPUT}"), |
| 244 | [SIMPLE_FUNCTION_CALL, MORE_TYPES_FUNCTION_CALL], |
| 245 | None, |
| 246 | id="parallel_calls_streaming", |
| 247 | ), |
| 248 | pytest.param( |
| 249 | False, |
| 250 | _wrap(f"{SIMPLE_FUNCTION_OUTPUT}, {MORE_TYPES_FUNCTION_OUTPUT}"), |
| 251 | [SIMPLE_FUNCTION_CALL, MORE_TYPES_FUNCTION_CALL], |
| 252 | None, |
| 253 | id="parallel_calls_nonstreaming", |
| 254 | ), |
| 255 | # LFM2.5 specific: content AFTER tool call |
| 256 | pytest.param( |
| 257 | False, |
| 258 | _wrap( |
| 259 | SIMPLE_FUNCTION_OUTPUT, |
| 260 | content_after="Checking the current status of candidate ID 12345.", |
| 261 | ), |
| 262 | [SIMPLE_FUNCTION_CALL], |
| 263 | "Checking the current status of candidate ID 12345.", |
| 264 | id="content_after_tool_call_nonstreaming", |
| 265 | ), |
| 266 | # Dotted / class-method function names: grocery.orderIngredients(...) |
| 267 | pytest.param( |
| 268 | True, |
| 269 | _wrap(DOTTED_NAME_FUNCTION_OUTPUT), |
| 270 | [DOTTED_NAME_FUNCTION_CALL], |
| 271 | None, |
| 272 | id="dotted_name_streaming", |
| 273 | ), |
| 274 | pytest.param( |
| 275 | False, |
| 276 | _wrap(DOTTED_NAME_FUNCTION_OUTPUT), |
| 277 | [DOTTED_NAME_FUNCTION_CALL], |
| 278 | None, |
| 279 | id="dotted_name_nonstreaming", |
| 280 | ), |
| 281 | # Messy agentic shell commands: literal bracket inside a string argument |
| 282 | pytest.param( |
| 283 | True, |
| 284 | _wrap(BRACKET_IN_STRING_FUNCTION_OUTPUT), |
| 285 | [BRACKET_IN_STRING_FUNCTION_CALL], |
| 286 | None, |
| 287 | id="bracket_in_string_streaming", |
| 288 | ), |
| 289 | pytest.param( |
| 290 | False, |
| 291 | _wrap(BRACKET_IN_STRING_FUNCTION_OUTPUT), |
| 292 | [BRACKET_IN_STRING_FUNCTION_CALL], |
| 293 | None, |
| 294 | id="bracket_in_string_nonstreaming", |
| 295 | ), |
| 296 | # Multi-line string argument (raw newlines in a shell command) |
| 297 | pytest.param( |
| 298 | True, |
| 299 | _wrap(MULTILINE_FUNCTION_OUTPUT), |
| 300 | [MULTILINE_FUNCTION_CALL], |
| 301 | None, |
| 302 | id="multiline_string_arg_streaming", |
| 303 | ), |
| 304 | pytest.param( |
| 305 | False, |
| 306 | _wrap(MULTILINE_FUNCTION_OUTPUT), |
| 307 | [MULTILINE_FUNCTION_CALL], |
| 308 | None, |
| 309 | id="multiline_string_arg_nonstreaming", |
| 310 | ), |
| 311 | # Set argument decoded as a list (JSON has no set type) |
| 312 | pytest.param( |
| 313 | True, |
| 314 | _wrap(SET_ARG_FUNCTION_OUTPUT), |
| 315 | [SET_ARG_FUNCTION_CALL], |
| 316 | None, |
| 317 | id="set_arg_streaming", |
| 318 | ), |
| 319 | pytest.param( |
| 320 | False, |
| 321 | _wrap(SET_ARG_FUNCTION_OUTPUT), |
| 322 | [SET_ARG_FUNCTION_CALL], |
| 323 | None, |
| 324 | id="set_arg_nonstreaming", |
| 325 | ), |
| 326 | # Unescaped same-style quotes nested in a shell command argument |
| 327 | pytest.param( |
| 328 | True, |
| 329 | _wrap(NESTED_QUOTE_FUNCTION_OUTPUT), |
| 330 | [NESTED_QUOTE_FUNCTION_CALL], |
| 331 | None, |
| 332 | id="nested_quote_streaming", |
| 333 | ), |
| 334 | pytest.param( |
| 335 | False, |
| 336 | _wrap(NESTED_QUOTE_FUNCTION_OUTPUT), |
| 337 | [NESTED_QUOTE_FUNCTION_CALL], |
| 338 | None, |
| 339 | id="nested_quote_nonstreaming", |
| 340 | ), |
| 341 | # Doubly nested: multi-line python -c payload with its own inner quotes |
| 342 | pytest.param( |
| 343 | True, |
| 344 | _wrap(NESTED_PYTHON_C_FUNCTION_OUTPUT), |
| 345 | [NESTED_PYTHON_C_FUNCTION_CALL], |
| 346 | None, |
| 347 | id="nested_python_c_streaming", |
| 348 | ), |
| 349 | pytest.param( |
| 350 | False, |
| 351 | _wrap(NESTED_PYTHON_C_FUNCTION_OUTPUT), |
| 352 | [NESTED_PYTHON_C_FUNCTION_CALL], |
| 353 | None, |
| 354 | id="nested_python_c_nonstreaming", |
| 355 | ), |
| 356 | # Zero-padded integer arguments (SyntaxError: leading zeros) |
| 357 | pytest.param( |
| 358 | True, |
| 359 | _wrap(LEADING_ZERO_FUNCTION_OUTPUT), |
| 360 | [LEADING_ZERO_FUNCTION_CALL], |
| 361 | None, |
| 362 | id="leading_zero_int_streaming", |
| 363 | ), |
| 364 | pytest.param( |
| 365 | False, |
| 366 | _wrap(LEADING_ZERO_FUNCTION_OUTPUT), |
| 367 | [LEADING_ZERO_FUNCTION_CALL], |
| 368 | None, |
| 369 | id="leading_zero_int_nonstreaming", |
| 370 | ), |
| 371 | # NUL byte in a string argument (ValueError from ast.parse, not |
| 372 | # SyntaxError) |
| 373 | pytest.param( |
| 374 | True, |
| 375 | _wrap(NUL_BYTE_FUNCTION_OUTPUT), |
| 376 | [NUL_BYTE_FUNCTION_CALL], |
| 377 | None, |
| 378 | id="nul_byte_string_arg_streaming", |
| 379 | ), |
| 380 | pytest.param( |
| 381 | False, |
| 382 | _wrap(NUL_BYTE_FUNCTION_OUTPUT), |
| 383 | [NUL_BYTE_FUNCTION_CALL], |
| 384 | None, |
| 385 | id="nul_byte_string_arg_nonstreaming", |
| 386 | ), |
| 387 | ] |
| 388 | |
| 389 | |
| 390 | @pytest.mark.parametrize( |
| 391 | "streaming, model_output, expected_tool_calls, expected_content", |
| 392 | TEST_CASES, |
| 393 | ) |
| 394 | def test_tool_call( |
| 395 | streaming: bool, |
| 396 | model_output: str, |
| 397 | expected_tool_calls: list[FunctionCall], |
| 398 | expected_content: str | None, |
| 399 | lfm2_tokenizer: TokenizerLike, |
| 400 | ): |
| 401 | tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer) |
| 402 | |
| 403 | content, tool_calls = run_tool_extraction( |
| 404 | tool_parser, model_output, streaming=streaming |
| 405 | ) |
| 406 | |
| 407 | if expected_content and not streaming: |
| 408 | assert content == expected_content |
| 409 | assert len(tool_calls) == len(expected_tool_calls) |
| 410 | for actual, expected in zip(tool_calls, expected_tool_calls): |
| 411 | assert actual.type == "function" |
| 412 | assert actual.function == expected |
| 413 | |
| 414 | |
| 415 | @pytest.mark.parametrize("body", ["", " "]) |
| 416 | def test_empty_tool_call_block(body: str, lfm2_tokenizer: TokenizerLike): |
| 417 | """An empty block ([] or [ ]) means the model called no tools; it must |
| 418 | not report tools_called=True with zero tool calls (the invariant |
| 419 | run_tool_extraction asserts).""" |
| 420 | cls = ToolParserManager.get_tool_parser("lfm2") |
| 421 | model_output = f"{TOOL_CALL_START}[{body}]{TOOL_CALL_END}" |
| 422 | |
| 423 | content, tool_calls = run_tool_extraction( |
| 424 | cls(lfm2_tokenizer), model_output, streaming=False |
| 425 | ) |
| 426 | assert len(tool_calls) == 0 |
| 427 | |
| 428 | reconstructor = run_tool_extraction_streaming( |
| 429 | cls(lfm2_tokenizer), [model_output], assert_one_tool_per_delta=False |
| 430 | ) |
| 431 | assert len(reconstructor.tool_calls) == 0 |
| 432 | |
| 433 | |
| 434 | def test_whitespace_after_start_token(lfm2_tokenizer: TokenizerLike): |
| 435 | """Whitespace between <|tool_call_start|> and the opening bracket must |
| 436 | not break parsing. The streaming path used to feed the indented text to |
| 437 | ast.parse verbatim, so every completion candidate raised |
| 438 | IndentationError and the call was silently dropped (non-streaming |
| 439 | stripped it and succeeded).""" |
| 440 | cls = ToolParserManager.get_tool_parser("lfm2") |
| 441 | model_output = f"{TOOL_CALL_START} [{SIMPLE_FUNCTION_OUTPUT}]{TOOL_CALL_END}" |
| 442 | |
| 443 | content, tool_calls = run_tool_extraction( |
| 444 | cls(lfm2_tokenizer), model_output, streaming=False |
| 445 | ) |
| 446 | assert len(tool_calls) == 1 |
| 447 | assert tool_calls[0].function == SIMPLE_FUNCTION_CALL |
| 448 | |
| 449 | reconstructor = run_tool_extraction_streaming( |
| 450 | cls(lfm2_tokenizer), [model_output], assert_one_tool_per_delta=False |
| 451 | ) |
| 452 | assert len(reconstructor.tool_calls) == 1 |
| 453 | assert reconstructor.tool_calls[0].function == SIMPLE_FUNCTION_CALL |
| 454 | |
| 455 | |
| 456 | def test_streaming_tool_call_with_large_steps(lfm2_tokenizer: TokenizerLike): |
| 457 | tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer) |
| 458 | model_output_deltas = [ |
| 459 | f"{TOOL_CALL_START}[get_candidate_status(candidate_id='12345'), " |
| 460 | f"{PARAMETERLESS_FUNCTION_OUTPUT}, " |
| 461 | f"{EMPTY_LIST_FUNCTION_OUTPUT}]{TOOL_CALL_END}", |
| 462 | ] |
| 463 | |
| 464 | reconstructor = run_tool_extraction_streaming( |
| 465 | tool_parser, model_output_deltas, assert_one_tool_per_delta=False |
| 466 | ) |
| 467 | |
| 468 | assert len(reconstructor.tool_calls) == 3 |
| 469 | assert reconstructor.tool_calls[0].function == SIMPLE_FUNCTION_CALL |
| 470 | assert reconstructor.tool_calls[1].function == PARAMETERLESS_FUNCTION_CALL |
| 471 | assert reconstructor.tool_calls[2].function == EMPTY_LIST_FUNCTION_CALL |
| 472 | |
| 473 | |
| 474 | def test_streaming_full_block_and_trailing_in_single_delta( |
| 475 | lfm2_tokenizer: TokenizerLike, |
| 476 | ): |
| 477 | """The entire <|tool_call_start|>[...]<|tool_call_end|> block plus |
| 478 | trailing assistant text arrive in one delta. Trailing content must |
| 479 | still be emitted — not silently dropped.""" |
| 480 | tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer) |
| 481 | full_text = f"{TOOL_CALL_START}[{SIMPLE_FUNCTION_OUTPUT}]{TOOL_CALL_END}\nDone." |
| 482 | |
| 483 | reconstructor = run_tool_extraction_streaming(tool_parser, [full_text]) |
| 484 | |
| 485 | assert len(reconstructor.tool_calls) == 1 |
| 486 | assert reconstructor.tool_calls[0].function == SIMPLE_FUNCTION_CALL |
| 487 | assert "Done." in reconstructor.other_content |
| 488 | |
| 489 | |
| 490 | def test_streaming_leading_content_and_full_block_in_single_delta( |
| 491 | lfm2_tokenizer: TokenizerLike, |
| 492 | ): |
| 493 | """Leading assistant text plus the entire tool block arrive in one |
| 494 | delta. Leading content must be emitted — not silently dropped.""" |
| 495 | tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer) |
| 496 | full_text = ( |
| 497 | f"Let me check. {TOOL_CALL_START}[{SIMPLE_FUNCTION_OUTPUT}]{TOOL_CALL_END}" |
| 498 | ) |
| 499 | |
| 500 | reconstructor = run_tool_extraction_streaming(tool_parser, [full_text]) |
| 501 | |
| 502 | assert len(reconstructor.tool_calls) == 1 |
| 503 | assert reconstructor.tool_calls[0].function == SIMPLE_FUNCTION_CALL |
| 504 | assert "Let me check." in reconstructor.other_content |
| 505 | |
| 506 | |
| 507 | def test_streaming_leading_block_and_trailing_in_single_delta( |
| 508 | lfm2_tokenizer: TokenizerLike, |
| 509 | ): |
| 510 | """Leading text + complete tool block + trailing text in one delta. |
| 511 | Both leading and trailing content must be preserved.""" |
| 512 | tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer) |
| 513 | full_text = ( |
| 514 | "Let me check. " |
| 515 | f"{TOOL_CALL_START}[{SIMPLE_FUNCTION_OUTPUT}]{TOOL_CALL_END}\nDone." |
| 516 | ) |
| 517 | |
| 518 | reconstructor = run_tool_extraction_streaming(tool_parser, [full_text]) |
| 519 | |
| 520 | assert len(reconstructor.tool_calls) == 1 |
| 521 | assert reconstructor.tool_calls[0].function == SIMPLE_FUNCTION_CALL |
| 522 | assert "Let me check." in reconstructor.other_content |
| 523 | assert "Done." in reconstructor.other_content |
| 524 | |
| 525 | |
| 526 | def test_echoed_tool_call_body_not_leaked_to_content( |
| 527 | lfm2_tokenizer: TokenizerLike, |
| 528 | ): |
| 529 | """LFM2 sometimes emits the tool call body again after the first |
| 530 | <|tool_call_end|>, capped with a second <|tool_call_end|>. The |
| 531 | echoed body must not surface as assistant content — neither in |
| 532 | streaming nor non-streaming paths.""" |
| 533 | tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer) |
| 534 | body = ( |
| 535 | "[grocery.orderIngredients(" |
| 536 | "ingredientList=[{'name': 'apple', 'quantity': '2'}], " |
| 537 | "deliveryAddress='123 Main St')]" |
| 538 | ) |
| 539 | model_output = f"{TOOL_CALL_START}{body}{TOOL_CALL_END}{body}{TOOL_CALL_END}" |
| 540 | |
| 541 | # Non-streaming |
| 542 | content_ns, tool_calls_ns = run_tool_extraction( |
| 543 | tool_parser, model_output, streaming=False |
| 544 | ) |
| 545 | assert len(tool_calls_ns) == 1 |
| 546 | assert tool_calls_ns[0].function.name == "grocery.orderIngredients" |
| 547 | assert content_ns in (None, "") |
| 548 | |
| 549 | # Streaming: re-fetch a fresh parser since state was mutated above. |
| 550 | tool_parser2: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer) |
| 551 | content_s, tool_calls_s = run_tool_extraction( |
| 552 | tool_parser2, model_output, streaming=True |
| 553 | ) |
| 554 | assert len(tool_calls_s) == 1 |
| 555 | assert tool_calls_s[0].function.name == "grocery.orderIngredients" |
| 556 | # Echoed body must not leak as content. |
| 557 | assert content_s in (None, "") |
| 558 | assert "grocery.orderIngredients" not in (content_s or "") |
| 559 | assert TOOL_CALL_END not in (content_s or "") |
| 560 | |
| 561 | |
| 562 | def test_streaming_char_by_char_multi_dict_list(lfm2_tokenizer: TokenizerLike): |
| 563 | """Stream a tool call containing a list of multiple dicts one |
| 564 | character at a time. Every prefix lands in some partial-parse state |
| 565 | (mid-key, mid-value, open quote inside dict, empty dict, etc.). The |
| 566 | parser must not raise — incomplete prefixes should silently wait for |
| 567 | more text instead of logging exceptions.""" |
| 568 | tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer) |
| 569 | full_text = ( |
| 570 | f"{TOOL_CALL_START}[grocery.orderIngredients(" |
| 571 | "ingredientList=[" |
| 572 | '{"name": "apple", "quantity": "2"}, ' |
| 573 | '{"name": "bread", "quantity": "1"}' |
| 574 | f"])]{TOOL_CALL_END}" |
| 575 | ) |
| 576 | deltas = [c for c in full_text] |
| 577 | |
| 578 | reconstructor = run_tool_extraction_streaming( |
| 579 | tool_parser, deltas, assert_one_tool_per_delta=False |
| 580 | ) |
| 581 | |
| 582 | assert len(reconstructor.tool_calls) == 1 |
| 583 | assert reconstructor.tool_calls[0].function.name == "grocery.orderIngredients" |
| 584 | import json |
| 585 | |
| 586 | args = json.loads(reconstructor.tool_calls[0].function.arguments) |
| 587 | assert args == { |
| 588 | "ingredientList": [ |
| 589 | {"name": "apple", "quantity": "2"}, |
| 590 | {"name": "bread", "quantity": "1"}, |
| 591 | ] |
| 592 | } |
| 593 | |
| 594 | |
| 595 | def test_streaming_dotted_name_in_single_delta(lfm2_tokenizer: TokenizerLike): |
| 596 | """A pythonic call with a dotted/attribute function name (e.g. |
| 597 | ``domain.method(arg=...)``) must be parsed correctly in streaming mode |
| 598 | just as in non-streaming mode.""" |
| 599 | tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer) |
| 600 | full_text = f"{TOOL_CALL_START}[{DOTTED_NAME_FUNCTION_OUTPUT}]{TOOL_CALL_END}" |
| 601 | |
| 602 | reconstructor = run_tool_extraction_streaming(tool_parser, [full_text]) |
| 603 | |
| 604 | assert len(reconstructor.tool_calls) == 1 |
| 605 | assert reconstructor.tool_calls[0].function == DOTTED_NAME_FUNCTION_CALL |
| 606 | |
| 607 | |
| 608 | def test_adjust_request_disables_skip_special_tokens( |
| 609 | lfm2_tokenizer: TokenizerLike, |
| 610 | ): |
| 611 | """When tools are present, the parser must force |
| 612 | ``skip_special_tokens=False`` so the engine does not strip the |
| 613 | <|tool_call_start|>/<|tool_call_end|> sentinels before they reach the |
| 614 | parser.""" |
| 615 | from vllm.entrypoints.openai.chat_completion.protocol import ( |
| 616 | ChatCompletionRequest, |
| 617 | ) |
| 618 | |
| 619 | tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer) |
| 620 | |
| 621 | request_with_tools = ChatCompletionRequest( |
| 622 | messages=[], |
| 623 | model="test-model", |
| 624 | tools=[ |
| 625 | { |
| 626 | "type": "function", |
| 627 | "function": { |
| 628 | "name": "get_weather", |
| 629 | "parameters": { |
| 630 | "type": "object", |
| 631 | "properties": {"city": {"type": "string"}}, |
| 632 | }, |
| 633 | }, |
| 634 | } |
| 635 | ], |
| 636 | ) |
| 637 | assert request_with_tools.skip_special_tokens is True |
| 638 | adjusted = tool_parser.adjust_request(request_with_tools) |
| 639 | assert adjusted.skip_special_tokens is False |
| 640 | |
| 641 | # No tools → no override; default behaviour preserved. |
| 642 | request_no_tools = ChatCompletionRequest(messages=[], model="test-model") |
| 643 | assert request_no_tools.skip_special_tokens is True |
| 644 | adjusted_no_tools = tool_parser.adjust_request(request_no_tools) |
| 645 | assert adjusted_no_tools.skip_special_tokens is True |
| 646 | |
| 647 | |
| 648 | @pytest.mark.parametrize("streaming", [False]) |
| 649 | def test_regex_timeout_handling(streaming: bool, lfm2_tokenizer: TokenizerLike): |
| 650 | """Test regex timeout is handled gracefully.""" |
| 651 | tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer) |
| 652 | |
| 653 | fake_input = f"{TOOL_CALL_START}[A(A=" + "\t)A(A=,\t" * 2 |
| 654 | fake_input += f"]{TOOL_CALL_END}" |
| 655 | |
| 656 | mock_regex = MagicMock() |
| 657 | mock_regex.match.side_effect = TimeoutError("Regex timeout") |
| 658 | |
| 659 | with patch.object(tool_parser, "TOOL_CALL_REGEX", mock_regex): |
| 660 | content, tool_calls = run_tool_extraction( |
| 661 | tool_parser, fake_input, streaming=streaming |
| 662 | ) |
| 663 | |
| 664 | assert content == fake_input |
| 665 | assert len(tool_calls) == 0 |
| 666 | mock_regex.match.assert_called_once() |