| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Tests for the engine-based Inkling parser. |
| 4 | |
| 5 | Inkling output is a sequence of typed content blocks delimited by dedicated |
| 6 | special tokens; the tool-call payload is ``{"name":...,"args":{...}}`` |
| 7 | between ``<|content_invoke_tool_json|>`` and ``<|end_message|>``. The |
| 8 | cases mirror the Rust unified parser's tests |
| 9 | (``rust/src/parser/src/unified/inkling.rs``) where applicable. |
| 10 | """ |
| 11 | |
| 12 | import json |
| 13 | |
| 14 | import pytest |
| 15 | |
| 16 | from tests.parser.engine.conftest import make_mock_tokenizer |
| 17 | from tests.parser.engine.streaming_helpers import ( |
| 18 | collect_content, |
| 19 | collect_function_name, |
| 20 | collect_tool_arguments, |
| 21 | ) |
| 22 | from vllm.entrypoints.openai.chat_completion.protocol import ( |
| 23 | ChatCompletionToolsParam, |
| 24 | FunctionDefinition, |
| 25 | ) |
| 26 | from vllm.parser.abstract_parser import DelegatingParser |
| 27 | from vllm.parser.engine.events import EventType |
| 28 | from vllm.parser.engine.parser_engine_config import ParserState |
| 29 | from vllm.parser.engine.registered_adapters import ( |
| 30 | InklingParserReasoningAdapter, |
| 31 | InklingParserToolAdapter, |
| 32 | ) |
| 33 | from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine |
| 34 | from vllm.parser.inkling import InklingParser, _inkling_arg_converter, inkling_config |
| 35 | from vllm.parser.parser_manager import ParserManager |
| 36 | |
| 37 | MSG_MODEL = "<|message_model|>" |
| 38 | TEXT_START = "<|content_text|>" |
| 39 | THINK_START = "<|content_thinking|>" |
| 40 | TOOL_JSON = "<|content_invoke_tool_json|>" |
| 41 | TOOL_TEXT = "<|content_invoke_tool_text|>" |
| 42 | TOOL_ERROR = "<|content_tool_error|>" |
| 43 | END_MESSAGE = "<|end_message|>" |
| 44 | END_SAMPLING = "<|content_model_end_sampling|>" |
| 45 | |
| 46 | _TML_VOCAB = { |
| 47 | MSG_MODEL: 200001, |
| 48 | TEXT_START: 200004, |
| 49 | END_SAMPLING: 200006, |
| 50 | THINK_START: 200008, |
| 51 | END_MESSAGE: 200010, |
| 52 | TOOL_ERROR: 200022, |
| 53 | TOOL_JSON: 200049, |
| 54 | TOOL_TEXT: 200057, |
| 55 | } |
| 56 | |
| 57 | |
| 58 | @pytest.fixture |
| 59 | def mock_tokenizer(): |
| 60 | return make_mock_tokenizer(_TML_VOCAB) |
| 61 | |
| 62 | |
| 63 | @pytest.fixture |
| 64 | def parser(mock_tokenizer): |
| 65 | return InklingParser(mock_tokenizer) |
| 66 | |
| 67 | |
| 68 | def _tool_block(name: str, args: str) -> str: |
| 69 | return f'{TOOL_JSON}{{"name":"{name}","args":{args}}}{END_MESSAGE}' |
| 70 | |
| 71 | |
| 72 | _MARKERS = sorted(_TML_VOCAB, key=len, reverse=True) |
| 73 | |
| 74 | |
| 75 | def _tokenize(text: str) -> list[tuple[int, str]]: |
| 76 | """Tokenize like the real stream: markers are atomic special tokens, |
| 77 | plain text becomes one token per character (matching the mock |
| 78 | tokenizer's ``chr``-based decode).""" |
| 79 | tokens: list[tuple[int, str]] = [] |
| 80 | i = 0 |
| 81 | while i < len(text): |
| 82 | for marker in _MARKERS: |
| 83 | if text.startswith(marker, i): |
| 84 | tokens.append((_TML_VOCAB[marker], marker)) |
| 85 | i += len(marker) |
| 86 | break |
| 87 | else: |
| 88 | tokens.append((ord(text[i]), text[i])) |
| 89 | i += 1 |
| 90 | return tokens |
| 91 | |
| 92 | |
| 93 | def _stream(parser, request, text: str, chunk_size: int): |
| 94 | """Stream production-shaped deltas: ``chunk_size`` tokens per delta, |
| 95 | with delta_token_ids covering every token (specials and text).""" |
| 96 | tokens = _tokenize(text) |
| 97 | results = [] |
| 98 | previous_text = "" |
| 99 | previous_token_ids: list[int] = [] |
| 100 | for start in range(0, len(tokens), chunk_size): |
| 101 | batch = tokens[start : start + chunk_size] |
| 102 | delta_text = "".join(t for _, t in batch) |
| 103 | delta_token_ids = [tid for tid, _ in batch] |
| 104 | current_text = previous_text + delta_text |
| 105 | current_token_ids = previous_token_ids + delta_token_ids |
| 106 | delta = parser.extract_tool_calls_streaming( |
| 107 | previous_text=previous_text, |
| 108 | current_text=current_text, |
| 109 | delta_text=delta_text, |
| 110 | previous_token_ids=tuple(previous_token_ids), |
| 111 | current_token_ids=tuple(current_token_ids), |
| 112 | delta_token_ids=tuple(delta_token_ids), |
| 113 | request=request, |
| 114 | ) |
| 115 | results.append((delta, current_text)) |
| 116 | previous_text = current_text |
| 117 | previous_token_ids = current_token_ids |
| 118 | finish = parser.finish_streaming() |
| 119 | if finish is not None: |
| 120 | results.append((finish, text)) |
| 121 | return results |
| 122 | |
| 123 | |
| 124 | def _stream_text_only(parser, request, text: str, chunk_size: int): |
| 125 | """Stream text-only deltas (no token ids), chunked at arbitrary |
| 126 | character boundaries — exercises the text-lexing fallback path, |
| 127 | including markers split across chunks.""" |
| 128 | results = [] |
| 129 | previous_text = "" |
| 130 | for start in range(0, len(text), chunk_size): |
| 131 | delta_text = text[start : start + chunk_size] |
| 132 | current_text = previous_text + delta_text |
| 133 | delta = parser.extract_tool_calls_streaming( |
| 134 | previous_text=previous_text, |
| 135 | current_text=current_text, |
| 136 | delta_text=delta_text, |
| 137 | previous_token_ids=(), |
| 138 | current_token_ids=(), |
| 139 | delta_token_ids=(), |
| 140 | request=request, |
| 141 | ) |
| 142 | results.append((delta, current_text)) |
| 143 | previous_text = current_text |
| 144 | finish = parser.finish_streaming() |
| 145 | if finish is not None: |
| 146 | results.append((finish, text)) |
| 147 | return results |
| 148 | |
| 149 | |
| 150 | def _collect_reasoning(results) -> str: |
| 151 | return "".join(d.reasoning for d, _ in results if d and d.reasoning) |
| 152 | |
| 153 | |
| 154 | def _function_tool(name: str = "get_weather") -> ChatCompletionToolsParam: |
| 155 | """A real function-tool definition, as a request would carry it.""" |
| 156 | return ChatCompletionToolsParam( |
| 157 | function=FunctionDefinition( |
| 158 | name=name, |
| 159 | parameters={"type": "object", "properties": {}}, |
| 160 | ), |
| 161 | ) |
| 162 | |
| 163 | |
| 164 | def _delegating(mock_tokenizer, tools=None): |
| 165 | """Build the served reasoning+tool DelegatingParser for Inkling.""" |
| 166 | parser_cls = ParserManager.get_parser( |
| 167 | tool_parser_name="inkling", |
| 168 | reasoning_parser_name="inkling", |
| 169 | enable_auto_tools=True, |
| 170 | ) |
| 171 | return parser_cls(mock_tokenizer, tools or []) |
| 172 | |
| 173 | |
| 174 | class _TwoPassInklingParser(DelegatingParser): |
| 175 | reasoning_parser_cls = InklingParserReasoningAdapter |
| 176 | tool_parser_cls = InklingParserToolAdapter |
| 177 | |
| 178 | |
| 179 | def _stream_delegating(parser, request, text, chunk_size, prompt_token_ids): |
| 180 | """Stream ``text`` through ``DelegatingParser.parse_delta``, ``chunk_size`` |
| 181 | tokens per delta; return ``(content, reasoning, ordered tool names, |
| 182 | ordered tool arguments)``. Arguments arrive in fragments, so they are |
| 183 | concatenated per tool index.""" |
| 184 | tokens = _tokenize(text) |
| 185 | content, reasoning = "", "" |
| 186 | tools: dict[int, str] = {} |
| 187 | args: dict[int, str] = {} |
| 188 | for start in range(0, len(tokens), chunk_size): |
| 189 | batch = tokens[start : start + chunk_size] |
| 190 | delta = parser.parse_delta( |
| 191 | "".join(t for _, t in batch), |
| 192 | [tid for tid, _ in batch], |
| 193 | request, |
| 194 | prompt_token_ids=prompt_token_ids if start == 0 else None, |
| 195 | finished=(start + chunk_size >= len(tokens)), |
| 196 | ) |
| 197 | if delta and delta.content: |
| 198 | content += delta.content |
| 199 | if delta and delta.reasoning: |
| 200 | reasoning += delta.reasoning |
| 201 | if delta and delta.tool_calls: |
| 202 | for tc in delta.tool_calls: |
| 203 | if tc.function and tc.function.name: |
| 204 | tools[tc.index] = tc.function.name |
| 205 | if tc.function and tc.function.arguments: |
| 206 | args[tc.index] = args.get(tc.index, "") + tc.function.arguments |
| 207 | return ( |
| 208 | content, |
| 209 | reasoning, |
| 210 | [tools[k] for k in sorted(tools)], |
| 211 | [args.get(k, "") for k in sorted(tools)], |
| 212 | ) |
| 213 | |
| 214 | |
| 215 | class TestArgConverter: |
| 216 | def test_complete_wrapper(self): |
| 217 | raw = '{"name":"get_weather","args":{"city":"SF"}}' |
| 218 | assert _inkling_arg_converter(raw, False) == '{"city":"SF"}' |
| 219 | |
| 220 | def test_partial_before_args(self): |
| 221 | assert _inkling_arg_converter('{"name":"get_w', True) == "" |
| 222 | |
| 223 | def test_partial_inside_args(self): |
| 224 | raw = '{"name":"x","args":{"a":1' |
| 225 | assert _inkling_arg_converter(raw, True) == '{"a":1' |
| 226 | |
| 227 | def test_prefix_stability(self): |
| 228 | full = '{"name":"x","args":{"a":{"b":[1,2]},"c":"d"}}' |
| 229 | prev = "" |
| 230 | for end in range(len(full)): |
| 231 | out = _inkling_arg_converter(full[:end], True) |
| 232 | assert out.startswith(prev) or prev.startswith(out) or not prev |
| 233 | if out.startswith(prev): |
| 234 | prev = out |
| 235 | |
| 236 | def test_args_value_appearing_in_name(self): |
| 237 | raw = '{"name":"args","args":{"k":1}}' |
| 238 | assert _inkling_arg_converter(raw, False) == '{"k":1}' |
| 239 | |
| 240 | def test_whitespace_tolerated(self): |
| 241 | raw = '{ "name" : "x" , "args" : {"a": 1} }' |
| 242 | assert _inkling_arg_converter(raw, False) == '{"a": 1}' |
| 243 | |
| 244 | def test_missing_args_defaults_empty(self): |
| 245 | assert _inkling_arg_converter('{"name":"x"}', False) == "{}" |
| 246 | |
| 247 | def test_non_object_args_rejected(self): |
| 248 | with pytest.raises(ValueError, match="JSON object"): |
| 249 | _inkling_arg_converter('{"name":"x","args":[1]}', False) |
| 250 | |
| 251 | |
| 252 | class TestNonStreaming: |
| 253 | @pytest.mark.parametrize("suffix", ["", END_MESSAGE, END_SAMPLING]) |
| 254 | def test_bare_text_after_model_opener(self, parser, mock_request, suffix): |
| 255 | reasoning, content, tools = parser.parse(f"hello world{suffix}", mock_request) |
| 256 | assert reasoning is None |
| 257 | assert content == "hello world" |
| 258 | assert tools is None |
| 259 | |
| 260 | def test_plain_text(self, parser, mock_request): |
| 261 | reasoning, content, tools = parser.parse( |
| 262 | f"{TEXT_START}hello world{END_MESSAGE}", mock_request |
| 263 | ) |
| 264 | assert reasoning is None |
| 265 | assert content == "hello world" |
| 266 | assert tools is None |
| 267 | |
| 268 | def test_reasoning_text_tool(self, parser, mock_request): |
| 269 | text = ( |
| 270 | f"{THINK_START}I should check the weather.{END_MESSAGE}" |
| 271 | f"{MSG_MODEL}{TEXT_START}Let me check.{END_MESSAGE}" |
| 272 | f"{MSG_MODEL}" + _tool_block("get_weather", '{"city":"SF"}') |
| 273 | ) |
| 274 | reasoning, content, tools = parser.parse(text, mock_request) |
| 275 | assert reasoning == "I should check the weather." |
| 276 | assert content == "Let me check." |
| 277 | assert [t.name for t in tools] == ["get_weather"] |
| 278 | assert json.loads(tools[0].arguments) == {"city": "SF"} |
| 279 | |
| 280 | def test_tool_header_name_is_not_visible_content(self, parser, mock_request): |
| 281 | text = "get_weather" + _tool_block("get_weather", '{"city":"SF"}') |
| 282 | _, content, tools = parser.parse(text, mock_request) |
| 283 | assert content is None |
| 284 | assert [tool.name for tool in tools] == ["get_weather"] |
| 285 | |
| 286 | def test_parallel_tool_calls(self, parser, mock_request): |
| 287 | text = _tool_block("a", "{}") + MSG_MODEL + _tool_block("b", '{"x":[1,2]}') |
| 288 | _, _, tools = parser.parse(text, mock_request) |
| 289 | assert [t.name for t in tools] == ["a", "b"] |
| 290 | assert json.loads(tools[0].arguments) == {} |
| 291 | assert json.loads(tools[1].arguments) == {"x": [1, 2]} |
| 292 | |
| 293 | def test_nested_args(self, parser, mock_request): |
| 294 | args = '{"q":{"deep":{"list":[{"k":"v"}]}},"s":"a}b"}' |
| 295 | _, _, tools = parser.parse(_tool_block("f", args), mock_request) |
| 296 | assert json.loads(tools[0].arguments) == json.loads(args) |
| 297 | |
| 298 | def test_invoke_tool_text_is_visible_text(self, parser, mock_request): |
| 299 | reasoning, content, tools = parser.parse( |
| 300 | f"{TOOL_TEXT}do something{END_MESSAGE}", mock_request |
| 301 | ) |
| 302 | assert content == "do something" |
| 303 | assert tools is None |
| 304 | |
| 305 | def test_tool_error_is_visible_text(self, parser, mock_request): |
| 306 | _, content, tools = parser.parse(f"{TOOL_ERROR}boom{END_MESSAGE}", mock_request) |
| 307 | assert content == "boom" |
| 308 | assert tools is None |
| 309 | |
| 310 | def test_end_sampling_closes_blocks(self, parser, mock_request): |
| 311 | reasoning, content, _ = parser.parse( |
| 312 | f"{THINK_START}hm{END_MESSAGE}{MSG_MODEL}{TEXT_START}hi{END_SAMPLING}", |
| 313 | mock_request, |
| 314 | ) |
| 315 | assert reasoning == "hm" |
| 316 | assert content == "hi" |
| 317 | |
| 318 | def test_multiple_reasoning_blocks_concatenate(self, parser, mock_request): |
| 319 | text = ( |
| 320 | f"{THINK_START}one{END_MESSAGE}" |
| 321 | f"{MSG_MODEL}{TEXT_START}mid{END_MESSAGE}" |
| 322 | f"{MSG_MODEL}{THINK_START}two{END_MESSAGE}" |
| 323 | ) |
| 324 | reasoning, content, _ = parser.parse(text, mock_request) |
| 325 | assert reasoning == "onetwo" |
| 326 | assert content == "mid" |
| 327 | |
| 328 | def test_text_after_tool_call(self, parser, mock_request): |
| 329 | text = _tool_block("f", "{}") + f"{MSG_MODEL}{TEXT_START}done{END_MESSAGE}" |
| 330 | _, content, tools = parser.parse(text, mock_request) |
| 331 | assert [t.name for t in tools] == ["f"] |
| 332 | assert content == "done" |
| 333 | |
| 334 | def test_incomplete_tool_call_at_eos(self, parser, mock_request): |
| 335 | # Engine convention: best-effort with what arrived. (The Rust |
| 336 | # parser instead errors with "incomplete Inkling tool call".) |
| 337 | _, _, tools = parser.parse( |
| 338 | f'{TOOL_JSON}{{"name":"d","args":{{"k":"v"', mock_request |
| 339 | ) |
| 340 | assert [t.name for t in tools] == ["d"] |
| 341 | |
| 342 | def test_prose_marker_without_token_ids_is_structural(self, parser, mock_request): |
| 343 | # Inkling opts into text-lexer terminal recognition so held-back |
| 344 | # structural marker text from the detokenizer is still parsed. |
| 345 | _, content, _ = parser.parse( |
| 346 | f"{TEXT_START}see {TEXT_START} token{END_MESSAGE}", mock_request |
| 347 | ) |
| 348 | assert content == "see token" |
| 349 | |
| 350 | |
| 351 | class TestStreaming: |
| 352 | @pytest.mark.parametrize("chunk_size", [1, 3, 7, 64, 4096]) |
| 353 | def test_chunk_invariance_tool_call(self, mock_tokenizer, mock_request, chunk_size): |
| 354 | parser = InklingParser(mock_tokenizer) |
| 355 | text = f"{TEXT_START}Check this.{END_MESSAGE}{MSG_MODEL}" + _tool_block( |
| 356 | "get_weather", '{"city":"San Francisco"}' |
| 357 | ) |
| 358 | results = _stream(parser, mock_request, text, chunk_size) |
| 359 | assert collect_content(results) == "Check this." |
| 360 | assert collect_function_name(results) == "get_weather" |
| 361 | assert json.loads(collect_tool_arguments(results)) == {"city": "San Francisco"} |
| 362 | |
| 363 | @pytest.mark.parametrize("chunk_size", [1, 3, 7, 64]) |
| 364 | def test_chunk_invariance_tool_call_text_only( |
| 365 | self, mock_tokenizer, mock_request, chunk_size |
| 366 | ): |
| 367 | # Same case through the text-lexing fallback (no token ids), |
| 368 | # with markers split at arbitrary character boundaries. |
| 369 | parser = InklingParser(mock_tokenizer) |
| 370 | text = f"{TEXT_START}Check this.{END_MESSAGE}{MSG_MODEL}" + _tool_block( |
| 371 | "get_weather", '{"city":"San Francisco"}' |
| 372 | ) |
| 373 | results = _stream_text_only(parser, mock_request, text, chunk_size) |
| 374 | assert collect_content(results) == "Check this." |
| 375 | assert collect_function_name(results) == "get_weather" |
| 376 | assert json.loads(collect_tool_arguments(results)) == {"city": "San Francisco"} |
| 377 | |
| 378 | @pytest.mark.parametrize("chunk_size", [1, 5, 11]) |
| 379 | def test_chunk_invariance_reasoning(self, mock_tokenizer, mock_request, chunk_size): |
| 380 | parser = InklingParser(mock_tokenizer) |
| 381 | text = ( |
| 382 | f"{THINK_START}thinking...{END_MESSAGE}" |
| 383 | f"{MSG_MODEL}{TEXT_START}answer{END_MESSAGE}" |
| 384 | ) |
| 385 | results = _stream(parser, mock_request, text, chunk_size) |
| 386 | assert _collect_reasoning(results) == "thinking..." |
| 387 | assert collect_content(results) == "answer" |
| 388 | |
| 389 | def test_split_marker_held_across_chunks(self, parser, mock_request): |
| 390 | # Mirrors Rust `inkling_streaming_holds_split_markers`. |
| 391 | text = f"{TEXT_START}hello{END_MESSAGE}" |
| 392 | results = _stream_text_only(parser, mock_request, text, 9) |
| 393 | assert collect_content(results) == "hello" |
| 394 | |
| 395 | def test_name_streams_before_args_complete(self, parser, mock_request): |
| 396 | # Feed only up to the name's closing quote — the name delta must |
| 397 | # already be emitted before any args arrive. |
| 398 | prefix = f'{TOOL_JSON}{{"name":"get_weather",' |
| 399 | results = _stream(parser, mock_request, prefix, 4096) |
| 400 | assert collect_function_name(results) == "get_weather" |
| 401 | |
| 402 | def test_combined_parser_reasoning_to_tool_handoff_uses_text_markers( |
| 403 | self, mock_tokenizer, mock_request |
| 404 | ): |
| 405 | parser_cls = ParserManager.get_parser( |
| 406 | tool_parser_name="inkling", |
| 407 | reasoning_parser_name="inkling", |
| 408 | enable_auto_tools=True, |
| 409 | ) |
| 410 | parser = parser_cls(mock_tokenizer, []) |
| 411 | |
| 412 | first = parser.parse_delta( |
| 413 | THINK_START, |
| 414 | [_TML_VOCAB[THINK_START]], |
| 415 | mock_request, |
| 416 | prompt_token_ids=[_TML_VOCAB[MSG_MODEL]], |
| 417 | finished=False, |
| 418 | ) |
| 419 | assert first is None |
| 420 | |
| 421 | second = parser.parse_delta( |
| 422 | "thinking", |
| 423 | [ord(c) for c in "thinking"], |
| 424 | mock_request, |
| 425 | finished=False, |
| 426 | ) |
| 427 | assert second is not None |
| 428 | assert second.reasoning == "thinking" |
| 429 | |
| 430 | # Mirrors the DelegatingParser handoff after reasoning closes: the |
| 431 | # tool pass receives reconstructed text that starts at the Inkling |
| 432 | # tool marker, while the token-id slice has already moved past it. |
| 433 | body = ( |
| 434 | "get_weather" |
| 435 | f'{TOOL_JSON}{{"name":"get_weather","args":{{"city":"Seattle"}}}}' |
| 436 | f"{END_MESSAGE}" |
| 437 | ) |
| 438 | third = parser.parse_delta( |
| 439 | body, |
| 440 | [_TML_VOCAB[END_MESSAGE], _TML_VOCAB[END_SAMPLING]], |
| 441 | mock_request, |
| 442 | finished=True, |
| 443 | ) |
| 444 | assert third is not None |
| 445 | assert third.tool_calls |
| 446 | assert third.tool_calls[0].function.name == "get_weather" |
| 447 | assert third.tool_calls[0].function.arguments == '{"city":"Seattle"}' |
| 448 | assert TOOL_JSON not in ((third.content or "") + (third.reasoning or "")) |
| 449 | |
| 450 | def test_streamed_args_are_object_only(self, parser, mock_request): |
| 451 | # The streamed `arguments` must be the bare args object, never |
| 452 | # the `{"name":...}` wrapper. |
| 453 | text = _tool_block("f", '{"a":1}') |
| 454 | results = _stream(parser, mock_request, text, 3) |
| 455 | args = collect_tool_arguments(results) |
| 456 | assert json.loads(args) == {"a": 1} |
| 457 | assert "name" not in args |
| 458 | |
| 459 | @pytest.mark.parametrize("chunk_size", [1, 9]) |
| 460 | def test_parallel_calls_streaming(self, mock_tokenizer, mock_request, chunk_size): |
| 461 | parser = InklingParser(mock_tokenizer) |
| 462 | text = _tool_block("a", '{"i":1}') + MSG_MODEL + _tool_block("b", '{"i":2}') |
| 463 | results = _stream(parser, mock_request, text, chunk_size) |
| 464 | indexed: dict[int, dict[str, str]] = {} |
| 465 | for delta, _ in results: |
| 466 | if not (delta and delta.tool_calls): |
| 467 | continue |
| 468 | for tc in delta.tool_calls: |
| 469 | slot = indexed.setdefault(tc.index, {"name": "", "args": ""}) |
| 470 | if tc.function and tc.function.name: |
| 471 | slot["name"] = tc.function.name |
| 472 | if tc.function and tc.function.arguments: |
| 473 | slot["args"] += tc.function.arguments |
| 474 | assert indexed[0]["name"] == "a" |
| 475 | assert indexed[1]["name"] == "b" |
| 476 | assert json.loads(indexed[0]["args"]) == {"i": 1} |
| 477 | assert json.loads(indexed[1]["args"]) == {"i": 2} |
| 478 | |
| 479 | |
| 480 | class TestPromptSeededState: |
| 481 | def test_prompt_ending_in_thinking_starts_reasoning(self, parser, mock_request): |
| 482 | parser.adjust_initial_state_from_prompt([200001, _TML_VOCAB[THINK_START]]) |
| 483 | assert parser._engine.state == ParserState.REASONING |
| 484 | |
| 485 | def test_prompt_ending_in_text_starts_content(self, parser): |
| 486 | parser.adjust_initial_state_from_prompt([200001, _TML_VOCAB[TEXT_START]]) |
| 487 | assert parser._engine.state == ParserState.CONTENT |
| 488 | |
| 489 | def test_generation_prompt_tail_starts_message_header(self, parser): |
| 490 | parser.adjust_initial_state_from_prompt( |
| 491 | [_TML_VOCAB[END_MESSAGE], _TML_VOCAB[MSG_MODEL]] |
| 492 | ) |
| 493 | assert parser._engine.state == ParserState.MESSAGE_HEADER |
| 494 | |
| 495 | def test_generation_prompt_header_hides_tool_name(self, parser, mock_request): |
| 496 | text = "get_weather" + _tool_block("get_weather", '{"city":"SF"}') |
| 497 | delta = parser.parse_delta( |
| 498 | text, |
| 499 | [token_id for token_id, _ in _tokenize(text)], |
| 500 | mock_request, |
| 501 | prompt_token_ids=[_TML_VOCAB[END_MESSAGE], _TML_VOCAB[MSG_MODEL]], |
| 502 | finished=True, |
| 503 | ) |
| 504 | assert delta is not None |
| 505 | assert delta.content is None |
| 506 | assert delta.tool_calls[0].function.name == "get_weather" |
| 507 | |
| 508 | def test_generation_prompt_header_flushes_bare_text_at_finish( |
| 509 | self, parser, mock_request |
| 510 | ): |
| 511 | prompt_token_ids = [_TML_VOCAB[END_MESSAGE], _TML_VOCAB[MSG_MODEL]] |
| 512 | first = parser.parse_delta( |
| 513 | "plain ", |
| 514 | [ord(char) for char in "plain "], |
| 515 | mock_request, |
| 516 | prompt_token_ids=prompt_token_ids, |
| 517 | finished=False, |
| 518 | ) |
| 519 | assert first is None |
| 520 | |
| 521 | second = parser.parse_delta( |
| 522 | "answer", |
| 523 | [ord(char) for char in "answer"], |
| 524 | mock_request, |
| 525 | finished=True, |
| 526 | ) |
| 527 | assert second is not None |
| 528 | assert second.content == "plain answer" |
| 529 | |
| 530 | |
| 531 | class TestToolCallFiltering: |
| 532 | """Inkling equivalents of the generic tool-call-filtering replay tests |
| 533 | (Inkling is excluded from those in test_replay.py: its structural |
| 534 | role/kind tokens and shared block-end token don't fit the generic |
| 535 | reasoning/tool split model).""" |
| 536 | |
| 537 | def test_skip_tool_parsing_round_trip(self, mock_tokenizer, mock_request): |
| 538 | # First pass (reasoning adapter, skip_tool_parsing): reasoning is |
| 539 | # classified as reasoning while tool markup survives in content; |
| 540 | # second pass (tool adapter) re-extracts the calls from it. |
| 541 | text = ( |
| 542 | f"{THINK_START}plan{END_MESSAGE}{MSG_MODEL}" |
| 543 | + _tool_block("f", '{"a":1}') |
| 544 | + MSG_MODEL |
| 545 | + _tool_block("g", '{"b":[2]}') |
| 546 | ) |
| 547 | first = InklingParser(mock_tokenizer) |
| 548 | first.skip_tool_parsing = True |
| 549 | reasoning, content = first.extract_reasoning(text, mock_request) |
| 550 | assert reasoning == "plan" |
| 551 | assert content.count(TOOL_JSON) == 2 |
| 552 | |
| 553 | second = InklingParser(mock_tokenizer) |
| 554 | result = second.extract_tool_calls_from_content(content, mock_request) |
| 555 | assert result.tools_called |
| 556 | assert [tc.function.name for tc in result.tool_calls] == ["f", "g"] |
| 557 | assert json.loads(result.tool_calls[0].function.arguments) == {"a": 1} |
| 558 | assert json.loads(result.tool_calls[1].function.arguments) == {"b": [2]} |
| 559 | |
| 560 | @pytest.fixture |
| 561 | def none_request(self, mock_request): |
| 562 | mock_request.tools = [{"type": "function", "function": {"name": "f"}}] |
| 563 | mock_request.tool_choice = "none" |
| 564 | return mock_request |
| 565 | |
| 566 | def test_tool_choice_none_non_streaming(self, mock_tokenizer, none_request): |
| 567 | parser = InklingParser(mock_tokenizer) |
| 568 | text = ( |
| 569 | f"{THINK_START}plan{END_MESSAGE}" |
| 570 | f"{MSG_MODEL}{TEXT_START}visible{END_MESSAGE}" |
| 571 | f"{MSG_MODEL}" + _tool_block("f", '{"a":1}') |
| 572 | ) |
| 573 | reasoning, content, tools = parser.parse(text, none_request) |
| 574 | assert reasoning == "plan" |
| 575 | assert content == "visible" |
| 576 | assert not tools |
| 577 | |
| 578 | def test_tool_choice_none_streaming(self, mock_tokenizer, none_request): |
| 579 | parser = InklingParser(mock_tokenizer) |
| 580 | text = f"{TEXT_START}visible{END_MESSAGE}{MSG_MODEL}" + _tool_block( |
| 581 | "f", '{"a":1}' |
| 582 | ) |
| 583 | results = _stream(parser, none_request, text, 3) |
| 584 | assert collect_content(results) == "visible" |
| 585 | assert all(not (d and d.tool_calls) for d, _ in results) |
| 586 | |
| 587 | |
| 588 | class TestRegisteredAdapters: |
| 589 | def test_adapters_resolve(self): |
| 590 | from vllm.reasoning import ReasoningParserManager |
| 591 | from vllm.tool_parsers import ToolParserManager |
| 592 | |
| 593 | reasoning_cls = ReasoningParserManager.get_reasoning_parser("inkling") |
| 594 | tool_cls = ToolParserManager.get_tool_parser("inkling") |
| 595 | assert reasoning_cls._parser_engine_cls is InklingParser |
| 596 | assert tool_cls._parser_engine_cls is InklingParser |
| 597 | assert tool_cls.supports_required_and_named is False |
| 598 | |
| 599 | def test_adapter_round_trip(self, mock_tokenizer, mock_request): |
| 600 | from vllm.tool_parsers import ToolParserManager |
| 601 | |
| 602 | tool_cls = ToolParserManager.get_tool_parser("inkling") |
| 603 | adapter = tool_cls(mock_tokenizer) |
| 604 | result = adapter.extract_tool_calls(_tool_block("f", '{"a":1}'), mock_request) |
| 605 | assert result.tools_called |
| 606 | assert result.tool_calls[0].function.name == "f" |
| 607 | assert json.loads(result.tool_calls[0].function.arguments) == {"a": 1} |
| 608 | |
| 609 | |
| 610 | class TestDelegatingTwoPass: |
| 611 | """Served reasoning+tool ``DelegatingParser`` path (issue #51387). |
| 612 | |
| 613 | Unlike ``TestStreaming`` above (single engine, ``skip_tool_parsing=False``), |
| 614 | these drive the real two-pass parser, where the reasoning pass runs with |
| 615 | ``skip_tool_parsing=True`` — the path where the trailing-marker leak lived. |
| 616 | |
| 617 | The bug is specific to the *tools-enabled* plain-text/reasoning path, so the |
| 618 | content-only cases (which emit no tool call and would still pass in a |
| 619 | no-tools mode) offer a real function tool on both the request and the |
| 620 | parser, keeping ``tool_choice="auto"``. |
| 621 | """ |
| 622 | |
| 623 | GEN_PROMPT = [_TML_VOCAB[MSG_MODEL]] |
| 624 | |
| 625 | def test_plain_text_non_streaming(self, mock_tokenizer, mock_request): |
| 626 | tools = [_function_tool()] |
| 627 | mock_request.tools = tools |
| 628 | _, content, calls = _delegating(mock_tokenizer, tools).parse( |
| 629 | f"{TEXT_START}The answer is 42.{END_MESSAGE}", |
| 630 | mock_request, |
| 631 | enable_auto_tools=True, |
| 632 | ) |
| 633 | assert content == "The answer is 42." |
| 634 | assert END_MESSAGE not in content |
| 635 | assert not calls |
| 636 | |
| 637 | @pytest.mark.parametrize("chunk_size", [1, 3, 7, 64]) |
| 638 | def test_plain_text_streaming(self, mock_tokenizer, mock_request, chunk_size): |
| 639 | tools = [_function_tool()] |
| 640 | mock_request.tools = tools |
| 641 | content, _, calls, _ = _stream_delegating( |
| 642 | _delegating(mock_tokenizer, tools), |
| 643 | mock_request, |
| 644 | f"{TEXT_START}The answer is 42.{END_MESSAGE}", |
| 645 | chunk_size, |
| 646 | self.GEN_PROMPT, |
| 647 | ) |
| 648 | assert content == "The answer is 42." |
| 649 | assert END_MESSAGE not in content |
| 650 | assert not calls |
| 651 | |
| 652 | def test_reasoning_then_text(self, mock_tokenizer, mock_request): |
| 653 | tools = [_function_tool()] |
| 654 | mock_request.tools = tools |
| 655 | reasoning, content, _ = _delegating(mock_tokenizer, tools).parse( |
| 656 | f"{THINK_START}plan{END_MESSAGE}" |
| 657 | f"{MSG_MODEL}{TEXT_START}Let me look.{END_MESSAGE}", |
| 658 | mock_request, |
| 659 | enable_auto_tools=True, |
| 660 | ) |
| 661 | assert reasoning == "plan" |
| 662 | assert content == "Let me look." |
| 663 | assert END_MESSAGE not in content |
| 664 | |
| 665 | def test_multi_tool_round_trip(self, mock_tokenizer, mock_request): |
| 666 | _, _, tools = _delegating(mock_tokenizer).parse( |
| 667 | _tool_block("f", '{"a":1}') + MSG_MODEL + _tool_block("g", '{"b":2}'), |
| 668 | mock_request, |
| 669 | enable_auto_tools=True, |
| 670 | ) |
| 671 | assert [t.name for t in tools] == ["f", "g"] |
| 672 | |
| 673 | def test_text_then_tool(self, mock_tokenizer, mock_request): |
| 674 | _, content, tools = _delegating(mock_tokenizer).parse( |
| 675 | f"{TEXT_START}intro{END_MESSAGE}{MSG_MODEL}" + _tool_block("f", "{}"), |
| 676 | mock_request, |
| 677 | enable_auto_tools=True, |
| 678 | ) |
| 679 | assert content == "intro" |
| 680 | assert [t.name for t in tools] == ["f"] |
| 681 | |
| 682 | def test_tool_then_text(self, mock_tokenizer, mock_request): |
| 683 | _, content, tools = _delegating(mock_tokenizer).parse( |
| 684 | _tool_block("f", "{}") + MSG_MODEL + f"{TEXT_START}done{END_MESSAGE}", |
| 685 | mock_request, |
| 686 | enable_auto_tools=True, |
| 687 | ) |
| 688 | assert content == "done" |
| 689 | assert [t.name for t in tools] == ["f"] |
| 690 | |
| 691 | def test_two_tools_then_text(self, mock_tokenizer, mock_request): |
| 692 | _, content, tools = _delegating(mock_tokenizer).parse( |
| 693 | _tool_block("f", "{}") |
| 694 | + MSG_MODEL |
| 695 | + _tool_block("g", "{}") |
| 696 | + MSG_MODEL |
| 697 | + f"{TEXT_START}after{END_MESSAGE}", |
| 698 | mock_request, |
| 699 | enable_auto_tools=True, |
| 700 | ) |
| 701 | assert content == "after" |
| 702 | assert [t.name for t in tools] == ["f", "g"] |
| 703 | |
| 704 | def test_reasoning_then_tool_non_streaming(self, mock_tokenizer, mock_request): |
| 705 | reasoning, _, tools = _delegating(mock_tokenizer).parse( |
| 706 | f"{THINK_START}think{END_MESSAGE}{MSG_MODEL}" + _tool_block("f", '{"x":1}'), |
| 707 | mock_request, |
| 708 | enable_auto_tools=True, |
| 709 | ) |
| 710 | assert reasoning == "think" |
| 711 | assert [t.name for t in tools] == ["f"] |
| 712 | |
| 713 | def test_end_sampling_text_closer_consumed(self, mock_tokenizer, mock_request): |
| 714 | tools = [_function_tool()] |
| 715 | mock_request.tools = tools |
| 716 | _, content, calls = _delegating(mock_tokenizer, tools).parse( |
| 717 | f"{TEXT_START}hi{END_SAMPLING}", |
| 718 | mock_request, |
| 719 | enable_auto_tools=True, |
| 720 | ) |
| 721 | assert content == "hi" |
| 722 | assert END_SAMPLING not in content |
| 723 | assert not calls |
| 724 | |
| 725 | def test_end_sampling_tool_closer_round_trips(self, mock_tokenizer, mock_request): |
| 726 | _, _, tools = _delegating(mock_tokenizer).parse( |
| 727 | f'{TOOL_JSON}{{"name":"f","args":{{}}}}{END_SAMPLING}', |
| 728 | mock_request, |
| 729 | enable_auto_tools=True, |
| 730 | ) |
| 731 | assert [t.name for t in tools] == ["f"] |
| 732 | |
| 733 | def test_incomplete_tool_at_eos(self, mock_tokenizer, mock_request): |
| 734 | _, _, tools = _delegating(mock_tokenizer).parse( |
| 735 | f'{TOOL_JSON}{{"name":"d","args":{{"k":"v"', |
| 736 | mock_request, |
| 737 | enable_auto_tools=True, |
| 738 | ) |
| 739 | assert [t.name for t in tools] == ["d"] |
| 740 | |
| 741 | def test_reset_reuse_after_incomplete_span(self, mock_tokenizer, mock_request): |
| 742 | parser = _delegating(mock_tokenizer) |
| 743 | parser.parse( |
| 744 | f'{TOOL_JSON}{{"name":"d","args":{{"k":"v"', |
| 745 | mock_request, |
| 746 | enable_auto_tools=True, |
| 747 | ) |
| 748 | _, content, _ = parser.parse( |
| 749 | f"{TEXT_START}fresh{END_MESSAGE}", |
| 750 | mock_request, |
| 751 | enable_auto_tools=True, |
| 752 | ) |
| 753 | assert content == "fresh" |
| 754 | assert END_MESSAGE not in content |
| 755 | |
| 756 | @pytest.mark.parametrize("chunk_size", [1, 3, 64]) |
| 757 | def test_reasoning_then_tool_streaming( |
| 758 | self, mock_tokenizer, mock_request, chunk_size |
| 759 | ): |
| 760 | content, _, tools, _ = _stream_delegating( |
| 761 | _delegating(mock_tokenizer), |
| 762 | mock_request, |
| 763 | f"{THINK_START}plan{END_MESSAGE}{MSG_MODEL}" |
| 764 | + _tool_block("get_weather", '{"city":"SF"}'), |
| 765 | chunk_size, |
| 766 | self.GEN_PROMPT, |
| 767 | ) |
| 768 | assert tools == ["get_weather"] |
| 769 | assert TOOL_JSON not in content |
| 770 | assert END_MESSAGE not in content |
| 771 | |
| 772 | def test_function_name_not_in_content_after_reasoning( |
| 773 | self, mock_tokenizer, mock_request |
| 774 | ): |
| 775 | name = "search_documents_by_keyword" |
| 776 | tools = [_function_tool(name)] |
| 777 | mock_request.tools = tools |
| 778 | text = ( |
| 779 | f"{THINK_START} context.{END_MESSAGE}{MSG_MODEL}" |
| 780 | f"{name}{_tool_block(name, '{}')}" |
| 781 | ) |
| 782 | |
| 783 | content, reasoning, names, args = _stream_delegating( |
| 784 | _TwoPassInklingParser(mock_tokenizer, tools), |
| 785 | mock_request, |
| 786 | text, |
| 787 | chunk_size=12, |
| 788 | prompt_token_ids=self.GEN_PROMPT, |
| 789 | ) |
| 790 | |
| 791 | assert reasoning == " context." |
| 792 | assert content == "" |
| 793 | assert names == [name] |
| 794 | assert json.loads(args[0]) == {} |
| 795 | |
| 796 | @pytest.mark.parametrize("opener", [TEXT_START, TOOL_TEXT, TOOL_ERROR]) |
| 797 | @pytest.mark.parametrize("chunk_size", [1, 3, 64]) |
| 798 | def test_visible_text_then_tool_streaming( |
| 799 | self, mock_tokenizer, mock_request, chunk_size, opener |
| 800 | ): |
| 801 | """Visible content before a tool call, with no thinking block. |
| 802 | |
| 803 | The reasoning pass leaves its reasoning phase only on an explicit |
| 804 | reasoning-end event. A response that opens with visible content never |
| 805 | emitted one, so the tool pass never ran: the entire tool block came |
| 806 | back as assistant content and no tool call was parsed. Every opener |
| 807 | that starts a visible block has to confirm the boundary, which is why |
| 808 | ``TOOL_TEXT`` and ``TOOL_ERROR`` are covered alongside ``TEXT_START``. |
| 809 | """ |
| 810 | tools = [_function_tool()] |
| 811 | mock_request.tools = tools |
| 812 | content, _, names, args = _stream_delegating( |
| 813 | _delegating(mock_tokenizer, tools), |
| 814 | mock_request, |
| 815 | f"{opener}let me check{END_MESSAGE}{MSG_MODEL}" |
| 816 | + _tool_block("get_weather", '{"city":"SF"}'), |
| 817 | chunk_size, |
| 818 | self.GEN_PROMPT, |
| 819 | ) |
| 820 | assert content == "let me check" |
| 821 | assert names == ["get_weather"] |
| 822 | assert json.loads(args[0]) == {"city": "SF"} |
| 823 | assert TOOL_JSON not in content |
| 824 | assert END_MESSAGE not in content |
| 825 | |
| 826 | @pytest.mark.parametrize("chunk_size", [1, 3, 7, 64]) |
| 827 | def test_tool_start_from_message_header_streaming( |
| 828 | self, mock_tokenizer, mock_request, chunk_size |
| 829 | ): |
| 830 | """A tool call with no block of any kind ahead of it. |
| 831 | |
| 832 | The generation prompt ends in ``<|message_model|>``, so this fires |
| 833 | TOOL_START straight from MESSAGE_HEADER. A tool block is the one |
| 834 | opener that confirms no reasoning is open without rendering visible |
| 835 | content, so it is the case the visible-block openers cannot cover. |
| 836 | """ |
| 837 | tools = [_function_tool()] |
| 838 | mock_request.tools = tools |
| 839 | content, _, names, args = _stream_delegating( |
| 840 | _delegating(mock_tokenizer, tools), |
| 841 | mock_request, |
| 842 | _tool_block("get_weather", '{"city":"Seattle"}'), |
| 843 | chunk_size, |
| 844 | self.GEN_PROMPT, |
| 845 | ) |
| 846 | assert names == ["get_weather"] |
| 847 | assert json.loads(args[0]) == {"city": "Seattle"} |
| 848 | assert content == "" |
| 849 | assert TOOL_JSON not in content |
| 850 | assert END_MESSAGE not in content |
| 851 | |
| 852 | def test_function_name_header_before_tool_start_streaming( |
| 853 | self, mock_tokenizer, mock_request |
| 854 | ): |
| 855 | """The optional function name between ``<|message_model|>`` and the |
| 856 | content-kind marker is metadata: the buffered header must be |
| 857 | discarded on the way out, not flushed into content.""" |
| 858 | tools = [_function_tool()] |
| 859 | mock_request.tools = tools |
| 860 | content, _, names, args = _stream_delegating( |
| 861 | _delegating(mock_tokenizer, tools), |
| 862 | mock_request, |
| 863 | "someFn" + _tool_block("get_weather", '{"city":"Seattle"}'), |
| 864 | 1, |
| 865 | self.GEN_PROMPT, |
| 866 | ) |
| 867 | assert names == ["get_weather"] |
| 868 | assert json.loads(args[0]) == {"city": "Seattle"} |
| 869 | assert content == "" |
| 870 | |
| 871 | def test_content_state_tool_start_streaming(self, mock_tokenizer, mock_request): |
| 872 | """Same opener reached from CONTENT rather than MESSAGE_HEADER: a |
| 873 | text block closed with no ``<|message_model|>`` before the tool |
| 874 | block. Preceding text must reach content exactly once, unmarked.""" |
| 875 | tools = [_function_tool()] |
| 876 | mock_request.tools = tools |
| 877 | content, _, names, args = _stream_delegating( |
| 878 | _delegating(mock_tokenizer, tools), |
| 879 | mock_request, |
| 880 | f"{TEXT_START}intro{END_MESSAGE}" |
| 881 | + _tool_block("get_weather", '{"city":"Seattle"}'), |
| 882 | 1, |
| 883 | self.GEN_PROMPT, |
| 884 | ) |
| 885 | assert names == ["get_weather"] |
| 886 | assert json.loads(args[0]) == {"city": "Seattle"} |
| 887 | assert content == "intro" |
| 888 | |
| 889 | |
| 890 | def test_content_tool_start_emits_reasoning_end_in_reasoning_pass(): |
| 891 | """(CONTENT, TOOL_START) must carry REASONING_END through the |
| 892 | reasoning pass. Every visible-block opener already confirms the |
| 893 | boundary (#49876), but the header-flush path |
| 894 | (MESSAGE_HEADER --END_MESSAGE--> CONTENT) reaches CONTENT without |
| 895 | one, so a tool block opening from there relies on this transition |
| 896 | alone to hand off to the tool pass.""" |
| 897 | engine = StreamingParserEngine(inkling_config(), tokenizer=None) |
| 898 | engine.skip_tool_parsing = True |
| 899 | engine.reset(initial_state=ParserState.CONTENT) |
| 900 | events = engine.parse_complete(f'{TOOL_JSON}{{"name":"f","args":{{}}}}') |
| 901 | assert [e.type for e in events[:2]] == [ |
| 902 | EventType.REASONING_END, |
| 903 | EventType.TEXT_CHUNK, |
| 904 | ] |
| 905 | assert events[1].value == TOOL_JSON |
| 906 | |
| 907 | |
| 908 | def test_skip_reasoning_parsing_inert_for_shared_markers(): |
| 909 | """``skip_reasoning_parsing`` may only bypass reasoning-exclusive |
| 910 | markers. Inkling has none — ``<|end_message|>`` is labelled THINK_END |
| 911 | yet also closes text, header, and tool blocks — so the flag must be |
| 912 | inert: a tool block still closes through its transition and following |
| 913 | text returns to CONTENT instead of leaking into the argument stream.""" |
| 914 | engine = StreamingParserEngine(inkling_config(), tokenizer=None) |
| 915 | engine.skip_reasoning_parsing = True |
| 916 | engine.reset(initial_state=ParserState.CONTENT) |
| 917 | events = engine.parse_complete( |
| 918 | f'{TOOL_JSON}{{"name":"f","args":{{}}}}{END_MESSAGE}after' |
| 919 | ) |
| 920 | types = [e.type for e in events] |
| 921 | end_idx = types.index(EventType.TOOL_CALL_END) |
| 922 | text_after = [e.value for e in events[end_idx:] if e.type == EventType.TEXT_CHUNK] |
| 923 | assert text_after == ["after"] |
| 924 | |
| 925 | |
| 926 | class TestToolParserWithoutReasoningParser: |
| 927 | """Inkling served with only the tool parser, no ``--reasoning-parser``. |
| 928 | |
| 929 | That configuration turns on ``skip_reasoning_parsing``, and Inkling |
| 930 | labels ``<|end_message|>`` as THINK_END; a bypass keyed on the label |
| 931 | would neutralize the closer of every block kind, leaking markers into |
| 932 | content and leaving tool calls unterminated. Structure must keep |
| 933 | parsing exactly as with the reasoning parser attached.""" |
| 934 | |
| 935 | GEN_PROMPT = [_TML_VOCAB[MSG_MODEL]] |
| 936 | |
| 937 | def _tool_only(self, mock_tokenizer, tools=None): |
| 938 | parser_cls = ParserManager.get_parser( |
| 939 | tool_parser_name="inkling", |
| 940 | enable_auto_tools=True, |
| 941 | ) |
| 942 | return parser_cls(mock_tokenizer, tools or []) |
| 943 | |
| 944 | def test_plain_text_non_streaming(self, mock_tokenizer, mock_request): |
| 945 | tools = [_function_tool()] |
| 946 | mock_request.tools = tools |
| 947 | _, content, calls = self._tool_only(mock_tokenizer, tools).parse( |
| 948 | f"{TEXT_START}The answer is 42.{END_MESSAGE}", |
| 949 | mock_request, |
| 950 | enable_auto_tools=True, |
| 951 | ) |
| 952 | assert content == "The answer is 42." |
| 953 | assert not calls |
| 954 | |
| 955 | @pytest.mark.parametrize("chunk_size", [1, 3, 64]) |
| 956 | def test_plain_text_streaming(self, mock_tokenizer, mock_request, chunk_size): |
| 957 | tools = [_function_tool()] |
| 958 | mock_request.tools = tools |
| 959 | content, _, names, _ = _stream_delegating( |
| 960 | self._tool_only(mock_tokenizer, tools), |
| 961 | mock_request, |
| 962 | f"{TEXT_START}The answer is 42.{END_MESSAGE}", |
| 963 | chunk_size, |
| 964 | self.GEN_PROMPT, |
| 965 | ) |
| 966 | assert content == "The answer is 42." |
| 967 | assert not names |
| 968 | |
| 969 | def test_tool_block_non_streaming(self, mock_tokenizer, mock_request): |
| 970 | _, content, calls = self._tool_only(mock_tokenizer).parse( |
| 971 | _tool_block("get_weather", '{"city":"SF"}'), |
| 972 | mock_request, |
| 973 | enable_auto_tools=True, |
| 974 | ) |
| 975 | assert [c.name for c in calls] == ["get_weather"] |
| 976 | assert not content |
| 977 | |
| 978 | @pytest.mark.parametrize("chunk_size", [1, 3, 64]) |
| 979 | def test_tool_block_streaming(self, mock_tokenizer, mock_request, chunk_size): |
| 980 | tools = [_function_tool()] |
| 981 | mock_request.tools = tools |
| 982 | content, _, names, args = _stream_delegating( |
| 983 | self._tool_only(mock_tokenizer, tools), |
| 984 | mock_request, |
| 985 | _tool_block("get_weather", '{"city":"SF"}'), |
| 986 | chunk_size, |
| 987 | self.GEN_PROMPT, |
| 988 | ) |
| 989 | assert names == ["get_weather"] |
| 990 | assert json.loads(args[0]) == {"city": "SF"} |
| 991 | assert content == "" |
| 992 | |
| 993 | def test_thinking_then_text_keeps_structure(self, mock_tokenizer, mock_request): |
| 994 | tools = [_function_tool()] |
| 995 | mock_request.tools = tools |
| 996 | _, content, calls = self._tool_only(mock_tokenizer, tools).parse( |
| 997 | f"{THINK_START}plan{END_MESSAGE}" |
| 998 | f"{MSG_MODEL}{TEXT_START}Let me look.{END_MESSAGE}", |
| 999 | mock_request, |
| 1000 | enable_auto_tools=True, |
| 1001 | ) |
| 1002 | assert content == "Let me look." |
| 1003 | assert END_MESSAGE not in content |
| 1004 | assert THINK_START not in content |
| 1005 | assert not calls |
| 1006 | |
| 1007 | def test_thinking_then_tool_still_promotes(self, mock_tokenizer, mock_request): |
| 1008 | _, content, calls = self._tool_only(mock_tokenizer).parse( |
| 1009 | f"{THINK_START}think{END_MESSAGE}{MSG_MODEL}" |
| 1010 | + _tool_block("get_weather", '{"x":1}'), |
| 1011 | mock_request, |
| 1012 | enable_auto_tools=True, |
| 1013 | ) |
| 1014 | assert [c.name for c in calls] == ["get_weather"] |
| 1015 | assert not content |