| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | """Unit tests for the MuseGlimmer ATEM tool parser and reasoning parser. |
| 5 | |
| 6 | MuseGlimmer writes every turn as a sequence of channel-scoped messages rather |
| 7 | than JSON, so the two parsers are tested together: the reasoning parser strips |
| 8 | the reasoning span and forwards the remaining channels as content, and the tool |
| 9 | parser reads ATEM markup out of those channels. |
| 10 | |
| 11 | Four areas, in order: |
| 12 | |
| 13 | 1. non-streaming tool-call extraction, including channel scoping (an |
| 14 | ``<atem:invoke>`` echoed inside reasoning must never become a call); |
| 15 | 2. the reasoning -> tool-parser handoff, which regressed once by returning |
| 16 | ``content=None`` and starving the tool parser; |
| 17 | 3. streaming, where markers routinely straddle chunk boundaries, plus |
| 18 | truncation isolation for an unterminated ``to=self`` block; |
| 19 | 4. tool-name normalization against the tools registered on the request. |
| 20 | |
| 21 | These drive the parsers directly and need no checkpoint. The tests that require |
| 22 | a real tokenizer live in ``test_muse_glimmer_parse_delta.py``. |
| 23 | """ |
| 24 | |
| 25 | import json |
| 26 | from types import SimpleNamespace |
| 27 | from typing import cast |
| 28 | |
| 29 | import pytest |
| 30 | |
| 31 | from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest |
| 32 | from vllm.reasoning.muse_glimmer_reasoning_parser import MuseGlimmerReasoningParser |
| 33 | from vllm.tool_parsers.muse_glimmer_tool_parser import MuseGlimmerToolParser |
| 34 | |
| 35 | R: MuseGlimmerReasoningParser |
| 36 | T: MuseGlimmerToolParser |
| 37 | |
| 38 | |
| 39 | @pytest.fixture(autouse=True) |
| 40 | def _fresh_parsers(): |
| 41 | """Give each test request-scoped parser state through the real constructors.""" |
| 42 | global R, T |
| 43 | R = MuseGlimmerReasoningParser(object()) |
| 44 | T = MuseGlimmerToolParser(object()) |
| 45 | |
| 46 | |
| 47 | # Any framing token that must NEVER appear in surfaced reasoning/content. |
| 48 | _FRAMING = [ |
| 49 | "<|start|>", |
| 50 | "<|message|>", |
| 51 | "<|eom|>", |
| 52 | "<|eot|>", |
| 53 | "to=self", |
| 54 | "to=user", |
| 55 | "to=read.read", |
| 56 | "<atem:", |
| 57 | ] |
| 58 | |
| 59 | |
| 60 | class _FakeReq: |
| 61 | """Minimal ChatCompletionRequest stand-in (no registered tools).""" |
| 62 | |
| 63 | tools = None |
| 64 | |
| 65 | |
| 66 | def _fake_req() -> ChatCompletionRequest: |
| 67 | return cast(ChatCompletionRequest, _FakeReq()) |
| 68 | |
| 69 | |
| 70 | # The parsers accept a missing request at runtime. |
| 71 | _NO_REQ = cast(ChatCompletionRequest, None) |
| 72 | |
| 73 | |
| 74 | def _req(*names: str) -> ChatCompletionRequest: |
| 75 | """A request with ``names`` registered as tools.""" |
| 76 | return cast( |
| 77 | ChatCompletionRequest, |
| 78 | SimpleNamespace( |
| 79 | tools=[SimpleNamespace(function=SimpleNamespace(name=n)) for n in names] |
| 80 | ), |
| 81 | ) |
| 82 | |
| 83 | |
| 84 | def _call(name): |
| 85 | """One tool-call turn invoking ``name``.""" |
| 86 | return ( |
| 87 | f"<|start|>assistant to={name}<|message|>" |
| 88 | f'<atem:function_calls>\n<atem:invoke name="{name}">\n' |
| 89 | f'<atem:parameter name="city">Paris</atem:parameter>\n' |
| 90 | f"</atem:invoke>\n</atem:function_calls>" |
| 91 | ) |
| 92 | |
| 93 | |
| 94 | # ---------------------------------------------------------------- tool calls |
| 95 | |
| 96 | |
| 97 | def test_single_tool_call_after_reasoning(): |
| 98 | raw = ( |
| 99 | "to=self<|message|>Let me check the weather.<|eom|>" |
| 100 | "<|start|>assistant to=weather.get<|message|>" |
| 101 | '<atem:function_calls>\n<atem:invoke name="weather.get">\n' |
| 102 | '<atem:parameter name="city">Paris</atem:parameter>\n' |
| 103 | '<atem:parameter name="units">celsius</atem:parameter>\n' |
| 104 | "</atem:invoke>\n</atem:function_calls><|eot|>" |
| 105 | ) |
| 106 | out = MuseGlimmerToolParser.extract_tool_calls(T, raw, _NO_REQ) |
| 107 | assert out.tools_called and len(out.tool_calls) == 1 |
| 108 | assert out.tool_calls[0].function.name == "weather.get" |
| 109 | assert json.loads(out.tool_calls[0].function.arguments) == { |
| 110 | "city": "Paris", |
| 111 | "units": "celsius", |
| 112 | } |
| 113 | |
| 114 | |
| 115 | def test_parallel_calls_across_eom_boundaries(): |
| 116 | raw = ( |
| 117 | "<|start|>assistant to=math.add<|message|>" |
| 118 | '<atem:function_calls>\n<atem:invoke name="math.add">\n' |
| 119 | '<atem:parameter name="a">1</atem:parameter>\n' |
| 120 | '<atem:parameter name="b">2</atem:parameter>\n' |
| 121 | "</atem:invoke>\n</atem:function_calls><|eom|>" |
| 122 | "<|start|>assistant to=math.mul<|message|>" |
| 123 | '<atem:function_calls>\n<atem:invoke name="math.mul">\n' |
| 124 | '<atem:parameter name="a">3</atem:parameter>\n' |
| 125 | '<atem:parameter name="b">4</atem:parameter>\n' |
| 126 | "</atem:invoke>\n</atem:function_calls><|eot|>" |
| 127 | ) |
| 128 | out = MuseGlimmerToolParser.extract_tool_calls(T, raw, _NO_REQ) |
| 129 | assert out.tools_called and len(out.tool_calls) == 2, len(out.tool_calls) |
| 130 | assert [t.function.name for t in out.tool_calls] == ["math.add", "math.mul"] |
| 131 | # JSON-typed values decode to ints |
| 132 | assert json.loads(out.tool_calls[0].function.arguments) == {"a": 1, "b": 2} |
| 133 | |
| 134 | |
| 135 | def test_echoed_invoke_in_reasoning_is_not_parsed(): |
| 136 | """Channel scoping: an invoke quoted inside reasoning is not a call.""" |
| 137 | raw = ( |
| 138 | 'to=self<|message|>I could call <atem:invoke name="evil.fn">' |
| 139 | '<atem:parameter name="x">1</atem:parameter></atem:invoke> ' |
| 140 | "but I will not.<|eom|>" |
| 141 | "<|start|>assistant to=user<|message|>The answer is 42.<|eot|>" |
| 142 | ) |
| 143 | out = MuseGlimmerToolParser.extract_tool_calls(T, raw, _NO_REQ) |
| 144 | assert not out.tools_called, "channel scoping failed -- echoed invoke parsed!" |
| 145 | assert out.content == "The answer is 42.", repr(out.content) |
| 146 | |
| 147 | |
| 148 | def test_plain_answer_yields_no_tool_calls(): |
| 149 | out = MuseGlimmerToolParser.extract_tool_calls( |
| 150 | T, "to=user<|message|>Just a plain answer.<|eot|>", _NO_REQ |
| 151 | ) |
| 152 | assert not out.tools_called |
| 153 | |
| 154 | |
| 155 | def test_json_object_array_and_bool_params_decode(): |
| 156 | raw = ( |
| 157 | "<|start|>assistant to=api.call<|message|>" |
| 158 | '<atem:function_calls>\n<atem:invoke name="api.call">\n' |
| 159 | '<atem:parameter name="payload">{"nested": [1, 2, 3]}</atem:parameter>\n' |
| 160 | '<atem:parameter name="flag">true</atem:parameter>\n' |
| 161 | "</atem:invoke>\n</atem:function_calls><|eot|>" |
| 162 | ) |
| 163 | out = MuseGlimmerToolParser.extract_tool_calls(T, raw, _NO_REQ) |
| 164 | assert json.loads(out.tool_calls[0].function.arguments) == { |
| 165 | "payload": {"nested": [1, 2, 3]}, |
| 166 | "flag": True, |
| 167 | } |
| 168 | |
| 169 | |
| 170 | # ------------------------------------------------- reasoning -> tool handoff |
| 171 | |
| 172 | |
| 173 | def test_reasoning_to_toolcall_handoff(): |
| 174 | """The regression: content=None here starved the tool parser.""" |
| 175 | raw = ( |
| 176 | " to=self<|message|>Let me call the tool.<|eom|>" |
| 177 | "<|start|>assistant to=weather.get<|message|>" |
| 178 | '<atem:function_calls>\n<atem:invoke name="weather.get">\n' |
| 179 | '<atem:parameter name="city">Paris</atem:parameter>\n' |
| 180 | "</atem:invoke>\n</atem:function_calls>" |
| 181 | ) |
| 182 | reasoning, content = MuseGlimmerReasoningParser.extract_reasoning(R, raw, _NO_REQ) |
| 183 | assert reasoning == "Let me call the tool.", repr(reasoning) |
| 184 | assert content is not None and "<atem:invoke" in content, repr(content) |
| 185 | out = MuseGlimmerToolParser.extract_tool_calls(T, content, _NO_REQ) |
| 186 | assert out.tools_called and len(out.tool_calls) == 1 |
| 187 | assert out.tool_calls[0].function.name == "weather.get" |
| 188 | |
| 189 | |
| 190 | def test_reasoning_then_user_answer(): |
| 191 | raw = ( |
| 192 | " to=self<|message|>thinking<|eom|>" |
| 193 | "<|start|>assistant to=user<|message|>The answer is 42.<|eot|>" |
| 194 | ) |
| 195 | reasoning, content = MuseGlimmerReasoningParser.extract_reasoning(R, raw, _NO_REQ) |
| 196 | assert reasoning == "thinking", repr(reasoning) |
| 197 | assert content == "The answer is 42.", repr(content) |
| 198 | assert not MuseGlimmerToolParser.extract_tool_calls( |
| 199 | T, content, _NO_REQ |
| 200 | ).tools_called |
| 201 | |
| 202 | |
| 203 | def test_plain_content_without_framing_passes_through(): |
| 204 | reasoning, content = MuseGlimmerReasoningParser.extract_reasoning( |
| 205 | R, "Just a direct answer.", _NO_REQ |
| 206 | ) |
| 207 | assert reasoning is None and content == "Just a direct answer.", ( |
| 208 | reasoning, |
| 209 | content, |
| 210 | ) |
| 211 | |
| 212 | |
| 213 | def test_reasoning_then_parallel_calls(): |
| 214 | raw = ( |
| 215 | " to=self<|message|>need two calls<|eom|>" |
| 216 | "<|start|>assistant to=math.add<|message|>" |
| 217 | '<atem:function_calls>\n<atem:invoke name="math.add">\n' |
| 218 | '<atem:parameter name="a">1</atem:parameter>\n</atem:invoke>\n' |
| 219 | "</atem:function_calls><|eom|>" |
| 220 | "<|start|>assistant to=math.mul<|message|>" |
| 221 | '<atem:function_calls>\n<atem:invoke name="math.mul">\n' |
| 222 | '<atem:parameter name="a">3</atem:parameter>\n</atem:invoke>\n' |
| 223 | "</atem:function_calls><|eot|>" |
| 224 | ) |
| 225 | reasoning, content = MuseGlimmerReasoningParser.extract_reasoning(R, raw, _NO_REQ) |
| 226 | assert reasoning == "need two calls", repr(reasoning) |
| 227 | assert content is not None |
| 228 | out = MuseGlimmerToolParser.extract_tool_calls(T, content, _NO_REQ) |
| 229 | assert [t.function.name for t in out.tool_calls] == ["math.add", "math.mul"], ( |
| 230 | out.tool_calls |
| 231 | ) |
| 232 | |
| 233 | |
| 234 | # ----------------------------------------------------------------- streaming |
| 235 | |
| 236 | |
| 237 | def _stream(raw: str, chunk: int): |
| 238 | """Feed ``raw`` incrementally in ``chunk``-char steps through BOTH streaming |
| 239 | parsers; return (reasoning, content, tool_calls).""" |
| 240 | reasoning, content, toolcalls = [], [], [] |
| 241 | prev = "" |
| 242 | i = 0 |
| 243 | while i < len(raw): |
| 244 | cur = raw[: i + chunk] |
| 245 | delta = cur[len(prev) :] |
| 246 | dm = MuseGlimmerReasoningParser.extract_reasoning_streaming( |
| 247 | R, prev, cur, delta, [], [], [] |
| 248 | ) |
| 249 | if dm is not None: |
| 250 | if dm.reasoning: |
| 251 | reasoning.append(dm.reasoning) |
| 252 | content_delta = getattr(dm, "content", None) |
| 253 | # Tool-channel content is an internal handoff to the tool parser, |
| 254 | # not client-visible content from the unified parser. |
| 255 | if content_delta and "<atem:function_calls>" not in content_delta: |
| 256 | content.append(content_delta) |
| 257 | dt = MuseGlimmerToolParser.extract_tool_calls_streaming( |
| 258 | T, prev, cur, delta, [], [], [], _fake_req() |
| 259 | ) |
| 260 | if dt is not None and dt.tool_calls: |
| 261 | toolcalls.extend(dt.tool_calls) |
| 262 | prev = cur |
| 263 | i += chunk |
| 264 | return "".join(reasoning), "".join(content), toolcalls |
| 265 | |
| 266 | |
| 267 | def _fn_of(tc): |
| 268 | fn = tc.function |
| 269 | if isinstance(fn, dict): |
| 270 | return fn.get("name"), fn.get("arguments") |
| 271 | return fn.name, fn.arguments |
| 272 | |
| 273 | |
| 274 | RAW_TOOLCALL = ( |
| 275 | " to=self<|message|>I should read the hostname file to answer.<|eom|>" |
| 276 | "<|start|>assistant to=read.read<|message|>" |
| 277 | '<atem:function_calls>\n<atem:invoke name="read.read">\n' |
| 278 | '<atem:parameter name="path">/etc/hostname</atem:parameter>\n' |
| 279 | "</atem:invoke>\n</atem:function_calls>" |
| 280 | ) |
| 281 | |
| 282 | RAW_ANSWER = ( |
| 283 | " to=self<|message|>Think about it.<|eom|>" |
| 284 | "<|start|>assistant to=user<|message|>The answer is 42.<|eot|>" |
| 285 | ) |
| 286 | |
| 287 | # NO closing <|eom|> -> truncated CoT |
| 288 | RAW_TRUNCATED = ( |
| 289 | " to=self<|message|>Maybe I should call " |
| 290 | '<atem:function_calls>\n<atem:invoke name="read.read">\n' |
| 291 | '<atem:parameter name="path">/etc/hostname</atem:parameter>\n' |
| 292 | "</atem:invoke>\n</atem:function_calls> but wait" |
| 293 | ) |
| 294 | |
| 295 | |
| 296 | def _check_toolcall_stream(chunk): |
| 297 | reasoning, content, tcs = _stream(RAW_TOOLCALL, chunk) |
| 298 | # (a) no framing token leaks into reasoning or content |
| 299 | for f in _FRAMING: |
| 300 | assert f not in reasoning, ( |
| 301 | f"framing {f!r} leaked into reasoning (chunk={chunk})" |
| 302 | ) |
| 303 | assert f not in content, f"framing {f!r} leaked into content (chunk={chunk})" |
| 304 | # (b) exactly one tool_call with correct name + args |
| 305 | assert len(tcs) == 1, f"expected 1 tool_call, got {len(tcs)} (chunk={chunk})" |
| 306 | name, args = _fn_of(tcs[0]) |
| 307 | assert name == "read.read", name |
| 308 | assert json.loads(args) == {"path": "/etc/hostname"}, args |
| 309 | assert tcs[0].index == 0 and tcs[0].type == "function" and tcs[0].id |
| 310 | # (c) reasoning captured separately and clean |
| 311 | assert reasoning == "I should read the hostname file to answer.", repr(reasoning) |
| 312 | assert content == "", repr(content) |
| 313 | |
| 314 | |
| 315 | def test_streaming_toolcall_chunk3(): |
| 316 | _check_toolcall_stream(3) |
| 317 | |
| 318 | |
| 319 | def test_streaming_toolcall_charwise(): |
| 320 | # worst case: markers arrive one char at a time (mid-marker deltas) |
| 321 | _check_toolcall_stream(1) |
| 322 | |
| 323 | |
| 324 | def test_streaming_toolcall_bigchunks(): |
| 325 | _check_toolcall_stream(17) |
| 326 | |
| 327 | |
| 328 | def test_streaming_reasoning_then_content(): |
| 329 | reasoning, content, tcs = _stream(RAW_ANSWER, 3) |
| 330 | for f in _FRAMING: |
| 331 | assert f not in reasoning and f not in content, f |
| 332 | assert reasoning == "Think about it.", repr(reasoning) |
| 333 | assert content == "The answer is 42.", repr(content) |
| 334 | assert tcs == [] |
| 335 | |
| 336 | |
| 337 | def test_truncated_cot_no_toolcall_nonstreaming(): |
| 338 | out = MuseGlimmerToolParser.extract_tool_calls(T, RAW_TRUNCATED, _fake_req()) |
| 339 | assert not out.tools_called and out.tool_calls == [] |
| 340 | # partial reasoning must still be recovered by the reasoning parser |
| 341 | reasoning, _ = MuseGlimmerReasoningParser.extract_reasoning( |
| 342 | R, RAW_TRUNCATED, _fake_req() |
| 343 | ) |
| 344 | assert reasoning and "Maybe I should call" in reasoning, repr(reasoning) |
| 345 | |
| 346 | |
| 347 | def test_truncated_cot_no_toolcall_streaming(): |
| 348 | _, _, tcs = _stream(RAW_TRUNCATED, 3) |
| 349 | assert tcs == [], f"truncated CoT invoke leaked as streaming tool call: {tcs}" |
| 350 | |
| 351 | |
| 352 | # ------------------------------------------------------- name normalization |
| 353 | # |
| 354 | # MuseGlimmer emits `get_weather.get_weather` for a bare-registered |
| 355 | # `get_weather`, and `weather.get` verbatim for a namespaced one. The parser |
| 356 | # normalizes against the tools actually registered on the request. |
| 357 | |
| 358 | |
| 359 | def test_doubled_bare_name_collapses(): |
| 360 | out = MuseGlimmerToolParser.extract_tool_calls( |
| 361 | T, _call("get_weather.get_weather"), _req("get_weather") |
| 362 | ) |
| 363 | assert out.tools_called and out.tool_calls[0].function.name == "get_weather", ( |
| 364 | out.tool_calls[0].function.name |
| 365 | ) |
| 366 | |
| 367 | |
| 368 | def test_namespaced_name_preserved(): |
| 369 | out = MuseGlimmerToolParser.extract_tool_calls( |
| 370 | T, _call("weather.get"), _req("weather.get") |
| 371 | ) |
| 372 | assert out.tool_calls[0].function.name == "weather.get" |
| 373 | |
| 374 | |
| 375 | def test_unregistered_namespace_is_preserved(): |
| 376 | # Suffix-only matching can silently dispatch a tool from the wrong namespace. |
| 377 | out = MuseGlimmerToolParser.extract_tool_calls( |
| 378 | T, _call("foo.get_weather"), _req("get_weather") |
| 379 | ) |
| 380 | assert out.tool_calls[0].function.name == "foo.get_weather" |
| 381 | |
| 382 | |
| 383 | def test_trailing_segment_ambiguous_left_alone(): |
| 384 | # two registered tools share leaf 'get' -> ambiguous -> do NOT rewrite |
| 385 | out = MuseGlimmerToolParser.extract_tool_calls( |
| 386 | T, _call("x.get"), _req("weather.get", "time.get") |
| 387 | ) |
| 388 | assert out.tool_calls[0].function.name == "x.get" |
| 389 | |
| 390 | |
| 391 | def test_no_registered_tools_passthrough(): |
| 392 | out = MuseGlimmerToolParser.extract_tool_calls( |
| 393 | T, _call("get_weather.get_weather"), _NO_REQ |
| 394 | ) |
| 395 | assert out.tool_calls[0].function.name == "get_weather.get_weather" |
| 396 | |
| 397 | |
| 398 | def test_exact_match_kept(): |
| 399 | out = MuseGlimmerToolParser.extract_tool_calls( |
| 400 | T, _call("get_weather"), _req("get_weather") |
| 401 | ) |
| 402 | assert out.tool_calls[0].function.name == "get_weather" |