| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Tests for the unified Gemma4 parser engine.""" |
| 4 | |
| 5 | import json |
| 6 | from unittest.mock import MagicMock |
| 7 | |
| 8 | import pytest |
| 9 | |
| 10 | from tests.parser.engine.conftest import make_mock_tokenizer |
| 11 | from tests.parser.engine.streaming_helpers import ( |
| 12 | collect_content, |
| 13 | collect_function_name, |
| 14 | collect_tool_arguments, |
| 15 | simulate_tool_streaming, |
| 16 | ) |
| 17 | from vllm.entrypoints.generate.base.protocol import DeltaMessage |
| 18 | from vllm.entrypoints.openai.chat_completion.protocol import ( |
| 19 | ChatCompletionRequest, |
| 20 | ) |
| 21 | from vllm.parser.gemma4 import Gemma4Parser |
| 22 | |
| 23 | # ── Special token IDs (arbitrary but consistent) ───────────────────── |
| 24 | CHANNEL_START_ID = 50 # <|channel> |
| 25 | CHANNEL_END_ID = 51 # <channel|> |
| 26 | TOOL_CALL_START_ID = 48 # <|tool_call> |
| 27 | TOOL_CALL_END_ID = 49 # <tool_call|> |
| 28 | QUOTED_ID = 52 # <|"|> |
| 29 | NEW_TURN_ID = 53 # <|turn> |
| 30 | SPECIAL_TOKEN_MAP = { |
| 31 | CHANNEL_START_ID: "<|channel>", |
| 32 | CHANNEL_END_ID: "<channel|>", |
| 33 | TOOL_CALL_START_ID: "<|tool_call>", |
| 34 | TOOL_CALL_END_ID: "<tool_call|>", |
| 35 | QUOTED_ID: '<|"|>', |
| 36 | NEW_TURN_ID: "<|turn>", |
| 37 | } |
| 38 | |
| 39 | SPECIAL_TEXT_TO_ID = {v: k for k, v in SPECIAL_TOKEN_MAP.items()} |
| 40 | |
| 41 | |
| 42 | def _make_tokenizer(sequence: list[tuple[int, str]]) -> MagicMock: |
| 43 | decode_map: dict[int, str] = dict(SPECIAL_TOKEN_MAP) |
| 44 | for tid, text in sequence: |
| 45 | decode_map[tid] = text |
| 46 | |
| 47 | tokenizer = MagicMock() |
| 48 | tokenizer.get_vocab.return_value = dict(SPECIAL_TEXT_TO_ID) |
| 49 | tokenizer.encode.return_value = [tid for tid, _ in sequence] |
| 50 | |
| 51 | def decode(ids, skip_special_tokens=False): |
| 52 | parts = [] |
| 53 | for tid in ids: |
| 54 | if skip_special_tokens and tid in SPECIAL_TOKEN_MAP: |
| 55 | continue |
| 56 | text = decode_map.get(tid, f"?{tid}?") |
| 57 | parts.append(text) |
| 58 | return "".join(parts) |
| 59 | |
| 60 | tokenizer.decode.side_effect = decode |
| 61 | tokenizer.all_special_tokens = list(SPECIAL_TOKEN_MAP.values()) |
| 62 | tokenizer.all_special_ids = list(SPECIAL_TOKEN_MAP.keys()) |
| 63 | return tokenizer |
| 64 | |
| 65 | |
| 66 | # ── Model output ──────────────────────────────────────────────────── |
| 67 | |
| 68 | REASONING_TEXT = ( |
| 69 | "The user is asking for the current weather in Dallas, Texas, " |
| 70 | "and specifically requests the temperature in Fahrenheit. " |
| 71 | "I have a tool `get_current_weather` that can provide this " |
| 72 | "information. I should call this tool with `city='Dallas'`, " |
| 73 | "`state='TX'`, and `unit='fahrenheit'`." |
| 74 | ) |
| 75 | |
| 76 | # Break reasoning into word-level tokens |
| 77 | _reasoning_words = REASONING_TEXT.split(" ") |
| 78 | _REGULAR_TOKEN_START = 1000 |
| 79 | REASONING_TOKENS: list[tuple[int, str]] = [] |
| 80 | for i, word in enumerate(_reasoning_words): |
| 81 | prefix = " " if i > 0 else "" |
| 82 | REASONING_TOKENS.append((_REGULAR_TOKEN_START + i, prefix + word)) |
| 83 | |
| 84 | # Tool call body tokens |
| 85 | TOOL_BODY_TOKENS: list[tuple[int, str]] = [ |
| 86 | (2000, "call"), |
| 87 | (2001, ":"), |
| 88 | (2002, "get_current_weather"), |
| 89 | (2003, "{"), |
| 90 | (2004, "city"), |
| 91 | (2005, ":"), |
| 92 | (2006, "Dallas"), |
| 93 | (2007, ","), |
| 94 | (2008, "state"), |
| 95 | (2009, ":"), |
| 96 | (2010, "TX"), |
| 97 | (2011, ","), |
| 98 | (2012, "unit"), |
| 99 | (2013, ":"), |
| 100 | (2014, "fahrenheit"), |
| 101 | (2015, "}"), |
| 102 | ] |
| 103 | |
| 104 | FULL_TOKEN_SEQUENCE: list[tuple[int, str]] = [] |
| 105 | FULL_TOKEN_SEQUENCE.append((CHANNEL_START_ID, "<|channel>")) |
| 106 | FULL_TOKEN_SEQUENCE.append((3000, "thought")) |
| 107 | FULL_TOKEN_SEQUENCE.append((3001, "\n")) |
| 108 | FULL_TOKEN_SEQUENCE.extend(REASONING_TOKENS) |
| 109 | FULL_TOKEN_SEQUENCE.append((CHANNEL_END_ID, "<channel|>")) |
| 110 | FULL_TOKEN_SEQUENCE.append((TOOL_CALL_START_ID, "<|tool_call>")) |
| 111 | FULL_TOKEN_SEQUENCE.extend(TOOL_BODY_TOKENS[:4]) |
| 112 | FULL_TOKEN_SEQUENCE.extend(TOOL_BODY_TOKENS[4:6]) |
| 113 | FULL_TOKEN_SEQUENCE.append((QUOTED_ID, '<|"|>')) |
| 114 | FULL_TOKEN_SEQUENCE.append(TOOL_BODY_TOKENS[6]) |
| 115 | FULL_TOKEN_SEQUENCE.append((QUOTED_ID, '<|"|>')) |
| 116 | FULL_TOKEN_SEQUENCE.extend(TOOL_BODY_TOKENS[7:10]) |
| 117 | FULL_TOKEN_SEQUENCE.append((QUOTED_ID, '<|"|>')) |
| 118 | FULL_TOKEN_SEQUENCE.append(TOOL_BODY_TOKENS[10]) |
| 119 | FULL_TOKEN_SEQUENCE.append((QUOTED_ID, '<|"|>')) |
| 120 | FULL_TOKEN_SEQUENCE.extend(TOOL_BODY_TOKENS[11:14]) |
| 121 | FULL_TOKEN_SEQUENCE.append((QUOTED_ID, '<|"|>')) |
| 122 | FULL_TOKEN_SEQUENCE.append(TOOL_BODY_TOKENS[14]) |
| 123 | FULL_TOKEN_SEQUENCE.append((QUOTED_ID, '<|"|>')) |
| 124 | FULL_TOKEN_SEQUENCE.append(TOOL_BODY_TOKENS[15]) |
| 125 | FULL_TOKEN_SEQUENCE.append((TOOL_CALL_END_ID, "<tool_call|>")) |
| 126 | |
| 127 | # Full model output as a single string |
| 128 | FULL_MODEL_OUTPUT = "".join(text for _, text in FULL_TOKEN_SEQUENCE) |
| 129 | |
| 130 | # ── Helpers ────────────────────────────────────────────────────────── |
| 131 | |
| 132 | |
| 133 | def _stream_tokens_batched( |
| 134 | parser, tokenizer, request, batch_size=10, prompt_token_ids=None |
| 135 | ) -> list[DeltaMessage | None]: |
| 136 | """Feed tokens in batches through parse_delta.""" |
| 137 | token_ids = tokenizer.encode("", add_special_tokens=False) |
| 138 | results: list[DeltaMessage | None] = [] |
| 139 | n = len(token_ids) |
| 140 | |
| 141 | for start in range(0, n, batch_size): |
| 142 | batch_ids = token_ids[start : start + batch_size] |
| 143 | delta_text = tokenizer.decode(batch_ids) |
| 144 | result = parser.parse_delta( |
| 145 | delta_text, |
| 146 | batch_ids, |
| 147 | request, |
| 148 | prompt_token_ids=prompt_token_ids, |
| 149 | finished=(start + batch_size >= n), |
| 150 | ) |
| 151 | prompt_token_ids = None |
| 152 | results.append(result) |
| 153 | return results |
| 154 | |
| 155 | |
| 156 | def _collect_fields(results): |
| 157 | reasoning = "".join(r.reasoning for r in results if r and r.reasoning) |
| 158 | content = "".join(r.content for r in results if r and r.content) |
| 159 | tool_calls = [tc for r in results if r and r.tool_calls for tc in r.tool_calls] |
| 160 | return reasoning, content, tool_calls |
| 161 | |
| 162 | |
| 163 | # ── Fixtures ───────────────────────────────────────────────────────── |
| 164 | |
| 165 | |
| 166 | @pytest.fixture |
| 167 | def mock_tokenizer(): |
| 168 | return _make_tokenizer(FULL_TOKEN_SEQUENCE) |
| 169 | |
| 170 | |
| 171 | @pytest.fixture |
| 172 | def parser(mock_tokenizer): |
| 173 | return Gemma4Parser(mock_tokenizer, chat_template_kwargs={"enable_thinking": True}) |
| 174 | |
| 175 | |
| 176 | @pytest.fixture |
| 177 | def request_obj(): |
| 178 | return ChatCompletionRequest( |
| 179 | model="test-model", |
| 180 | messages=[{"role": "user", "content": "hi"}], |
| 181 | ) |
| 182 | |
| 183 | |
| 184 | # ── Tests ──────────────────────────────────────────────────────────── |
| 185 | |
| 186 | |
| 187 | class TestGemma4StreamingReasoningThenToolCall: |
| 188 | """Streaming: reasoning followed by a tool call.""" |
| 189 | |
| 190 | def test_tool_call_extracted(self, parser, mock_tokenizer, request_obj): |
| 191 | """Tool calls must be extracted from streaming output.""" |
| 192 | results = _stream_tokens_batched( |
| 193 | parser, |
| 194 | mock_tokenizer, |
| 195 | request_obj, |
| 196 | batch_size=10, |
| 197 | prompt_token_ids=[], |
| 198 | ) |
| 199 | |
| 200 | reasoning, content, tool_calls = _collect_fields(results) |
| 201 | |
| 202 | assert len(tool_calls) > 0, ( |
| 203 | f"Expected tool_calls but got none. " |
| 204 | f"content={content!r}, reasoning={reasoning[:80]!r}..." |
| 205 | ) |
| 206 | |
| 207 | names = [ |
| 208 | tc.function.name for tc in tool_calls if tc.function and tc.function.name |
| 209 | ] |
| 210 | assert "get_current_weather" in names, ( |
| 211 | f"Expected get_current_weather, got {names}" |
| 212 | ) |
| 213 | |
| 214 | args_text = "".join( |
| 215 | tc.function.arguments |
| 216 | for tc in tool_calls |
| 217 | if tc.function and tc.function.arguments |
| 218 | ) |
| 219 | if args_text: |
| 220 | parsed_args = json.loads(args_text) |
| 221 | assert parsed_args.get("city") == "Dallas" |
| 222 | assert parsed_args.get("state") == "TX" |
| 223 | assert parsed_args.get("unit") == "fahrenheit" |
| 224 | |
| 225 | def test_tool_call_text_not_in_content(self, parser, mock_tokenizer, request_obj): |
| 226 | """Tool call body must not leak into content.""" |
| 227 | results = _stream_tokens_batched( |
| 228 | parser, |
| 229 | mock_tokenizer, |
| 230 | request_obj, |
| 231 | batch_size=10, |
| 232 | prompt_token_ids=[], |
| 233 | ) |
| 234 | |
| 235 | _, content, _ = _collect_fields(results) |
| 236 | |
| 237 | assert "call:" not in content, ( |
| 238 | f"Tool call text leaked into content: {content!r}" |
| 239 | ) |
| 240 | assert "get_current_weather" not in content, ( |
| 241 | f"Function name leaked into content: {content!r}" |
| 242 | ) |
| 243 | |
| 244 | def test_reasoning_extracted(self, parser, mock_tokenizer, request_obj): |
| 245 | """Reasoning content should be captured.""" |
| 246 | results = _stream_tokens_batched( |
| 247 | parser, |
| 248 | mock_tokenizer, |
| 249 | request_obj, |
| 250 | batch_size=10, |
| 251 | prompt_token_ids=[], |
| 252 | ) |
| 253 | |
| 254 | reasoning, _, _ = _collect_fields(results) |
| 255 | |
| 256 | assert "weather" in reasoning.lower(), ( |
| 257 | f"Expected reasoning about weather, got: {reasoning[:100]!r}" |
| 258 | ) |
| 259 | |
| 260 | |
| 261 | # ── Prompt ends inside an open <|channel>thought\n block ───────────── |
| 262 | |
| 263 | _OPEN_REASONING_GEN_SEQUENCE: list[tuple[int, str]] = [ |
| 264 | (7001, "Sure"), |
| 265 | (7002, ","), |
| 266 | (7003, " the"), |
| 267 | (7004, " answer"), |
| 268 | (7005, " is"), |
| 269 | (7006, " 42"), |
| 270 | (CHANNEL_END_ID, "<channel|>"), |
| 271 | (7007, "Hello"), |
| 272 | (7008, " world"), |
| 273 | ] |
| 274 | |
| 275 | |
| 276 | class TestGemma4PromptOpenReasoning: |
| 277 | """When ``add_generation_prompt=True`` after a final tool response with |
| 278 | ``enable_thinking=True``, the Gemma4 chat template leaves the prompt |
| 279 | ending with ``<|channel>thought\\n`` — i.e. inside an open reasoning |
| 280 | channel. Tokens generated before ``<channel|>`` must be classified as |
| 281 | ``reasoning``, not visible ``content``. |
| 282 | |
| 283 | Regression test for vllm-project/vllm#45834. |
| 284 | """ |
| 285 | |
| 286 | @pytest.fixture |
| 287 | def open_reasoning_tokenizer(self): |
| 288 | return _make_tokenizer(_OPEN_REASONING_GEN_SEQUENCE) |
| 289 | |
| 290 | @pytest.fixture |
| 291 | def open_reasoning_parser(self, open_reasoning_tokenizer): |
| 292 | return Gemma4Parser( |
| 293 | open_reasoning_tokenizer, |
| 294 | chat_template_kwargs={"enable_thinking": True}, |
| 295 | ) |
| 296 | |
| 297 | @staticmethod |
| 298 | def _prompt_ids_open_channel() -> list[int]: |
| 299 | # Mimics a prompt that ends with ``...<|channel>thought\n``. The |
| 300 | # specific token ids for ``thought`` and ``\n`` are arbitrary — only |
| 301 | # the trailing ``<|channel>`` start token matters for detection. |
| 302 | return [CHANNEL_START_ID, 3000, 3001] |
| 303 | |
| 304 | def test_reasoning_not_leaked_into_content( |
| 305 | self, open_reasoning_parser, open_reasoning_tokenizer, request_obj |
| 306 | ): |
| 307 | results = _stream_tokens_batched( |
| 308 | open_reasoning_parser, |
| 309 | open_reasoning_tokenizer, |
| 310 | request_obj, |
| 311 | batch_size=1, |
| 312 | prompt_token_ids=self._prompt_ids_open_channel(), |
| 313 | ) |
| 314 | |
| 315 | reasoning, content, _ = _collect_fields(results) |
| 316 | |
| 317 | assert "Sure, the answer is 42" in reasoning, ( |
| 318 | f"Expected pre-<channel|> tokens in reasoning, got " |
| 319 | f"reasoning={reasoning!r} content={content!r}" |
| 320 | ) |
| 321 | for leaked in ("Sure", "answer", "42"): |
| 322 | assert leaked not in content, ( |
| 323 | f"Reasoning text leaked into content: {content!r}" |
| 324 | ) |
| 325 | |
| 326 | def test_post_reasoning_text_in_content( |
| 327 | self, open_reasoning_parser, open_reasoning_tokenizer, request_obj |
| 328 | ): |
| 329 | results = _stream_tokens_batched( |
| 330 | open_reasoning_parser, |
| 331 | open_reasoning_tokenizer, |
| 332 | request_obj, |
| 333 | batch_size=1, |
| 334 | prompt_token_ids=self._prompt_ids_open_channel(), |
| 335 | ) |
| 336 | |
| 337 | _, content, _ = _collect_fields(results) |
| 338 | |
| 339 | assert "Hello world" in content, ( |
| 340 | f"Post-<channel|> text missing from content: {content!r}" |
| 341 | ) |
| 342 | |
| 343 | def test_new_turn_prompt_unchanged(self, parser, mock_tokenizer, request_obj): |
| 344 | """When the prompt does NOT end in an open reasoning channel (e.g. a |
| 345 | new turn that ends with ``<|turn>model\\n``), behaviour must match |
| 346 | the existing flow — the model itself opens ``<|channel>``. |
| 347 | """ |
| 348 | results = _stream_tokens_batched( |
| 349 | parser, |
| 350 | mock_tokenizer, |
| 351 | request_obj, |
| 352 | batch_size=10, |
| 353 | # No <|channel> in the prompt tail. |
| 354 | prompt_token_ids=[9000, 9001], |
| 355 | ) |
| 356 | |
| 357 | reasoning, content, tool_calls = _collect_fields(results) |
| 358 | |
| 359 | assert "weather" in reasoning.lower(), ( |
| 360 | f"Expected reasoning about weather, got: {reasoning[:100]!r}" |
| 361 | ) |
| 362 | assert len(tool_calls) > 0, f"Tool calls missing — content={content!r}" |
| 363 | |
| 364 | |
| 365 | # ── Engine pre-initialised to REASONING + model still emits channel open ── |
| 366 | |
| 367 | _PRE_INIT_THOUGHT_GEN_SEQUENCE: list[tuple[int, str]] = [ |
| 368 | # Model naively emits the full reasoning opener even though the engine |
| 369 | # was pre-initialised to REASONING from the prompt. |
| 370 | (CHANNEL_START_ID, "<|channel>"), |
| 371 | (8000, "thought"), |
| 372 | (8001, "\n"), |
| 373 | (8002, "Reason"), |
| 374 | (8003, "ing"), |
| 375 | (8004, " body"), |
| 376 | (CHANNEL_END_ID, "<channel|>"), |
| 377 | (8005, "Final"), |
| 378 | (8006, " content"), |
| 379 | ] |
| 380 | |
| 381 | |
| 382 | class TestGemma4PreInitReasoningRobustness: |
| 383 | """Tests for the ``(REASONING, THINK_START)`` no-op transition and |
| 384 | cooperating ``thought\\n`` prefix stripping. |
| 385 | |
| 386 | Covers both engine start states: a new-turn prompt (engine starts in |
| 387 | ``CONTENT`` and the model emits its own ``<|channel>`` opener) and a |
| 388 | prompt ending inside an open ``<|channel>`` block (engine |
| 389 | pre-initialised to ``REASONING``). In both cases the ``thought\\n`` |
| 390 | prefix must be stripped, the ``<|channel>`` must not leak as text, |
| 391 | and the post-``<channel|>`` text must appear as content. |
| 392 | """ |
| 393 | |
| 394 | @pytest.fixture |
| 395 | def pre_init_tokenizer(self): |
| 396 | return _make_tokenizer(_PRE_INIT_THOUGHT_GEN_SEQUENCE) |
| 397 | |
| 398 | @pytest.fixture |
| 399 | def pre_init_parser(self, pre_init_tokenizer): |
| 400 | return Gemma4Parser( |
| 401 | pre_init_tokenizer, chat_template_kwargs={"enable_thinking": True} |
| 402 | ) |
| 403 | |
| 404 | def test_model_emitted_channel_open_after_new_turn( |
| 405 | self, pre_init_parser, pre_init_tokenizer, request_obj |
| 406 | ): |
| 407 | # Prompt ends with a ``<|turn>model\n``-style sentinel, i.e. not |
| 408 | # inside an open ``<|channel>`` block, so the engine starts in |
| 409 | # ``CONTENT`` and the model's own opener drives the transition. |
| 410 | results = _stream_tokens_batched( |
| 411 | pre_init_parser, |
| 412 | pre_init_tokenizer, |
| 413 | request_obj, |
| 414 | batch_size=1, |
| 415 | prompt_token_ids=[NEW_TURN_ID, 9100, 9101], |
| 416 | ) |
| 417 | |
| 418 | reasoning, content, _ = _collect_fields(results) |
| 419 | |
| 420 | # ``thought\n`` prefix must be stripped from reasoning. |
| 421 | assert reasoning.startswith("Reason"), ( |
| 422 | f"thought\\n prefix leaked into reasoning: {reasoning!r}" |
| 423 | ) |
| 424 | assert "thought\n" not in reasoning, ( |
| 425 | f"thought\\n prefix leaked into reasoning: {reasoning!r}" |
| 426 | ) |
| 427 | assert "Reasoning body" in reasoning, f"Reasoning body missing: {reasoning!r}" |
| 428 | |
| 429 | # The redundant ``<|channel>`` opener must not appear as text. |
| 430 | assert "<|channel>" not in content, ( |
| 431 | f"<|channel> leaked into content: {content!r}" |
| 432 | ) |
| 433 | assert "<|channel>" not in reasoning, ( |
| 434 | f"<|channel> leaked into reasoning: {reasoning!r}" |
| 435 | ) |
| 436 | |
| 437 | # Post-``<channel|>`` text must appear as content. |
| 438 | assert "Final content" in content, ( |
| 439 | f"Post-<channel|> text missing from content: {content!r}" |
| 440 | ) |
| 441 | |
| 442 | def test_redundant_channel_open_swallowed_after_open_channel_prompt( |
| 443 | self, pre_init_parser, pre_init_tokenizer, request_obj |
| 444 | ): |
| 445 | # Prompt already ends inside an open ``<|channel>`` block. Engine |
| 446 | # is pre-initialised to ``REASONING`` via the start-token check. |
| 447 | # Even if the model redundantly re-emits ``<|channel>thought\n``, |
| 448 | # the no-op transition + prefix stripping must keep output clean. |
| 449 | results = _stream_tokens_batched( |
| 450 | pre_init_parser, |
| 451 | pre_init_tokenizer, |
| 452 | request_obj, |
| 453 | batch_size=1, |
| 454 | prompt_token_ids=[CHANNEL_START_ID, 3000, 3001], |
| 455 | ) |
| 456 | |
| 457 | reasoning, content, _ = _collect_fields(results) |
| 458 | |
| 459 | assert "<|channel>" not in content, ( |
| 460 | f"<|channel> leaked into content: {content!r}" |
| 461 | ) |
| 462 | assert "thought\n" not in reasoning, ( |
| 463 | f"thought\\n prefix leaked into reasoning: {reasoning!r}" |
| 464 | ) |
| 465 | assert "Reasoning body" in reasoning |
| 466 | assert "Final content" in content |
| 467 | |
| 468 | |
| 469 | # ── Channel-less direct answer (no reasoning markers at all) ────────── |
| 470 | |
| 471 | _PLAIN_ANSWER_TEXT = "This is a direct final answer without channel markers." |
| 472 | _PLAIN_ANSWER_TOKENS: list[tuple[int, str]] = [] |
| 473 | for _i, _word in enumerate(_PLAIN_ANSWER_TEXT.split(" ")): |
| 474 | _prefix = " " if _i > 0 else "" |
| 475 | _PLAIN_ANSWER_TOKENS.append((7000 + _i, _prefix + _word)) |
| 476 | |
| 477 | |
| 478 | class TestGemma4ChannelLessOutputConsistency: |
| 479 | """Streaming and non-streaming must classify channel-less output |
| 480 | identically. |
| 481 | |
| 482 | With ``enable_thinking=True``, a prompt that merely starts a new model |
| 483 | turn (ends with ``<|turn>model\\n``) used to pre-initialise the |
| 484 | streaming engine to ``REASONING``, so a direct answer containing no |
| 485 | channel markers was streamed entirely as ``reasoning`` while the |
| 486 | non-streaming path returned the same text as ``content``. |
| 487 | |
| 488 | Regression test for vllm-project/vllm#48217. |
| 489 | """ |
| 490 | |
| 491 | @pytest.fixture |
| 492 | def plain_tokenizer(self): |
| 493 | return _make_tokenizer(_PLAIN_ANSWER_TOKENS) |
| 494 | |
| 495 | @pytest.fixture |
| 496 | def plain_parser(self, plain_tokenizer): |
| 497 | return Gemma4Parser( |
| 498 | plain_tokenizer, chat_template_kwargs={"enable_thinking": True} |
| 499 | ) |
| 500 | |
| 501 | def test_streaming_channel_less_output_is_content( |
| 502 | self, plain_parser, plain_tokenizer, request_obj |
| 503 | ): |
| 504 | results = _stream_tokens_batched( |
| 505 | plain_parser, |
| 506 | plain_tokenizer, |
| 507 | request_obj, |
| 508 | batch_size=1, |
| 509 | # New-turn tail: ``<|turn>model\n``. |
| 510 | prompt_token_ids=[NEW_TURN_ID, 9100, 9101], |
| 511 | ) |
| 512 | |
| 513 | reasoning, content, _ = _collect_fields(results) |
| 514 | |
| 515 | assert content == _PLAIN_ANSWER_TEXT, ( |
| 516 | f"Channel-less output missing from content: {content!r}" |
| 517 | ) |
| 518 | assert not reasoning, ( |
| 519 | f"Channel-less output misclassified as reasoning: {reasoning!r}" |
| 520 | ) |
| 521 | |
| 522 | def test_streaming_matches_non_streaming( |
| 523 | self, plain_parser, plain_tokenizer, request_obj |
| 524 | ): |
| 525 | reasoning, content = plain_parser.extract_reasoning( |
| 526 | _PLAIN_ANSWER_TEXT, request_obj |
| 527 | ) |
| 528 | assert reasoning is None |
| 529 | assert content == _PLAIN_ANSWER_TEXT |
| 530 | |
| 531 | stream_parser = Gemma4Parser( |
| 532 | plain_tokenizer, chat_template_kwargs={"enable_thinking": True} |
| 533 | ) |
| 534 | results = _stream_tokens_batched( |
| 535 | stream_parser, |
| 536 | plain_tokenizer, |
| 537 | request_obj, |
| 538 | batch_size=1, |
| 539 | prompt_token_ids=[NEW_TURN_ID, 9100, 9101], |
| 540 | ) |
| 541 | stream_reasoning, stream_content, _ = _collect_fields(results) |
| 542 | |
| 543 | assert (stream_reasoning or None) == reasoning |
| 544 | assert (stream_content or None) == content |
| 545 | |
| 546 | |
| 547 | # ── Second model output: two tool calls with holdback ──────────────── |
| 548 | |
| 549 | REASONING_TEXT_2 = ( |
| 550 | "The user wants me to:\n" |
| 551 | "1. Perform some reasoning.\n" |
| 552 | "2. Call a tool to fetch the hostname.\n" |
| 553 | "3. Call a tool to fetch the current date.\n" |
| 554 | "\n" |
| 555 | "Since I am an AI assistant (opencode), I can use the " |
| 556 | "`bash` tool to execute commands.\n" |
| 557 | "To get the hostname, I can run `hostname`.\n" |
| 558 | "To get the current date, I can run `date`.\n" |
| 559 | "\n" |
| 560 | "I should do this in a single response with " |
| 561 | "multiple tool calls for efficiency." |
| 562 | ) |
| 563 | |
| 564 | _reasoning_words_2 = REASONING_TEXT_2.split(" ") |
| 565 | _R2_TOKEN_START = 4000 |
| 566 | REASONING_TOKENS_2: list[tuple[int, str]] = [] |
| 567 | for i, word in enumerate(_reasoning_words_2): |
| 568 | prefix = " " if i > 0 else "" |
| 569 | REASONING_TOKENS_2.append((_R2_TOKEN_START + i, prefix + word)) |
| 570 | |
| 571 | TOOL_BODY_TOKENS_2A: list[tuple[int, str]] = [ |
| 572 | (5000, "call"), |
| 573 | (5001, ":"), |
| 574 | (5002, "bash"), |
| 575 | (5003, "{"), |
| 576 | (5004, "command"), |
| 577 | (5005, ":"), |
| 578 | (5006, "hostname"), |
| 579 | (5007, ","), |
| 580 | (5008, "description"), |
| 581 | (5009, ":"), |
| 582 | (5010, "Fetch the hostname of the system."), |
| 583 | (5011, "}"), |
| 584 | ] |
| 585 | |
| 586 | TOOL_BODY_TOKENS_2B: list[tuple[int, str]] = [ |
| 587 | (6000, "call"), |
| 588 | (6001, ":"), |
| 589 | (6002, "bash"), |
| 590 | (6003, "{"), |
| 591 | (6004, "command"), |
| 592 | (6005, ":"), |
| 593 | (6006, "date"), |
| 594 | (6007, ","), |
| 595 | (6008, "description"), |
| 596 | (6009, ":"), |
| 597 | (6010, "Fetch the current system date and time."), |
| 598 | (6011, "}"), |
| 599 | ] |
| 600 | |
| 601 | FULL_TOKEN_SEQUENCE_2: list[tuple[int, str]] = [] |
| 602 | FULL_TOKEN_SEQUENCE_2.append((CHANNEL_START_ID, "<|channel>")) |
| 603 | FULL_TOKEN_SEQUENCE_2.append((3000, "thought")) |
| 604 | FULL_TOKEN_SEQUENCE_2.append((3001, "\n")) |
| 605 | FULL_TOKEN_SEQUENCE_2.extend(REASONING_TOKENS_2) |
| 606 | FULL_TOKEN_SEQUENCE_2.append((CHANNEL_END_ID, "<channel|>")) |
| 607 | FULL_TOKEN_SEQUENCE_2.append((TOOL_CALL_START_ID, "<|tool_call>")) |
| 608 | FULL_TOKEN_SEQUENCE_2.extend(TOOL_BODY_TOKENS_2A[:6]) |
| 609 | FULL_TOKEN_SEQUENCE_2.append((QUOTED_ID, '<|"|>')) |
| 610 | FULL_TOKEN_SEQUENCE_2.append(TOOL_BODY_TOKENS_2A[6]) |
| 611 | FULL_TOKEN_SEQUENCE_2.append((QUOTED_ID, '<|"|>')) |
| 612 | FULL_TOKEN_SEQUENCE_2.extend(TOOL_BODY_TOKENS_2A[7:10]) |
| 613 | FULL_TOKEN_SEQUENCE_2.append((QUOTED_ID, '<|"|>')) |
| 614 | FULL_TOKEN_SEQUENCE_2.append(TOOL_BODY_TOKENS_2A[10]) |
| 615 | FULL_TOKEN_SEQUENCE_2.append((QUOTED_ID, '<|"|>')) |
| 616 | FULL_TOKEN_SEQUENCE_2.append(TOOL_BODY_TOKENS_2A[11]) |
| 617 | FULL_TOKEN_SEQUENCE_2.append((TOOL_CALL_END_ID, "<tool_call|>")) |
| 618 | FULL_TOKEN_SEQUENCE_2.append((TOOL_CALL_START_ID, "<|tool_call>")) |
| 619 | FULL_TOKEN_SEQUENCE_2.extend(TOOL_BODY_TOKENS_2B[:6]) |
| 620 | FULL_TOKEN_SEQUENCE_2.append((QUOTED_ID, '<|"|>')) |
| 621 | FULL_TOKEN_SEQUENCE_2.append(TOOL_BODY_TOKENS_2B[6]) |
| 622 | FULL_TOKEN_SEQUENCE_2.append((QUOTED_ID, '<|"|>')) |
| 623 | FULL_TOKEN_SEQUENCE_2.extend(TOOL_BODY_TOKENS_2B[7:10]) |
| 624 | FULL_TOKEN_SEQUENCE_2.append((QUOTED_ID, '<|"|>')) |
| 625 | FULL_TOKEN_SEQUENCE_2.append(TOOL_BODY_TOKENS_2B[10]) |
| 626 | FULL_TOKEN_SEQUENCE_2.append((QUOTED_ID, '<|"|>')) |
| 627 | FULL_TOKEN_SEQUENCE_2.append(TOOL_BODY_TOKENS_2B[11]) |
| 628 | FULL_TOKEN_SEQUENCE_2.append((TOOL_CALL_END_ID, "<tool_call|>")) |
| 629 | |
| 630 | |
| 631 | def _stream_tokens_with_holdback( |
| 632 | parser, |
| 633 | tokenizer, |
| 634 | request, |
| 635 | batch_size=10, |
| 636 | holdback_chars=12, |
| 637 | prompt_token_ids=None, |
| 638 | ) -> list[DeltaMessage | None]: |
| 639 | """Feed tokens in batches with simulated detokenizer holdback.""" |
| 640 | token_ids = tokenizer.encode("", add_special_tokens=False) |
| 641 | results: list[DeltaMessage | None] = [] |
| 642 | prev_safe_text = "" |
| 643 | |
| 644 | for start in range(0, len(token_ids), batch_size): |
| 645 | batch_end = min(start + batch_size, len(token_ids)) |
| 646 | batch_ids = token_ids[start:batch_end] |
| 647 | |
| 648 | full_decoded = tokenizer.decode(token_ids[:batch_end]) |
| 649 | |
| 650 | if batch_end < len(token_ids): |
| 651 | safe_len = max(0, len(full_decoded) - holdback_chars) |
| 652 | safe_text = full_decoded[:safe_len] |
| 653 | else: |
| 654 | safe_text = full_decoded |
| 655 | |
| 656 | delta_text = safe_text[len(prev_safe_text) :] |
| 657 | prev_safe_text = safe_text |
| 658 | |
| 659 | result = parser.parse_delta( |
| 660 | delta_text, |
| 661 | batch_ids, |
| 662 | request, |
| 663 | prompt_token_ids=prompt_token_ids, |
| 664 | finished=False, |
| 665 | ) |
| 666 | prompt_token_ids = None |
| 667 | results.append(result) |
| 668 | return results |
| 669 | |
| 670 | |
| 671 | class TestGemma4ReasoningTruncationWithHoldback: |
| 672 | """Reasoning text must not be truncated when detokenizer holds back text.""" |
| 673 | |
| 674 | @pytest.fixture |
| 675 | def tokenizer_2(self): |
| 676 | return _make_tokenizer(FULL_TOKEN_SEQUENCE_2) |
| 677 | |
| 678 | @pytest.fixture |
| 679 | def parser_2(self, tokenizer_2): |
| 680 | return Gemma4Parser(tokenizer_2) |
| 681 | |
| 682 | def test_reasoning_not_truncated(self, parser_2, tokenizer_2, request_obj): |
| 683 | """Reasoning must include the full text up to <channel|>.""" |
| 684 | results = _stream_tokens_with_holdback( |
| 685 | parser_2, |
| 686 | tokenizer_2, |
| 687 | request_obj, |
| 688 | batch_size=10, |
| 689 | holdback_chars=12, |
| 690 | prompt_token_ids=[], |
| 691 | ) |
| 692 | |
| 693 | reasoning, content, tool_calls = _collect_fields(results) |
| 694 | |
| 695 | assert "efficiency" in reasoning, ( |
| 696 | f"Reasoning truncated — missing 'efficiency'. " |
| 697 | f"Reasoning ends with: {reasoning[-60:]!r}" |
| 698 | ) |
| 699 | |
| 700 | def test_both_tool_calls_extracted(self, parser_2, tokenizer_2, request_obj): |
| 701 | """Both bash tool calls must be extracted.""" |
| 702 | results = _stream_tokens_with_holdback( |
| 703 | parser_2, |
| 704 | tokenizer_2, |
| 705 | request_obj, |
| 706 | batch_size=10, |
| 707 | holdback_chars=12, |
| 708 | prompt_token_ids=[], |
| 709 | ) |
| 710 | |
| 711 | _, _, tool_calls = _collect_fields(results) |
| 712 | |
| 713 | names = [ |
| 714 | tc.function.name for tc in tool_calls if tc.function and tc.function.name |
| 715 | ] |
| 716 | assert len(names) >= 2, f"Expected 2 tool calls, got {len(names)}: {names}" |
| 717 | assert names.count("bash") >= 2, f"Expected 2 bash tool calls, got {names}" |
| 718 | |
| 719 | def test_tool_call_text_not_in_content(self, parser_2, tokenizer_2, request_obj): |
| 720 | """Tool call body must not leak into content.""" |
| 721 | results = _stream_tokens_with_holdback( |
| 722 | parser_2, |
| 723 | tokenizer_2, |
| 724 | request_obj, |
| 725 | batch_size=10, |
| 726 | holdback_chars=12, |
| 727 | prompt_token_ids=[], |
| 728 | ) |
| 729 | |
| 730 | _, content, _ = _collect_fields(results) |
| 731 | |
| 732 | assert "call:" not in content, ( |
| 733 | f"Tool call text leaked into content: {content!r}" |
| 734 | ) |
| 735 | |
| 736 | |
| 737 | # ── Simple mock tokenizer for tool-only tests ──────────────────────── |
| 738 | |
| 739 | |
| 740 | @pytest.fixture |
| 741 | def tool_call_tokenizer(): |
| 742 | """Mock tokenizer with Gemma4 special token vocab.""" |
| 743 | return make_mock_tokenizer( |
| 744 | vocab={ |
| 745 | "<|tool_call>": TOOL_CALL_START_ID, |
| 746 | "<tool_call|>": TOOL_CALL_END_ID, |
| 747 | "<|channel>": CHANNEL_START_ID, |
| 748 | "<channel|>": CHANNEL_END_ID, |
| 749 | '<|"|>': QUOTED_ID, |
| 750 | }, |
| 751 | ) |
| 752 | |
| 753 | |
| 754 | @pytest.fixture |
| 755 | def tool_call_parser(tool_call_tokenizer): |
| 756 | return Gemma4Parser(tool_call_tokenizer) |
| 757 | |
| 758 | |
| 759 | # ── Non-streaming tool call extraction tests ───────────────────────── |
| 760 | |
| 761 | |
| 762 | class TestNonStreamingToolCalls: |
| 763 | """Non-streaming tool call extraction via extract_tool_calls().""" |
| 764 | |
| 765 | def test_no_tool_calls(self, tool_call_parser, mock_request): |
| 766 | result = tool_call_parser.extract_tool_calls( |
| 767 | "Hello, how can I help you today?", |
| 768 | mock_request, |
| 769 | ) |
| 770 | assert result.tools_called is False |
| 771 | assert result.tool_calls == [] |
| 772 | assert result.content == "Hello, how can I help you today?" |
| 773 | |
| 774 | def test_single_tool_call(self, tool_call_parser, mock_request): |
| 775 | text = '<|tool_call>call:get_weather{location:<|"|>London<|"|>}<tool_call|>' |
| 776 | result = tool_call_parser.extract_tool_calls(text, mock_request) |
| 777 | |
| 778 | assert result.tools_called is True |
| 779 | assert len(result.tool_calls) == 1 |
| 780 | assert result.tool_calls[0].function.name == "get_weather" |
| 781 | args = json.loads(result.tool_calls[0].function.arguments) |
| 782 | assert args == {"location": "London"} |
| 783 | |
| 784 | def test_multiple_arguments(self, tool_call_parser, mock_request): |
| 785 | text = ( |
| 786 | "<|tool_call>call:get_weather{" |
| 787 | 'location:<|"|>San Francisco<|"|>,' |
| 788 | 'unit:<|"|>celsius<|"|>}' |
| 789 | "<tool_call|>" |
| 790 | ) |
| 791 | result = tool_call_parser.extract_tool_calls(text, mock_request) |
| 792 | |
| 793 | assert result.tools_called is True |
| 794 | assert result.tool_calls[0].function.name == "get_weather" |
| 795 | args = json.loads(result.tool_calls[0].function.arguments) |
| 796 | assert args == {"location": "San Francisco", "unit": "celsius"} |
| 797 | |
| 798 | def test_text_before_tool_call(self, tool_call_parser, mock_request): |
| 799 | text = ( |
| 800 | "Let me check the weather for you. " |
| 801 | '<|tool_call>call:get_weather{location:<|"|>Paris<|"|>}' |
| 802 | "<tool_call|>" |
| 803 | ) |
| 804 | result = tool_call_parser.extract_tool_calls(text, mock_request) |
| 805 | |
| 806 | assert result.tools_called is True |
| 807 | assert result.content is not None |
| 808 | assert "Let me check the weather" in result.content |
| 809 | assert result.tool_calls[0].function.name == "get_weather" |
| 810 | |
| 811 | def test_multiple_tool_calls(self, tool_call_parser, mock_request): |
| 812 | text = ( |
| 813 | '<|tool_call>call:get_weather{location:<|"|>London<|"|>}' |
| 814 | "<tool_call|>" |
| 815 | '<|tool_call>call:get_time{location:<|"|>London<|"|>}' |
| 816 | "<tool_call|>" |
| 817 | ) |
| 818 | result = tool_call_parser.extract_tool_calls(text, mock_request) |
| 819 | |
| 820 | assert result.tools_called is True |
| 821 | assert len(result.tool_calls) == 2 |
| 822 | assert result.tool_calls[0].function.name == "get_weather" |
| 823 | assert result.tool_calls[1].function.name == "get_time" |
| 824 | |
| 825 | def test_nested_arguments(self, tool_call_parser, mock_request): |
| 826 | text = ( |
| 827 | "<|tool_call>call:complex_function{" |
| 828 | 'nested:{inner:<|"|>value<|"|>},' |
| 829 | 'list:[<|"|>a<|"|>,<|"|>b<|"|>]}' |
| 830 | "<tool_call|>" |
| 831 | ) |
| 832 | result = tool_call_parser.extract_tool_calls(text, mock_request) |
| 833 | |
| 834 | assert result.tools_called is True |
| 835 | assert result.tool_calls[0].function.name == "complex_function" |
| 836 | args = json.loads(result.tool_calls[0].function.arguments) |
| 837 | assert args == {"nested": {"inner": "value"}, "list": ["a", "b"]} |
| 838 | |
| 839 | def test_number_and_boolean(self, tool_call_parser, mock_request): |
| 840 | text = ( |
| 841 | "<|tool_call>call:set_status{" |
| 842 | "is_active:true," |
| 843 | "count:42," |
| 844 | "score:3.14}" |
| 845 | "<tool_call|>" |
| 846 | ) |
| 847 | result = tool_call_parser.extract_tool_calls(text, mock_request) |
| 848 | |
| 849 | assert result.tools_called is True |
| 850 | args = json.loads(result.tool_calls[0].function.arguments) |
| 851 | assert args == {"is_active": "true", "count": "42", "score": "3.14"} |
| 852 | |
| 853 | def test_no_arguments(self, tool_call_parser, mock_request): |
| 854 | text = "<|tool_call>call:get_status{}<tool_call|>" |
| 855 | result = tool_call_parser.extract_tool_calls(text, mock_request) |
| 856 | |
| 857 | assert result.tools_called is True |
| 858 | assert result.tool_calls[0].function.name == "get_status" |
| 859 | args = json.loads(result.tool_calls[0].function.arguments) |
| 860 | assert args == {} |
| 861 | |
| 862 | def test_hyphenated_function_name(self, tool_call_parser, mock_request): |
| 863 | text = '<|tool_call>call:get-weather{location:<|"|>London<|"|>}<tool_call|>' |
| 864 | result = tool_call_parser.extract_tool_calls(text, mock_request) |
| 865 | |
| 866 | assert result.tools_called is True |
| 867 | assert result.tool_calls[0].function.name == "get-weather" |
| 868 | |
| 869 | def test_dotted_function_name(self, tool_call_parser, mock_request): |
| 870 | text = '<|tool_call>call:weather.get{location:<|"|>London<|"|>}<tool_call|>' |
| 871 | result = tool_call_parser.extract_tool_calls(text, mock_request) |
| 872 | |
| 873 | assert result.tools_called is True |
| 874 | assert result.tool_calls[0].function.name == "weather.get" |
| 875 | |
| 876 | |
| 877 | # ── Streaming tool call edge-case tests ────────────────────────────── |
| 878 | |
| 879 | |
| 880 | class TestStreamingToolCallEdgeCases: |
| 881 | """Streaming tool call extraction via extract_tool_calls_streaming().""" |
| 882 | |
| 883 | def test_basic_streaming(self, tool_call_parser, mock_request): |
| 884 | chunks = [ |
| 885 | "<|tool_call>", |
| 886 | "call:get_weather{", |
| 887 | 'location:<|"|>Paris', |
| 888 | ", France", |
| 889 | '<|"|>}', |
| 890 | "<tool_call|>", |
| 891 | ] |
| 892 | |
| 893 | results = simulate_tool_streaming(tool_call_parser, mock_request, chunks) |
| 894 | |
| 895 | name = collect_function_name(results) |
| 896 | assert name == "get_weather" |
| 897 | |
| 898 | args_text = collect_tool_arguments(results) |
| 899 | assert args_text |
| 900 | parsed = json.loads(args_text) |
| 901 | assert parsed == {"location": "Paris, France"} |
| 902 | |
| 903 | def test_streaming_multi_arg(self, tool_call_parser, mock_request): |
| 904 | chunks = [ |
| 905 | "<|tool_call>", |
| 906 | "call:get_weather{", |
| 907 | 'location:<|"|>Tokyo<|"|>,', |
| 908 | 'unit:<|"|>celsius<|"|>}', |
| 909 | "<tool_call|>", |
| 910 | ] |
| 911 | |
| 912 | results = simulate_tool_streaming(tool_call_parser, mock_request, chunks) |
| 913 | |
| 914 | name = collect_function_name(results) |
| 915 | assert name == "get_weather" |
| 916 | |
| 917 | args_text = collect_tool_arguments(results) |
| 918 | assert args_text |
| 919 | parsed = json.loads(args_text) |
| 920 | assert parsed == {"location": "Tokyo", "unit": "celsius"} |
| 921 | |
| 922 | def test_streaming_no_extra_brace(self, tool_call_parser, mock_request): |
| 923 | chunks = [ |
| 924 | "<|tool_call>", |
| 925 | "call:get_weather{", |
| 926 | 'location:<|"|>London<|"|>}', |
| 927 | "<tool_call|>", |
| 928 | ] |
| 929 | |
| 930 | results = simulate_tool_streaming(tool_call_parser, mock_request, chunks) |
| 931 | args_text = collect_tool_arguments(results) |
| 932 | assert args_text |
| 933 | |
| 934 | parsed = json.loads(args_text) |
| 935 | assert parsed == {"location": "London"} |
| 936 | assert args_text.count("}") <= 1 |
| 937 | |
| 938 | def test_streaming_text_before_tool(self, tool_call_parser, mock_request): |
| 939 | chunks = [ |
| 940 | "Let me check ", |
| 941 | "the weather. ", |
| 942 | "<|tool_call>", |
| 943 | "call:get_weather{", |
| 944 | 'location:<|"|>London<|"|>}', |
| 945 | "<tool_call|>", |
| 946 | ] |
| 947 | |
| 948 | results = simulate_tool_streaming(tool_call_parser, mock_request, chunks) |
| 949 | assert collect_content(results).strip().startswith("Let me check") |
| 950 | |
| 951 | def test_streaming_numeric_args(self, tool_call_parser, mock_request): |
| 952 | chunks = [ |
| 953 | "<|tool_call>", |
| 954 | "call:set_config{", |
| 955 | "count:42,", |
| 956 | "active:true}", |
| 957 | "<tool_call|>", |
| 958 | ] |
| 959 | |
| 960 | results = simulate_tool_streaming(tool_call_parser, mock_request, chunks) |
| 961 | args_text = collect_tool_arguments(results) |
| 962 | if args_text: |
| 963 | parsed = json.loads(args_text) |
| 964 | assert parsed["count"] == "42" |
| 965 | assert parsed["active"] == "true" |
| 966 | |
| 967 | def test_streaming_empty_args(self, tool_call_parser, mock_request): |
| 968 | chunks = [ |
| 969 | "<|tool_call>", |
| 970 | "call:get_status{}", |
| 971 | "<tool_call|>", |
| 972 | ] |
| 973 | |
| 974 | results = simulate_tool_streaming(tool_call_parser, mock_request, chunks) |
| 975 | name = collect_function_name(results) |
| 976 | assert name == "get_status" |
| 977 | |
| 978 | def test_streaming_split_delimiter(self, tool_call_parser, mock_request): |
| 979 | """Partial <|"|> delimiter must not leak into JSON.""" |
| 980 | chunks = [ |
| 981 | "<|tool_call>", |
| 982 | "call:todowrite{", |
| 983 | 'content:<|"|>Buy milk<|', |
| 984 | '"|>}', |
| 985 | "<tool_call|>", |
| 986 | ] |
| 987 | |
| 988 | results = simulate_tool_streaming(tool_call_parser, mock_request, chunks) |
| 989 | args_text = collect_tool_arguments(results) |
| 990 | assert args_text |
| 991 | parsed = json.loads(args_text) |
| 992 | assert parsed["content"] == "Buy milk" |
| 993 | assert "<|" not in args_text |
| 994 | |
| 995 | def test_streaming_bool_split(self, tool_call_parser, mock_request): |
| 996 | chunks = [ |
| 997 | "<|tool_call>", |
| 998 | "call:search{input:{all:t", |
| 999 | "rue}}", |
| 1000 | "<tool_call|>", |
| 1001 | ] |
| 1002 | |
| 1003 | results = simulate_tool_streaming(tool_call_parser, mock_request, chunks) |
| 1004 | args_text = collect_tool_arguments(results) |
| 1005 | assert args_text |
| 1006 | parsed = json.loads(args_text) |
| 1007 | assert parsed["input"]["all"] == "true" |
| 1008 | |
| 1009 | def test_streaming_number_split(self, tool_call_parser, mock_request): |
| 1010 | chunks = [ |
| 1011 | "<|tool_call>", |
| 1012 | "call:set{count:4", |
| 1013 | "2}", |
| 1014 | "<tool_call|>", |
| 1015 | ] |
| 1016 | |
| 1017 | results = simulate_tool_streaming(tool_call_parser, mock_request, chunks) |
| 1018 | args_text = collect_tool_arguments(results) |
| 1019 | assert args_text |
| 1020 | parsed = json.loads(args_text) |
| 1021 | assert parsed["count"] == "42" |
| 1022 | |
| 1023 | def test_streaming_trailing_bare_bool(self, tool_call_parser, mock_request): |
| 1024 | chunks = [ |
| 1025 | "<|tool_call>", |
| 1026 | "call:Edit{", |
| 1027 | 'file_path:<|"|>src/env.py<|"|>,', |
| 1028 | 'old_string:<|"|>old_val<|"|>,', |
| 1029 | 'new_string:<|"|>new_val<|"|>,', |
| 1030 | "replace_all:", |
| 1031 | "false}", |
| 1032 | "<tool_call|>", |
| 1033 | ] |
| 1034 | |
| 1035 | results = simulate_tool_streaming(tool_call_parser, mock_request, chunks) |
| 1036 | args_text = collect_tool_arguments(results) |
| 1037 | assert args_text |
| 1038 | |
| 1039 | parsed = json.loads(args_text) |
| 1040 | assert parsed == { |
| 1041 | "file_path": "src/env.py", |
| 1042 | "old_string": "old_val", |
| 1043 | "new_string": "new_val", |
| 1044 | "replace_all": "false", |
| 1045 | } |
| 1046 | |
| 1047 | assert args_text.count("replace_all") == 1 |
| 1048 | |
| 1049 | |
| 1050 | # ── Non-streaming reasoning + tool call extraction tests ────────── |
| 1051 | |
| 1052 | |
| 1053 | class TestNonStreamingReasoningPlusToolCalls: |
| 1054 | """Non-streaming extraction with reasoning + tool calls.""" |
| 1055 | |
| 1056 | def test_extract_tool_calls_from_full_text(self, parser, request_obj): |
| 1057 | """extract_tool_calls on full model output must find tools.""" |
| 1058 | model_output = FULL_MODEL_OUTPUT |
| 1059 | result = parser.extract_tool_calls(model_output, request_obj) |
| 1060 | |
| 1061 | assert result.tools_called is True |
| 1062 | assert len(result.tool_calls) == 1 |
| 1063 | assert result.tool_calls[0].function.name == "get_current_weather" |
| 1064 | args = json.loads(result.tool_calls[0].function.arguments) |
| 1065 | assert args["city"] == "Dallas" |
| 1066 | assert args["state"] == "TX" |
| 1067 | assert args["unit"] == "fahrenheit" |
| 1068 | |
| 1069 | def test_extract_reasoning_from_full_text(self, parser, request_obj): |
| 1070 | """extract_reasoning on full model output must find reasoning.""" |
| 1071 | model_output = FULL_MODEL_OUTPUT |
| 1072 | reasoning, content = parser.extract_reasoning(model_output, request_obj) |
| 1073 | |
| 1074 | assert reasoning is not None |
| 1075 | assert "weather" in reasoning.lower() |
| 1076 | assert not reasoning.startswith("thought") |
| 1077 | |
| 1078 | def test_bug_report_scenario(self, tool_call_parser, mock_request): |
| 1079 | """Exact scenario from the bug report: get_weather for Raleigh.""" |
| 1080 | model_output = ( |
| 1081 | "<|channel>thought\n" |
| 1082 | 'The user wants to get the weather for "Raleigh". ' |
| 1083 | "I should use the `get_weather` tool and pass " |
| 1084 | '"Raleigh" as the `city` argument.' |
| 1085 | "<channel|>" |
| 1086 | '<|tool_call>call:get_weather{city:<|"|>Raleigh<|"|>}' |
| 1087 | "<tool_call|>" |
| 1088 | ) |
| 1089 | result = tool_call_parser.extract_tool_calls(model_output, mock_request) |
| 1090 | |
| 1091 | assert result.tools_called is True, ( |
| 1092 | f"No tool calls found. content={result.content!r}" |
| 1093 | ) |
| 1094 | assert result.tool_calls[0].function.name == "get_weather" |
| 1095 | args = json.loads(result.tool_calls[0].function.arguments) |
| 1096 | assert args["city"] == "Raleigh" |
| 1097 | |
| 1098 | def test_both_extractions_independent(self, parser, request_obj): |
| 1099 | """Calling extract_reasoning then extract_tool_calls on the same |
| 1100 | parser instance should both work (each resets the engine).""" |
| 1101 | model_output = FULL_MODEL_OUTPUT |
| 1102 | |
| 1103 | reasoning, _ = parser.extract_reasoning(model_output, request_obj) |
| 1104 | result = parser.extract_tool_calls(model_output, request_obj) |
| 1105 | |
| 1106 | assert reasoning is not None |
| 1107 | assert "weather" in reasoning.lower() |
| 1108 | assert result.tools_called is True |
| 1109 | assert result.tool_calls[0].function.name == "get_current_weather" |
| 1110 | |
| 1111 | |
| 1112 | class TestAdapterExtractReasoning: |
| 1113 | """The reasoning adapter's extract_reasoning uses skip_tool_parsing |
| 1114 | so tool call text is preserved as content for the tool adapter.""" |
| 1115 | |
| 1116 | @pytest.fixture |
| 1117 | def adapter(self, mock_tokenizer): |
| 1118 | from vllm.parser.engine.adapters import make_adapters |
| 1119 | |
| 1120 | reasoning_cls, _ = make_adapters(Gemma4Parser) |
| 1121 | return reasoning_cls(mock_tokenizer) |
| 1122 | |
| 1123 | def test_preserves_tool_text_in_content(self, adapter, request_obj): |
| 1124 | """Tool call markers must appear in content after extraction.""" |
| 1125 | reasoning, content = adapter.extract_reasoning(FULL_MODEL_OUTPUT, request_obj) |
| 1126 | |
| 1127 | assert reasoning is not None |
| 1128 | assert "weather" in reasoning.lower() |
| 1129 | assert content is not None |
| 1130 | assert "<|tool_call>" in content |
| 1131 | assert "<tool_call|>" in content |
| 1132 | assert "get_current_weather" in content |
| 1133 | |
| 1134 | def test_skip_tool_parsing_restored_after_extraction(self, adapter, request_obj): |
| 1135 | """skip_tool_parsing must be restored to its prior value.""" |
| 1136 | engine = adapter._parser_engine._engine |
| 1137 | assert engine.skip_tool_parsing is False |
| 1138 | adapter.extract_reasoning(FULL_MODEL_OUTPUT, request_obj) |
| 1139 | assert engine.skip_tool_parsing is False |
| 1140 | |
| 1141 | def test_no_reasoning_returns_none(self, adapter, request_obj): |
| 1142 | """Content-only text returns (None, content).""" |
| 1143 | text = "Hello world, no thinking here." |
| 1144 | reasoning, content = adapter.extract_reasoning(text, request_obj) |
| 1145 | assert reasoning is None |
| 1146 | assert content == text |
| 1147 | |
| 1148 | |
| 1149 | # ── Schema-aware type coercion during streaming ──────────────────── |
| 1150 | |
| 1151 | |
| 1152 | class TestGemma4SchemaAwareTypeCoercion: |
| 1153 | """Verify that streaming and non-streaming produce identical |
| 1154 | type-fixed arguments when tool schemas declare string parameters |
| 1155 | but the model outputs bare numbers/booleans.""" |
| 1156 | |
| 1157 | @pytest.fixture |
| 1158 | def tools(self): |
| 1159 | from vllm.entrypoints.openai.chat_completion.protocol import ( |
| 1160 | ChatCompletionToolsParam, |
| 1161 | ) |
| 1162 | |
| 1163 | return [ |
| 1164 | ChatCompletionToolsParam( |
| 1165 | type="function", |
| 1166 | function={ |
| 1167 | "name": "update_record", |
| 1168 | "parameters": { |
| 1169 | "type": "object", |
| 1170 | "properties": { |
| 1171 | "zipcode": {"type": "string"}, |
| 1172 | "count": {"type": "integer"}, |
| 1173 | }, |
| 1174 | }, |
| 1175 | }, |
| 1176 | ) |
| 1177 | ] |
| 1178 | |
| 1179 | @pytest.fixture |
| 1180 | def parser_with_tools(self, tool_call_tokenizer, tools): |
| 1181 | return Gemma4Parser(tool_call_tokenizer, tools=tools) |
| 1182 | |
| 1183 | def test_streaming_string_param_not_coerced(self, parser_with_tools, mock_request): |
| 1184 | """A numeric value for a string-typed param must remain a string |
| 1185 | in the streamed output, matching the non-streaming result.""" |
| 1186 | chunks = [ |
| 1187 | "<|tool_call>", |
| 1188 | "call:update_record{", |
| 1189 | "zipcode:12345}", |
| 1190 | "<tool_call|>", |
| 1191 | ] |
| 1192 | |
| 1193 | results = simulate_tool_streaming(parser_with_tools, mock_request, chunks) |
| 1194 | args_text = collect_tool_arguments(results) |
| 1195 | parsed = json.loads(args_text) |
| 1196 | assert parsed["zipcode"] == "12345" |
| 1197 | |
| 1198 | def test_streaming_mixed_types(self, parser_with_tools, mock_request): |
| 1199 | """String params get type-fixed, integer params stay integers.""" |
| 1200 | chunks = [ |
| 1201 | "<|tool_call>", |
| 1202 | "call:update_record{", |
| 1203 | "zipcode:90210,", |
| 1204 | "count:42}", |
| 1205 | "<tool_call|>", |
| 1206 | ] |
| 1207 | |
| 1208 | results = simulate_tool_streaming(parser_with_tools, mock_request, chunks) |
| 1209 | args_text = collect_tool_arguments(results) |
| 1210 | parsed = json.loads(args_text) |
| 1211 | assert parsed["zipcode"] == "90210" |
| 1212 | assert parsed["count"] == 42 |
| 1213 | |
| 1214 | def test_streaming_matches_non_streaming(self, parser_with_tools, mock_request): |
| 1215 | """Concatenated streaming deltas must produce the same arguments |
| 1216 | as non-streaming extraction.""" |
| 1217 | text = "<|tool_call>call:update_record{zipcode:12345}<tool_call|>" |
| 1218 | |
| 1219 | non_streaming = parser_with_tools.extract_tool_calls(text, mock_request) |
| 1220 | ns_args = json.loads(non_streaming.tool_calls[0].function.arguments) |
| 1221 | |
| 1222 | chunks = [ |
| 1223 | "<|tool_call>", |
| 1224 | "call:update_record{", |
| 1225 | "zipcode:1234", |
| 1226 | "5}", |
| 1227 | "<tool_call|>", |
| 1228 | ] |
| 1229 | results = simulate_tool_streaming(parser_with_tools, mock_request, chunks) |
| 1230 | s_args = json.loads(collect_tool_arguments(results)) |
| 1231 | |
| 1232 | assert s_args == ns_args |
| 1233 | |
| 1234 | |
| 1235 | class TestGemma4SchemaCoercionBoolNumberNull: |
| 1236 | """Verify that _fix_arg_types coerces string values to non-string |
| 1237 | schema types for the Gemma4 parser.""" |
| 1238 | |
| 1239 | @pytest.fixture |
| 1240 | def tools(self): |
| 1241 | from vllm.entrypoints.openai.chat_completion.protocol import ( |
| 1242 | ChatCompletionToolsParam, |
| 1243 | ) |
| 1244 | |
| 1245 | return [ |
| 1246 | ChatCompletionToolsParam( |
| 1247 | type="function", |
| 1248 | function={ |
| 1249 | "name": "configure", |
| 1250 | "parameters": { |
| 1251 | "type": "object", |
| 1252 | "properties": { |
| 1253 | "enabled": {"type": "boolean"}, |
| 1254 | "ratio": {"type": "number"}, |
| 1255 | "label": {"type": "string"}, |
| 1256 | "value": {"type": ["string", "null"]}, |
| 1257 | }, |
| 1258 | }, |
| 1259 | }, |
| 1260 | ) |
| 1261 | ] |
| 1262 | |
| 1263 | @pytest.fixture |
| 1264 | def parser_with_tools(self, tool_call_tokenizer, tools): |
| 1265 | return Gemma4Parser(tool_call_tokenizer, tools=tools) |
| 1266 | |
| 1267 | def test_bool_param_coerced(self, parser_with_tools, mock_request): |
| 1268 | text = "<|tool_call>call:configure{enabled:true}<tool_call|>" |
| 1269 | result = parser_with_tools.extract_tool_calls(text, mock_request) |
| 1270 | args = json.loads(result.tool_calls[0].function.arguments) |
| 1271 | assert args["enabled"] is True |
| 1272 | assert isinstance(args["enabled"], bool) |
| 1273 | |
| 1274 | def test_number_whole_normalized(self, parser_with_tools, mock_request): |
| 1275 | text = "<|tool_call>call:configure{ratio:5.0}<tool_call|>" |
| 1276 | result = parser_with_tools.extract_tool_calls(text, mock_request) |
| 1277 | args = json.loads(result.tool_calls[0].function.arguments) |
| 1278 | assert args["ratio"] == 5 |
| 1279 | assert isinstance(args["ratio"], int) |
| 1280 | |
| 1281 | def test_null_coerced_when_nullable(self, parser_with_tools, mock_request): |
| 1282 | text = "<|tool_call>call:configure{value:null}<tool_call|>" |
| 1283 | result = parser_with_tools.extract_tool_calls(text, mock_request) |
| 1284 | args = json.loads(result.tool_calls[0].function.arguments) |
| 1285 | assert args["value"] is None |
| 1286 | |
| 1287 | def test_null_stays_string_without_null_schema( |
| 1288 | self, parser_with_tools, mock_request |
| 1289 | ): |
| 1290 | text = "<|tool_call>call:configure{label:null}<tool_call|>" |
| 1291 | result = parser_with_tools.extract_tool_calls(text, mock_request) |
| 1292 | args = json.loads(result.tool_calls[0].function.arguments) |
| 1293 | assert args["label"] == "null" |
| 1294 | assert isinstance(args["label"], str) |
| 1295 | |
| 1296 | def test_streaming_type_stability(self, parser_with_tools, mock_request): |
| 1297 | """Values streamed incrementally must not cause prefix |
| 1298 | incompatibility when types are coerced.""" |
| 1299 | text = ( |
| 1300 | "<|tool_call>call:configure{" |
| 1301 | "enabled:true," |
| 1302 | "ratio:3.14," |
| 1303 | "label:hello}" |
| 1304 | "<tool_call|>" |
| 1305 | ) |
| 1306 | non_stream = parser_with_tools.extract_tool_calls(text, mock_request) |
| 1307 | ns_args = json.loads(non_stream.tool_calls[0].function.arguments) |
| 1308 | |
| 1309 | chunks = [ |
| 1310 | "<|tool_call>", |
| 1311 | "call:configure{", |
| 1312 | "enabled:true,", |
| 1313 | "ratio:3.14,", |
| 1314 | "label:hello}", |
| 1315 | "<tool_call|>", |
| 1316 | ] |
| 1317 | results = simulate_tool_streaming(parser_with_tools, mock_request, chunks) |
| 1318 | s_args = json.loads(collect_tool_arguments(results)) |
| 1319 | |
| 1320 | assert s_args == ns_args |
| 1321 | assert ns_args == { |
| 1322 | "enabled": True, |
| 1323 | "ratio": pytest.approx(3.14), |
| 1324 | "label": "hello", |
| 1325 | } |
| 1326 | |
| 1327 | |
| 1328 | class TestGemma4NestedSchemaCoercion: |
| 1329 | """Verify that _fix_arg_types recurses into nested Gemma4 objects.""" |
| 1330 | |
| 1331 | @pytest.fixture |
| 1332 | def tools(self): |
| 1333 | from vllm.entrypoints.openai.chat_completion.protocol import ( |
| 1334 | ChatCompletionToolsParam, |
| 1335 | ) |
| 1336 | |
| 1337 | return [ |
| 1338 | ChatCompletionToolsParam( |
| 1339 | type="function", |
| 1340 | function={ |
| 1341 | "name": "search", |
| 1342 | "parameters": { |
| 1343 | "type": "object", |
| 1344 | "properties": { |
| 1345 | "query": {"type": "string"}, |
| 1346 | "filters": { |
| 1347 | "type": "object", |
| 1348 | "properties": { |
| 1349 | "language": {"type": "string"}, |
| 1350 | "min_stars": {"type": "integer"}, |
| 1351 | }, |
| 1352 | }, |
| 1353 | }, |
| 1354 | }, |
| 1355 | }, |
| 1356 | ) |
| 1357 | ] |
| 1358 | |
| 1359 | @pytest.fixture |
| 1360 | def parser_with_tools(self, tool_call_tokenizer, tools): |
| 1361 | return Gemma4Parser(tool_call_tokenizer, tools=tools) |
| 1362 | |
| 1363 | def test_nested_object_coerced(self, parser_with_tools, mock_request): |
| 1364 | text = ( |
| 1365 | "<|tool_call>call:search{" |
| 1366 | 'query:<|"|>vllm<|"|>,' |
| 1367 | "filters:{language:python,min_stars:100}}" |
| 1368 | "<tool_call|>" |
| 1369 | ) |
| 1370 | result = parser_with_tools.extract_tool_calls(text, mock_request) |
| 1371 | args = json.loads(result.tool_calls[0].function.arguments) |
| 1372 | assert args["query"] == "vllm" |
| 1373 | assert args["filters"]["language"] == "python" |
| 1374 | assert args["filters"]["min_stars"] == 100 |
| 1375 | assert isinstance(args["filters"]["min_stars"], int) |
| 1376 | |
| 1377 | |
| 1378 | # ── Tests for bare "thought" without channel opener ────────────────── |
| 1379 | |
| 1380 | BARE_THOUGHT_SEQUENCE: list[tuple[int, str]] = [] |
| 1381 | BARE_THOUGHT_SEQUENCE.append((3000, "thought")) |
| 1382 | BARE_THOUGHT_SEQUENCE.append((3001, "\n")) |
| 1383 | BARE_THOUGHT_SEQUENCE.extend(REASONING_TOKENS) |
| 1384 | BARE_THOUGHT_SEQUENCE.append((CHANNEL_END_ID, "<channel|>")) |
| 1385 | BARE_THOUGHT_SEQUENCE.append((TOOL_CALL_START_ID, "<|tool_call>")) |
| 1386 | BARE_THOUGHT_SEQUENCE.extend(TOOL_BODY_TOKENS[:4]) |
| 1387 | BARE_THOUGHT_SEQUENCE.extend(TOOL_BODY_TOKENS[4:6]) |
| 1388 | BARE_THOUGHT_SEQUENCE.append((QUOTED_ID, '<|"|>')) |
| 1389 | BARE_THOUGHT_SEQUENCE.append(TOOL_BODY_TOKENS[6]) # Dallas |
| 1390 | BARE_THOUGHT_SEQUENCE.append((QUOTED_ID, '<|"|>')) |
| 1391 | BARE_THOUGHT_SEQUENCE.append(TOOL_BODY_TOKENS[15]) # } |
| 1392 | BARE_THOUGHT_SEQUENCE.append((TOOL_CALL_END_ID, "<tool_call|>")) |
| 1393 | |
| 1394 | |
| 1395 | class TestBareThoughtWithoutChannelOpener: |
| 1396 | """When the model omits <|channel> and starts with bare ``thought``, |
| 1397 | the parser should auto-inject the channel opener so reasoning is |
| 1398 | captured correctly.""" |
| 1399 | |
| 1400 | @pytest.fixture |
| 1401 | def bare_thought_tokenizer(self): |
| 1402 | return _make_tokenizer(BARE_THOUGHT_SEQUENCE) |
| 1403 | |
| 1404 | @pytest.fixture |
| 1405 | def bare_thought_parser(self, bare_thought_tokenizer): |
| 1406 | return Gemma4Parser(bare_thought_tokenizer) |
| 1407 | |
| 1408 | def test_bare_thought_reasoning_then_tool_call( |
| 1409 | self, bare_thought_parser, bare_thought_tokenizer, request_obj |
| 1410 | ): |
| 1411 | results = _stream_tokens_batched( |
| 1412 | bare_thought_parser, |
| 1413 | bare_thought_tokenizer, |
| 1414 | request_obj, |
| 1415 | batch_size=1, |
| 1416 | prompt_token_ids=[], |
| 1417 | ) |
| 1418 | reasoning, content, tool_calls = _collect_fields(results) |
| 1419 | |
| 1420 | assert reasoning == REASONING_TEXT |
| 1421 | assert content == "" |
| 1422 | assert len(tool_calls) > 0 |
| 1423 | names = [ |
| 1424 | tc.function.name for tc in tool_calls if tc.function and tc.function.name |
| 1425 | ] |
| 1426 | assert "get_current_weather" in names |
| 1427 | |
| 1428 | def test_bare_thought_larger_batches( |
| 1429 | self, bare_thought_parser, bare_thought_tokenizer, request_obj |
| 1430 | ): |
| 1431 | results = _stream_tokens_batched( |
| 1432 | bare_thought_parser, |
| 1433 | bare_thought_tokenizer, |
| 1434 | request_obj, |
| 1435 | batch_size=10, |
| 1436 | prompt_token_ids=[], |
| 1437 | ) |
| 1438 | reasoning, content, tool_calls = _collect_fields(results) |
| 1439 | |
| 1440 | assert reasoning == REASONING_TEXT |
| 1441 | assert content == "" |
| 1442 | assert len(tool_calls) > 0 |
| 1443 | |
| 1444 | def test_normal_content_not_classified_as_reasoning(self, request_obj): |
| 1445 | content_seq: list[tuple[int, str]] = [ |
| 1446 | (6000, "The"), |
| 1447 | (6001, " answer"), |
| 1448 | (6002, " is"), |
| 1449 | (6003, " 42."), |
| 1450 | ] |
| 1451 | tokenizer = _make_tokenizer(content_seq) |
| 1452 | parser = Gemma4Parser(tokenizer) |
| 1453 | |
| 1454 | results = _stream_tokens_batched( |
| 1455 | parser, |
| 1456 | tokenizer, |
| 1457 | request_obj, |
| 1458 | batch_size=2, |
| 1459 | prompt_token_ids=[], |
| 1460 | ) |
| 1461 | reasoning, content, tool_calls = _collect_fields(results) |
| 1462 | |
| 1463 | assert reasoning == "" |
| 1464 | assert content == "The answer is 42." |
| 1465 | assert len(tool_calls) == 0 |
| 1466 | |
| 1467 | def test_bare_thought_token_at_end_of_stream(self, request_obj): |
| 1468 | """When the stream ends with just "thought" (no \\n), the parser |
| 1469 | should treat it as the thought prefix token, not real reasoning.""" |
| 1470 | seq: list[tuple[int, str]] = [ |
| 1471 | (CHANNEL_START_ID, "<|channel>"), |
| 1472 | (3000, "thought"), |
| 1473 | ] |
| 1474 | tokenizer = _make_tokenizer(seq) |
| 1475 | parser = Gemma4Parser(tokenizer) |
| 1476 | |
| 1477 | results = _stream_tokens_batched( |
| 1478 | parser, |
| 1479 | tokenizer, |
| 1480 | request_obj, |
| 1481 | batch_size=1, |
| 1482 | prompt_token_ids=[], |
| 1483 | ) |
| 1484 | reasoning, content, tool_calls = _collect_fields(results) |
| 1485 | |
| 1486 | assert reasoning == "" |
| 1487 | assert content == "" |
| 1488 | assert len(tool_calls) == 0 |
| 1489 | |
| 1490 | |
| 1491 | # ── Regression: commas inside <|"|>-delimited string values ───────── |
| 1492 | # |
| 1493 | # _make_tokenizer sets all_special_tokens, which activates the auto-drop |
| 1494 | # mechanism in _build_drop_info. If <|"|> is not in configured_texts, |
| 1495 | # it gets silently dropped and commas inside string values become field |
| 1496 | # separators, e.g. "San Francisco, CA" → {"location": "San Francisco"}. |
| 1497 | |
| 1498 | |
| 1499 | COMMA_TOKEN_SEQUENCE: list[tuple[int, str]] = [ |
| 1500 | (TOOL_CALL_START_ID, "<|tool_call>"), |
| 1501 | (4000, "call"), |
| 1502 | (4001, ":"), |
| 1503 | (4002, "get_weather"), |
| 1504 | (4003, "{"), |
| 1505 | (4004, "location"), |
| 1506 | (4005, ":"), |
| 1507 | (QUOTED_ID, '<|"|>'), |
| 1508 | (4006, "San Francisco"), |
| 1509 | (4007, ", CA"), |
| 1510 | (QUOTED_ID, '<|"|>'), |
| 1511 | (4008, ","), |
| 1512 | (4009, "unit"), |
| 1513 | (4010, ":"), |
| 1514 | (QUOTED_ID, '<|"|>'), |
| 1515 | (4011, "celsius"), |
| 1516 | (QUOTED_ID, '<|"|>'), |
| 1517 | (4012, "}"), |
| 1518 | (TOOL_CALL_END_ID, "<tool_call|>"), |
| 1519 | ] |
| 1520 | |
| 1521 | MULTI_COMMA_TOKEN_SEQUENCE: list[tuple[int, str]] = [ |
| 1522 | (TOOL_CALL_START_ID, "<|tool_call>"), |
| 1523 | (4000, "call"), |
| 1524 | (4001, ":"), |
| 1525 | (4020, "send_message"), |
| 1526 | (4003, "{"), |
| 1527 | (4021, "destination"), |
| 1528 | (4005, ":"), |
| 1529 | (QUOTED_ID, '<|"|>'), |
| 1530 | (4022, "456 Oakwood Avenue"), |
| 1531 | (4023, ", Rivermist"), |
| 1532 | (4024, ", 83214"), |
| 1533 | (QUOTED_ID, '<|"|>'), |
| 1534 | (4012, "}"), |
| 1535 | (TOOL_CALL_END_ID, "<tool_call|>"), |
| 1536 | ] |
| 1537 | |
| 1538 | |
| 1539 | class TestCommaInStringValueRegression: |
| 1540 | """Regression: <|"|> delimiters must not be auto-dropped. |
| 1541 | |
| 1542 | When _build_drop_info discovers <|"|> as a special token and it is |
| 1543 | not in configured_texts, the delimiter is silently removed. Without |
| 1544 | it, _parse_gemma4_args treats commas inside string values as field |
| 1545 | separators. |
| 1546 | """ |
| 1547 | |
| 1548 | @pytest.fixture |
| 1549 | def comma_tokenizer(self): |
| 1550 | return _make_tokenizer(COMMA_TOKEN_SEQUENCE) |
| 1551 | |
| 1552 | @pytest.fixture |
| 1553 | def comma_parser(self, comma_tokenizer): |
| 1554 | return Gemma4Parser(comma_tokenizer) |
| 1555 | |
| 1556 | @pytest.fixture |
| 1557 | def multi_comma_tokenizer(self): |
| 1558 | return _make_tokenizer(MULTI_COMMA_TOKEN_SEQUENCE) |
| 1559 | |
| 1560 | @pytest.fixture |
| 1561 | def multi_comma_parser(self, multi_comma_tokenizer): |
| 1562 | return Gemma4Parser(multi_comma_tokenizer) |
| 1563 | |
| 1564 | def test_batched_streaming_comma_in_value( |
| 1565 | self, comma_parser, comma_tokenizer, request_obj |
| 1566 | ): |
| 1567 | results = _stream_tokens_batched( |
| 1568 | comma_parser, |
| 1569 | comma_tokenizer, |
| 1570 | request_obj, |
| 1571 | batch_size=1, |
| 1572 | prompt_token_ids=[], |
| 1573 | ) |
| 1574 | _, _, tool_calls = _collect_fields(results) |
| 1575 | assert len(tool_calls) > 0 |
| 1576 | args_text = "".join( |
| 1577 | tc.function.arguments |
| 1578 | for tc in tool_calls |
| 1579 | if tc.function and tc.function.arguments |
| 1580 | ) |
| 1581 | parsed = json.loads(args_text) |
| 1582 | assert parsed["location"] == "San Francisco, CA" |
| 1583 | assert parsed["unit"] == "celsius" |
| 1584 | |
| 1585 | def test_batched_streaming_multiple_commas( |
| 1586 | self, multi_comma_parser, multi_comma_tokenizer, request_obj |
| 1587 | ): |
| 1588 | results = _stream_tokens_batched( |
| 1589 | multi_comma_parser, |
| 1590 | multi_comma_tokenizer, |
| 1591 | request_obj, |
| 1592 | batch_size=1, |
| 1593 | prompt_token_ids=[], |
| 1594 | ) |
| 1595 | _, _, tool_calls = _collect_fields(results) |
| 1596 | assert len(tool_calls) > 0 |
| 1597 | args_text = "".join( |
| 1598 | tc.function.arguments |
| 1599 | for tc in tool_calls |
| 1600 | if tc.function and tc.function.arguments |
| 1601 | ) |
| 1602 | parsed = json.loads(args_text) |
| 1603 | assert parsed["destination"] == "456 Oakwood Avenue, Rivermist, 83214" |
| 1604 | |
| 1605 | def test_non_streaming_comma_in_value(self, comma_parser, request_obj): |
| 1606 | text = "".join(text for _, text in COMMA_TOKEN_SEQUENCE) |
| 1607 | result = comma_parser.extract_tool_calls(text, request_obj) |
| 1608 | assert result.tools_called is True |
| 1609 | args = json.loads(result.tool_calls[0].function.arguments) |
| 1610 | assert args["location"] == "San Francisco, CA" |
| 1611 | assert args["unit"] == "celsius" |
| 1612 | |
| 1613 | def test_non_streaming_multiple_commas(self, multi_comma_parser, request_obj): |
| 1614 | text = "".join(text for _, text in MULTI_COMMA_TOKEN_SEQUENCE) |
| 1615 | result = multi_comma_parser.extract_tool_calls(text, request_obj) |
| 1616 | assert result.tools_called is True |
| 1617 | args = json.loads(result.tool_calls[0].function.arguments) |
| 1618 | assert args["destination"] == "456 Oakwood Avenue, Rivermist, 83214" |
| 1619 | |
| 1620 | |
| 1621 | class TestGemma4IsReasoningEnd: |
| 1622 | @staticmethod |
| 1623 | def _parser(thinking: bool): |
| 1624 | return Gemma4Parser( |
| 1625 | _make_tokenizer(_PLAIN_ANSWER_TOKENS), |
| 1626 | chat_template_kwargs={"enable_thinking": thinking}, |
| 1627 | ) |
| 1628 | |
| 1629 | @pytest.mark.parametrize( |
| 1630 | ("ids", "thinking", "ended"), |
| 1631 | [ |
| 1632 | ([CHANNEL_START_ID, 3000, CHANNEL_END_ID], True, True), |
| 1633 | ([CHANNEL_START_ID, 3000, TOOL_CALL_START_ID], True, True), |
| 1634 | ([CHANNEL_START_ID, 3000], True, False), |
| 1635 | ([NEW_TURN_ID, 9100], True, False), |
| 1636 | ([CHANNEL_END_ID, NEW_TURN_ID, 9100], True, False), |
| 1637 | ([NEW_TURN_ID, 9100], False, True), |
| 1638 | # The bundled template's thinking-off prompt: closed channel. |
| 1639 | ([NEW_TURN_ID, 9100, CHANNEL_START_ID, 3000, CHANNEL_END_ID], False, True), |
| 1640 | # A marker still wins with thinking off: the grammar honours it. |
| 1641 | ([CHANNEL_START_ID, 3000], False, False), |
| 1642 | # A tool-call opener after the channel closed: still ended. |
| 1643 | ( |
| 1644 | [CHANNEL_START_ID, 3000, CHANNEL_END_ID, TOOL_CALL_START_ID, 9200], |
| 1645 | True, |
| 1646 | True, |
| 1647 | ), |
| 1648 | ], |
| 1649 | ) |
| 1650 | def test_is_reasoning_end(self, ids, thinking, ended): |
| 1651 | assert self._parser(thinking).is_reasoning_end(ids) is ended |