| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import json |
| 5 | |
| 6 | import pytest |
| 7 | from mistral_common.protocol.instruct.messages import ( |
| 8 | AssistantMessage, |
| 9 | ThinkChunk, |
| 10 | ToolMessage, |
| 11 | UserMessage, |
| 12 | ) |
| 13 | from mistral_common.protocol.instruct.request import InstructRequest |
| 14 | from mistral_common.protocol.instruct.tool_calls import FunctionCall, ToolCall |
| 15 | |
| 16 | from tests.reasoning.utils import run_reasoning_extraction_mistral |
| 17 | from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest |
| 18 | from vllm.parser.parser_manager import ParserManager |
| 19 | from vllm.reasoning import ReasoningParser, ReasoningParserManager |
| 20 | from vllm.tokenizers.detokenizer_utils import detokenize_incrementally |
| 21 | from vllm.tokenizers.mistral import MistralTokenizer |
| 22 | |
| 23 | _PARSER_NAME = "mistral" |
| 24 | _MODEL_V13 = "mistralai/Magistral-Small-2509" |
| 25 | _MODEL_V11 = "mistralai/Magistral-Small-2506" |
| 26 | |
| 27 | _SAMPLE_TOOLS = [ |
| 28 | { |
| 29 | "type": "function", |
| 30 | "function": { |
| 31 | "name": "get_weather", |
| 32 | "description": "Get the current weather in a city.", |
| 33 | "parameters": { |
| 34 | "type": "object", |
| 35 | "properties": {"city": {"type": "string"}}, |
| 36 | "required": ["city"], |
| 37 | }, |
| 38 | }, |
| 39 | } |
| 40 | ] |
| 41 | |
| 42 | |
| 43 | @pytest.fixture(scope="module") |
| 44 | def mistral_tokenizer() -> MistralTokenizer: |
| 45 | """v13 Magistral tokenizer with `[THINK]`/`[/THINK]` special tokens.""" |
| 46 | return MistralTokenizer.from_pretrained(_MODEL_V13) |
| 47 | |
| 48 | |
| 49 | @pytest.fixture(scope="module") |
| 50 | def mistral_v11_tokenizer() -> MistralTokenizer: |
| 51 | """v11 Magistral tokenizer using plain-text `<think>`/`</think>` tags.""" |
| 52 | return MistralTokenizer.from_pretrained(_MODEL_V11) |
| 53 | |
| 54 | |
| 55 | def _encode_v13(tokenizer: MistralTokenizer, output: str) -> list[int]: |
| 56 | """Encode output string with `[THINK]`/`[/THINK]` markers into token IDs. |
| 57 | |
| 58 | `[THINK]` and `[/THINK]` placeholders in `output` are replaced by their |
| 59 | special-token IDs; all surrounding text is encoded with the base tokenizer. |
| 60 | Mirrors the encoding helper from the deleted |
| 61 | `tests/reasoning/test_mistral_reasoning_parser.py`. |
| 62 | """ |
| 63 | think_start = "[THINK]" |
| 64 | think_end = "[/THINK]" |
| 65 | len_start = len(think_start) |
| 66 | len_end = len(think_end) |
| 67 | |
| 68 | index_start = output.find(think_start) |
| 69 | index_end = output.find(think_end) |
| 70 | |
| 71 | out: list[int] = [] |
| 72 | if index_start != -1: |
| 73 | out += tokenizer.tokenizer.encode(output[:index_start], False, False) |
| 74 | out += [tokenizer.instruct.BEGIN_THINK] |
| 75 | |
| 76 | if index_end != -1: |
| 77 | middle = output[index_start + len_start : index_end] |
| 78 | suffix = output[index_end + len_end :] |
| 79 | out += tokenizer.tokenizer.encode(middle, False, False) |
| 80 | out += [tokenizer.instruct.END_THINK] |
| 81 | out += tokenizer.tokenizer.encode(suffix, False, False) |
| 82 | else: |
| 83 | out += tokenizer.tokenizer.encode( |
| 84 | output[index_start + len_start :], False, False |
| 85 | ) |
| 86 | elif index_end != -1: |
| 87 | out += tokenizer.tokenizer.encode(output[:index_end], False, False) |
| 88 | out += [tokenizer.instruct.END_THINK] |
| 89 | out += tokenizer.tokenizer.encode(output[index_end + len_end :], False, False) |
| 90 | else: |
| 91 | out += tokenizer.tokenizer.encode(output, False, False) |
| 92 | return out |
| 93 | |
| 94 | |
| 95 | _TEST_CASES = [ |
| 96 | # v13 special-token encoding: [THINK]…[/THINK] with trailing content. |
| 97 | pytest.param( |
| 98 | False, |
| 99 | "[THINK]r[/THINK]c", |
| 100 | "r", |
| 101 | "c", |
| 102 | id="v13_basic", |
| 103 | ), |
| 104 | pytest.param( |
| 105 | True, |
| 106 | "[THINK]r[/THINK]c", |
| 107 | "r", |
| 108 | "c", |
| 109 | id="v13_basic_streaming", |
| 110 | ), |
| 111 | # No trailing content after reasoning ends. |
| 112 | pytest.param( |
| 113 | False, |
| 114 | "[THINK]r[/THINK]", |
| 115 | "r", |
| 116 | None, |
| 117 | id="v13_no_trailing_content", |
| 118 | ), |
| 119 | pytest.param( |
| 120 | True, |
| 121 | "[THINK]r[/THINK]", |
| 122 | "r", |
| 123 | None, |
| 124 | id="v13_no_trailing_content_streaming", |
| 125 | ), |
| 126 | # Multi-line reasoning and multi-line content. |
| 127 | pytest.param( |
| 128 | False, |
| 129 | "[THINK]a\nb[/THINK]c\nd", |
| 130 | "a\nb", |
| 131 | "c\nd", |
| 132 | id="v13_multiline", |
| 133 | ), |
| 134 | pytest.param( |
| 135 | True, |
| 136 | "[THINK]a\nb[/THINK]c\nd", |
| 137 | "a\nb", |
| 138 | "c\nd", |
| 139 | id="v13_multiline_streaming", |
| 140 | ), |
| 141 | # No reasoning at all: output is pure content. |
| 142 | pytest.param( |
| 143 | False, |
| 144 | "hello world", |
| 145 | None, |
| 146 | "hello world", |
| 147 | id="v13_no_reasoning", |
| 148 | ), |
| 149 | pytest.param( |
| 150 | True, |
| 151 | "hello world", |
| 152 | None, |
| 153 | "hello world", |
| 154 | id="v13_no_reasoning_streaming", |
| 155 | ), |
| 156 | # [THINK] opened but never closed; reasoning accumulates, content is None. |
| 157 | pytest.param( |
| 158 | False, |
| 159 | "[THINK]r", |
| 160 | "r", |
| 161 | None, |
| 162 | id="v13_think_no_end", |
| 163 | ), |
| 164 | pytest.param( |
| 165 | True, |
| 166 | "[THINK]r", |
| 167 | "r", |
| 168 | None, |
| 169 | id="v13_think_no_end_streaming", |
| 170 | ), |
| 171 | # Stray [/THINK] with no matching [THINK]: no reasoning, content follows. |
| 172 | pytest.param( |
| 173 | False, |
| 174 | "[/THINK]This is the rest", |
| 175 | None, |
| 176 | "This is the rest", |
| 177 | id="v13_stray_think_end", |
| 178 | ), |
| 179 | pytest.param( |
| 180 | True, |
| 181 | "[/THINK]This is the rest", |
| 182 | None, |
| 183 | "This is the rest", |
| 184 | id="v13_stray_think_end_streaming", |
| 185 | ), |
| 186 | # Content before [THINK]: leading segment joins with trailing segment. |
| 187 | pytest.param( |
| 188 | False, |
| 189 | "Before\n[THINK]r[/THINK]\nAfter", |
| 190 | "r", |
| 191 | "Before\n\nAfter", |
| 192 | id="v13_leading_content", |
| 193 | ), |
| 194 | pytest.param( |
| 195 | True, |
| 196 | "Before\n[THINK]r[/THINK]\nAfter", |
| 197 | "r", |
| 198 | "Before\n\nAfter", |
| 199 | id="v13_leading_content_streaming", |
| 200 | ), |
| 201 | # Empty output: no reasoning, no content. |
| 202 | pytest.param( |
| 203 | False, |
| 204 | "", |
| 205 | None, |
| 206 | None, |
| 207 | id="v13_empty", |
| 208 | ), |
| 209 | pytest.param( |
| 210 | True, |
| 211 | "", |
| 212 | None, |
| 213 | None, |
| 214 | id="v13_empty_streaming", |
| 215 | ), |
| 216 | ] |
| 217 | |
| 218 | |
| 219 | @pytest.mark.parametrize( |
| 220 | "streaming, output, expected_reasoning, expected_content", |
| 221 | _TEST_CASES, |
| 222 | ) |
| 223 | def test_mistral_reasoning_v13( |
| 224 | streaming: bool, |
| 225 | output: str, |
| 226 | expected_reasoning: str | None, |
| 227 | expected_content: str | None, |
| 228 | mistral_tokenizer: MistralTokenizer, |
| 229 | ) -> None: |
| 230 | output_tokens = _encode_v13(mistral_tokenizer, output) |
| 231 | parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(_PARSER_NAME)( |
| 232 | mistral_tokenizer |
| 233 | ) |
| 234 | reasoning, content = run_reasoning_extraction_mistral( |
| 235 | parser, output_tokens, streaming=streaming |
| 236 | ) |
| 237 | assert reasoning == expected_reasoning |
| 238 | assert content == expected_content |
| 239 | |
| 240 | |
| 241 | def test_system_prompt_think_ignored( |
| 242 | mistral_tokenizer: MistralTokenizer, |
| 243 | ) -> None: |
| 244 | """Streaming content extraction must not |
| 245 | include `[THINK]`/`[/THINK]` examples from the system prompt. |
| 246 | """ |
| 247 | output = "[THINK]real[/THINK]answer" |
| 248 | output_tokens = _encode_v13(mistral_tokenizer, output) |
| 249 | for streaming in (False, True): |
| 250 | parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser( |
| 251 | _PARSER_NAME |
| 252 | )(mistral_tokenizer) |
| 253 | reasoning, content = run_reasoning_extraction_mistral( |
| 254 | parser, output_tokens, streaming=streaming |
| 255 | ) |
| 256 | assert reasoning == "real" |
| 257 | assert content == "answer" |
| 258 | |
| 259 | |
| 260 | def test_reasoning_with_tool_calls( |
| 261 | mistral_tokenizer: MistralTokenizer, |
| 262 | ) -> None: |
| 263 | """Reasoning extraction with a tool call in the generated output.""" |
| 264 | tool_calls_token_id = mistral_tokenizer.get_vocab().get("[TOOL_CALLS]") |
| 265 | if tool_calls_token_id is None: |
| 266 | pytest.skip("Tokenizer does not have [TOOL_CALLS] in vocab") |
| 267 | |
| 268 | # Token sequence representing: [THINK]r[/THINK][TOOL_CALLS]f{"a":1} |
| 269 | output_tokens = ( |
| 270 | [mistral_tokenizer.instruct.BEGIN_THINK] |
| 271 | + mistral_tokenizer.tokenizer.encode("r", False, False) |
| 272 | + [mistral_tokenizer.instruct.END_THINK] |
| 273 | + [tool_calls_token_id] |
| 274 | + mistral_tokenizer.tokenizer.encode('f{"a":1}', False, False) |
| 275 | ) |
| 276 | for streaming in (False, True): |
| 277 | parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser( |
| 278 | _PARSER_NAME |
| 279 | )(mistral_tokenizer) |
| 280 | reasoning, _content = run_reasoning_extraction_mistral( |
| 281 | parser, output_tokens, streaming=streaming |
| 282 | ) |
| 283 | assert reasoning == "r" |
| 284 | assert "[TOOL_CALLS]" not in (reasoning or "") |
| 285 | |
| 286 | |
| 287 | @pytest.mark.parametrize("streaming", [False, True], ids=["nonstream", "stream"]) |
| 288 | def test_reasoning_v11_plain_text_think( |
| 289 | mistral_v11_tokenizer: MistralTokenizer, streaming: bool |
| 290 | ) -> None: |
| 291 | """v11 tokenizers use plain-text `<think>`/`</think>` (no special tokens).""" |
| 292 | parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(_PARSER_NAME)( |
| 293 | mistral_v11_tokenizer |
| 294 | ) |
| 295 | |
| 296 | tokens = mistral_v11_tokenizer.tokenizer.encode("<think>r</think>c", False, False) |
| 297 | reasoning, content = run_reasoning_extraction_mistral( |
| 298 | parser, tokens, streaming=streaming |
| 299 | ) |
| 300 | assert reasoning == "r" |
| 301 | assert content == "c" |
| 302 | |
| 303 | |
| 304 | def test_is_reasoning_end( |
| 305 | mistral_tokenizer: MistralTokenizer, |
| 306 | ) -> None: |
| 307 | """is_reasoning_end returns True iff reasoning has finished.""" |
| 308 | parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(_PARSER_NAME)( |
| 309 | mistral_tokenizer |
| 310 | ) |
| 311 | |
| 312 | # Complete reasoning: END_THINK present → True. |
| 313 | complete_ids = _encode_v13(mistral_tokenizer, "[THINK]r[/THINK]c") |
| 314 | assert parser.is_reasoning_end(complete_ids) |
| 315 | |
| 316 | # Open-only: no END_THINK, BEGIN_THINK present → False. |
| 317 | open_ids = _encode_v13(mistral_tokenizer, "[THINK]r") |
| 318 | assert not parser.is_reasoning_end(open_ids) |
| 319 | |
| 320 | # No reasoning markers at all → False. |
| 321 | no_reasoning_ids = _encode_v13(mistral_tokenizer, "hello") |
| 322 | assert not parser.is_reasoning_end(no_reasoning_ids) |
| 323 | |
| 324 | # Explicit END_THINK closes reasoning → True. |
| 325 | explicit_end_ids = ( |
| 326 | [mistral_tokenizer.instruct.BEGIN_THINK] |
| 327 | + mistral_tokenizer.tokenizer.encode("r", False, False) |
| 328 | + [mistral_tokenizer.instruct.END_THINK] |
| 329 | ) |
| 330 | assert parser.is_reasoning_end(explicit_end_ids) |
| 331 | |
| 332 | # [TOOL_CALLS] acts as implicit reasoning-end marker. |
| 333 | tool_calls_token_id = mistral_tokenizer.get_vocab().get("[TOOL_CALLS]") |
| 334 | if tool_calls_token_id is None: |
| 335 | pytest.skip("Tokenizer does not have [TOOL_CALLS] in vocab") |
| 336 | |
| 337 | implicit_end_ids = ( |
| 338 | [mistral_tokenizer.instruct.BEGIN_THINK] |
| 339 | + mistral_tokenizer.tokenizer.encode("r", False, False) |
| 340 | + [tool_calls_token_id] |
| 341 | ) |
| 342 | assert parser.is_reasoning_end(implicit_end_ids) |
| 343 | |
| 344 | |
| 345 | def _stream_parse_delta(parser, tokenizer, gen_ids, request, prompt_ids): |
| 346 | """Stream `gen_ids` through `parse_delta` and reconstruct reasoning/content.""" |
| 347 | reasoning = "" |
| 348 | content = "" |
| 349 | previous_tokens: list[str] | None = None |
| 350 | prefix_offset = 0 |
| 351 | read_offset = 0 |
| 352 | for i, tok in enumerate(gen_ids): |
| 353 | new_tokens, delta_text, prefix_offset, read_offset = detokenize_incrementally( |
| 354 | tokenizer=tokenizer, |
| 355 | all_input_ids=gen_ids[: i + 1], |
| 356 | prev_tokens=previous_tokens, |
| 357 | prefix_offset=prefix_offset, |
| 358 | read_offset=read_offset, |
| 359 | skip_special_tokens=True, |
| 360 | spaces_between_special_tokens=True, |
| 361 | ) |
| 362 | previous_tokens = ( |
| 363 | previous_tokens + new_tokens if previous_tokens else new_tokens |
| 364 | ) |
| 365 | delta_message = parser.parse_delta( |
| 366 | delta_text=delta_text, |
| 367 | delta_token_ids=[tok], |
| 368 | request=request, |
| 369 | prompt_token_ids=prompt_ids, |
| 370 | finished=(i == len(gen_ids) - 1), |
| 371 | ) |
| 372 | if delta_message is not None: |
| 373 | reasoning += delta_message.reasoning or "" |
| 374 | content += delta_message.content or "" |
| 375 | return reasoning, content |
| 376 | |
| 377 | |
| 378 | def test_parse_delta_keeps_reasoning_open_after_closed_think_prompt( |
| 379 | mistral_tokenizer: MistralTokenizer, |
| 380 | ) -> None: |
| 381 | """A closed `[THINK]` block in the prompt must not finalize reasoning. |
| 382 | |
| 383 | Regression: reasoning system prompts end the prompt with a closed |
| 384 | `[THINK]...[/THINK]` block. With a parser-injected grammar the generation's |
| 385 | reasoning must still be captured as ``reasoning_content`` rather than |
| 386 | leaking into ``content`` (the prompt-based reasoning-end heuristic must be |
| 387 | skipped). This exercises ``DelegatingParser.parse_delta``'s prompt check, |
| 388 | which the isolated reasoning-parser tests do not cover. |
| 389 | """ |
| 390 | parser_cls = ParserManager.get_parser( |
| 391 | tool_parser_name="mistral", |
| 392 | reasoning_parser_name="mistral", |
| 393 | enable_auto_tools=True, |
| 394 | ) |
| 395 | parser = parser_cls(mistral_tokenizer, None) |
| 396 | request = parser.adjust_request( |
| 397 | ChatCompletionRequest( |
| 398 | messages=[], |
| 399 | model="test", |
| 400 | tools=_SAMPLE_TOOLS, |
| 401 | tool_choice="auto", |
| 402 | ) |
| 403 | ) |
| 404 | assert getattr(request, "_grammar_from_parser", False) |
| 405 | |
| 406 | prompt_ids = _encode_v13(mistral_tokenizer, "[THINK]system reasoning[/THINK]") |
| 407 | gen_ids = _encode_v13( |
| 408 | mistral_tokenizer, |
| 409 | "[THINK]because two plus two is four[/THINK]The answer is 4.", |
| 410 | ) |
| 411 | |
| 412 | reasoning, content = _stream_parse_delta( |
| 413 | parser, mistral_tokenizer, gen_ids, request, prompt_ids |
| 414 | ) |
| 415 | |
| 416 | assert "because two plus two is four" in reasoning |
| 417 | assert "because two plus two is four" not in content |
| 418 | assert "The answer is 4." in content |
| 419 | |
| 420 | |
| 421 | def _encode_gen(tokenizer: MistralTokenizer, text: str) -> list[int]: |
| 422 | """Encode `text` mapping `[THINK]`, `[/THINK]`, `[TOOL_CALLS]`, `[ARGS]` |
| 423 | to their special-token IDs; all other text uses the base tokenizer. |
| 424 | """ |
| 425 | vocab = tokenizer.get_vocab() |
| 426 | markers: dict[str, int] = { |
| 427 | "[THINK]": tokenizer.instruct.BEGIN_THINK, |
| 428 | "[/THINK]": tokenizer.instruct.END_THINK, |
| 429 | "[TOOL_CALLS]": vocab["[TOOL_CALLS]"], |
| 430 | "[ARGS]": vocab["[ARGS]"], |
| 431 | } |
| 432 | out: list[int] = [] |
| 433 | remaining = text |
| 434 | while remaining: |
| 435 | earliest: int | None = None |
| 436 | earliest_marker: str | None = None |
| 437 | for marker in markers: |
| 438 | idx = remaining.find(marker) |
| 439 | if idx != -1 and (earliest is None or idx < earliest): |
| 440 | earliest = idx |
| 441 | earliest_marker = marker |
| 442 | if earliest_marker is None: |
| 443 | out += tokenizer.tokenizer.encode(remaining, False, False) |
| 444 | break |
| 445 | assert earliest is not None # set together with earliest_marker |
| 446 | if earliest > 0: |
| 447 | out += tokenizer.tokenizer.encode(remaining[:earliest], False, False) |
| 448 | out.append(markers[earliest_marker]) |
| 449 | remaining = remaining[earliest + len(earliest_marker) :] |
| 450 | return out |
| 451 | |
| 452 | |
| 453 | def _stream_turn( |
| 454 | parser, |
| 455 | tokenizer: MistralTokenizer, |
| 456 | gen_ids: list[int], |
| 457 | request, |
| 458 | prompt_ids: list[int], |
| 459 | ) -> tuple[str, str, list[tuple[str, str]]]: |
| 460 | """Stream `gen_ids` through `parse_delta`, returning `(reasoning, content, |
| 461 | tool_calls)`. |
| 462 | |
| 463 | `tool_calls` is a list of `(name, arguments)` tuples ordered by |
| 464 | `DeltaToolCall.index`. Arguments are concatenated across deltas. |
| 465 | """ |
| 466 | reasoning = "" |
| 467 | content = "" |
| 468 | tool_call_acc: dict[int, dict[str, str]] = {} |
| 469 | previous_tokens: list[str] | None = None |
| 470 | prefix_offset = 0 |
| 471 | read_offset = 0 |
| 472 | for i, tok in enumerate(gen_ids): |
| 473 | new_tokens, delta_text, prefix_offset, read_offset = detokenize_incrementally( |
| 474 | tokenizer=tokenizer, |
| 475 | all_input_ids=gen_ids[: i + 1], |
| 476 | prev_tokens=previous_tokens, |
| 477 | prefix_offset=prefix_offset, |
| 478 | read_offset=read_offset, |
| 479 | skip_special_tokens=True, |
| 480 | spaces_between_special_tokens=True, |
| 481 | ) |
| 482 | previous_tokens = ( |
| 483 | previous_tokens + new_tokens if previous_tokens else new_tokens |
| 484 | ) |
| 485 | delta_message = parser.parse_delta( |
| 486 | delta_text=delta_text, |
| 487 | delta_token_ids=[tok], |
| 488 | request=request, |
| 489 | prompt_token_ids=prompt_ids, |
| 490 | finished=(i == len(gen_ids) - 1), |
| 491 | ) |
| 492 | if delta_message is not None: |
| 493 | reasoning += delta_message.reasoning or "" |
| 494 | content += delta_message.content or "" |
| 495 | for tc in delta_message.tool_calls: |
| 496 | if tc.index not in tool_call_acc: |
| 497 | tool_call_acc[tc.index] = {"name": "", "args": ""} |
| 498 | if tc.function: |
| 499 | if tc.function.name: |
| 500 | tool_call_acc[tc.index]["name"] = tc.function.name |
| 501 | if tc.function.arguments: |
| 502 | tool_call_acc[tc.index]["args"] += tc.function.arguments |
| 503 | tool_calls = [ |
| 504 | (tool_call_acc[k]["name"], tool_call_acc[k]["args"]) |
| 505 | for k in sorted(tool_call_acc) |
| 506 | ] |
| 507 | return reasoning, content, tool_calls |
| 508 | |
| 509 | |
| 510 | def test_parse_delta_multi_turn_reason_and_tool( |
| 511 | mistral_tokenizer: MistralTokenizer, |
| 512 | ) -> None: |
| 513 | """Two-turn loop: each assistant turn carries reasoning + a tool call. |
| 514 | |
| 515 | For each turn, the mistral-common conversation is encoded into `prompt_ids` |
| 516 | before streaming the next generation. Turn 2's prompt therefore contains |
| 517 | a `[THINK]`/`[/THINK]` block and `[TOOL_CALLS]` from turn 1 — the |
| 518 | accumulated tool-call/result history that single-turn tests never exercise. |
| 519 | |
| 520 | Asserts that across both turns the parser correctly classifies think tokens |
| 521 | as `reasoning_content` (never leaking into `content`), and that the tool |
| 522 | call name/arguments are reconstructed correctly from the streaming deltas. |
| 523 | """ |
| 524 | parser_cls = ParserManager.get_parser( |
| 525 | tool_parser_name="mistral", |
| 526 | reasoning_parser_name="mistral", |
| 527 | enable_auto_tools=True, |
| 528 | ) |
| 529 | request = parser_cls(mistral_tokenizer, None).adjust_request( |
| 530 | ChatCompletionRequest( |
| 531 | messages=[], |
| 532 | model="test", |
| 533 | tools=_SAMPLE_TOOLS, |
| 534 | tool_choice="auto", |
| 535 | ) |
| 536 | ) |
| 537 | assert getattr(request, "_grammar_from_parser", False) |
| 538 | |
| 539 | cities = ["Dallas", "Paris"] |
| 540 | conversation: list = [UserMessage(content="What is the weather in Dallas?")] |
| 541 | |
| 542 | for i, city in enumerate(cities): |
| 543 | # Fresh parser per turn: streaming state must not bleed across turns. |
| 544 | parser = parser_cls(mistral_tokenizer, None) |
| 545 | |
| 546 | prompt_ids = mistral_tokenizer.instruct.encode_instruct( |
| 547 | InstructRequest(messages=conversation) |
| 548 | ).tokens |
| 549 | |
| 550 | gen_str = ( |
| 551 | f"[THINK]I should check {city} weather.[/THINK]" |
| 552 | f'[TOOL_CALLS]get_weather[ARGS]{{"city": "{city}"}}' |
| 553 | ) |
| 554 | gen_ids = _encode_gen(mistral_tokenizer, gen_str) |
| 555 | |
| 556 | reasoning, content, tool_calls = _stream_turn( |
| 557 | parser, mistral_tokenizer, gen_ids, request, prompt_ids |
| 558 | ) |
| 559 | |
| 560 | assert f"check {city} weather" in reasoning, f"turn {i}: reasoning not captured" |
| 561 | assert f"check {city} weather" not in content, ( |
| 562 | f"turn {i}: reasoning leaked into content" |
| 563 | ) |
| 564 | assert "[THINK]" not in content, f"turn {i}: [THINK] leaked" |
| 565 | assert "[TOOL_CALLS]" not in content, f"turn {i}: [TOOL_CALLS] leaked" |
| 566 | assert len(tool_calls) == 1, f"turn {i}: expected 1 tool call" |
| 567 | name, args = tool_calls[0] |
| 568 | assert name == "get_weather", f"turn {i}: wrong tool name {name!r}" |
| 569 | assert city in args, f"turn {i}: city not in args {args!r}" |
| 570 | |
| 571 | # Extend conversation with this turn's history for the next turn. |
| 572 | tc_id = f"tc{i:07d}" # 9-char alphanumeric id: "tc0000000", "tc0000001" |
| 573 | conversation.extend( |
| 574 | [ |
| 575 | AssistantMessage( |
| 576 | content=[ |
| 577 | ThinkChunk( |
| 578 | thinking=f"I should check {city} weather.", |
| 579 | closed=True, |
| 580 | ) |
| 581 | ], |
| 582 | tool_calls=[ |
| 583 | ToolCall( |
| 584 | id=tc_id, |
| 585 | function=FunctionCall( |
| 586 | name="get_weather", |
| 587 | arguments=json.dumps({"city": city}), |
| 588 | ), |
| 589 | ) |
| 590 | ], |
| 591 | ), |
| 592 | ToolMessage( |
| 593 | content=f"{city}: 70F sunny", |
| 594 | tool_call_id=tc_id, |
| 595 | ), |
| 596 | ] |
| 597 | ) |
| 598 | if i < len(cities) - 1: |
| 599 | conversation.append(UserMessage(content=f"And in {cities[i + 1]}?")) |