@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c4a3f9d137
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

[Frontend] Add Streaming Parser Engine and new Qwen3 Parser (#45413)

Signed-off-by: Ben Browning <bbrownin@redhat.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>

Ben Browning committed Jun 14, 2026 at 23:59 UTC c4a3f9d137099023a93fbb2299f3c52477751176
33 files changed +8492 -902
tests/parser/engine/__init__.py
tests/parser/engine/conftest.py new
+40
@@ -0,0 +1,40 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from unittest.mock import MagicMock
5 +
6 +import pytest
7 +
8 +from vllm.entrypoints.openai.chat_completion.protocol import (
9 + ChatCompletionRequest,
10 +)
11 +
12 +
13 +@pytest.fixture()
14 +def should_do_global_cleanup_after_test() -> bool:
15 + return False
16 +
17 +
18 +def make_mock_tokenizer(vocab: dict[str, int]) -> MagicMock:
19 + """Create a mock tokenizer with the given special-token vocabulary.
20 +
21 + The returned mock supports get_vocab(), encode(), and decode().
22 + decode() maps known token IDs back to their text and falls back to
23 + chr(id) for ASCII IDs or ``<id>`` for others.
24 + """
25 + id_to_text = {v: k for k, v in vocab.items()}
26 + tokenizer = MagicMock()
27 + tokenizer.encode.return_value = [1, 2, 3]
28 + tokenizer.get_vocab.return_value = dict(vocab)
29 + tokenizer.decode.side_effect = lambda ids: "".join(
30 + id_to_text.get(i, chr(i) if i < 128 else f"<{i}>") for i in ids
31 + )
32 + return tokenizer
33 +
34 +
35 +@pytest.fixture
36 +def mock_request():
37 + req = MagicMock(spec=ChatCompletionRequest)
38 + req.tools = []
39 + req.tool_choice = "auto"
40 + return req
tests/parser/engine/replay_harness.py new
+377
@@ -0,0 +1,377 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Data-driven replay harness for parser engine testing.
4 +
5 +Replays token sequences through parsers at different chunk sizes to
6 +verify chunk-size invariance: the same token sequence must produce
7 +identical output regardless of how tokens are batched.
8 +"""
9 +
10 +from __future__ import annotations
11 +
12 +import json
13 +from collections.abc import Sequence
14 +from dataclasses import dataclass, field
15 +
16 +from vllm.entrypoints.openai.chat_completion.protocol import (
17 + ChatCompletionRequest,
18 +)
19 +from vllm.entrypoints.openai.engine.protocol import DeltaMessage
20 +
21 +
22 +@dataclass
23 +class Sample:
24 + """One test sample loaded from a JSONL file."""
25 +
26 + id: str
27 + description: str
28 + source: str
29 + vocab: dict[str, int]
30 + tokens: list[tuple[int, str]]
31 + expected_reasoning: str | None
32 + expected_content: str | None
33 + expected_tool_calls: list[dict] | None
34 + tools: list[dict] | None = None
35 + chat_template_kwargs: dict | None = None
36 +
37 +
38 +@dataclass
39 +class ParseOutput:
40 + """Accumulated parse output from replaying a token stream."""
41 +
42 + reasoning: str = ""
43 + content: str = ""
44 + tool_calls: list[dict] = field(default_factory=list)
45 +
46 +
47 +class MockTokenizer:
48 + """Lightweight tokenizer mock that avoids unittest.mock overhead.
49 +
50 + Used by ``benchmarks/benchmark_parsers.py`` in tight timing loops,
51 + so hot-path methods (``decode``, ``get_vocab``) must be cheap.
52 + MagicMock's call-recording machinery added ~40% overhead to small-
53 + sample benchmarks, inflating the per-token cost of the parser engine.
54 + """
55 +
56 + __slots__ = (
57 + "_vocab",
58 + "_token_ids",
59 + "_token_decode_map",
60 + "_special_ids",
61 + "eos_token_id",
62 + "bos_token_id",
63 + "pad_token_id",
64 + )
65 +
66 + def __init__(
67 + self,
68 + vocab: dict[str, int],
69 + tokens: list[tuple[int, str]],
70 + ) -> None:
71 + self._vocab = vocab
72 + self._token_ids = [tid for tid, _ in tokens]
73 + self._token_decode_map = {tid: text for tid, text in tokens}
74 + self._special_ids = set(vocab.values())
75 + self.eos_token_id = None
76 + self.bos_token_id = None
77 + self.pad_token_id = None
78 +
79 + def set_vocab(self, vocab: dict[str, int]) -> None:
80 + self._vocab = vocab
81 +
82 + def get_vocab(self) -> dict[str, int]:
83 + return self._vocab
84 +
85 + def encode(self, text: str, **kwargs) -> list[int]:
86 + return self._token_ids
87 +
88 + def decode(self, ids: list[int], skip_special_tokens: bool = False) -> str:
89 + parts: list[str] = []
90 + for tid in ids:
91 + if skip_special_tokens and tid in self._special_ids:
92 + continue
93 + text = self._token_decode_map.get(tid, f"?{tid}?")
94 + parts.append(text)
95 + return "".join(parts)
96 +
97 +
98 +def make_mock_tokenizer(sample: Sample) -> MockTokenizer:
99 + """Build a mock tokenizer from a sample's vocab and token data."""
100 + return MockTokenizer(
101 + vocab=dict(sample.vocab),
102 + tokens=sample.tokens,
103 + )
104 +
105 +
106 +def _test_request(
107 + tools: list[dict] | None = None,
108 +) -> ChatCompletionRequest:
109 + return ChatCompletionRequest(
110 + model="test-model",
111 + messages=[{"role": "user", "content": "test"}],
112 + tools=tools,
113 + )
114 +
115 +
116 +def replay_streaming(
117 + parser,
118 + tokens: list[tuple[int, str]],
119 + chunk_size: int | None = None,
120 + holdback_chars: int = 0,
121 + finished_on_last: bool = False,
122 + tools: list[dict] | None = None,
123 +) -> list[DeltaMessage | None]:
124 + """Feed tokens through ``parser.parse_delta()`` at a given chunk size.
125 +
126 + Args:
127 + parser: A :class:`Parser` instance with ``parse_delta()`` method.
128 + tokens: List of ``(token_id, decoded_text)`` pairs.
129 + chunk_size: Number of tokens per batch. ``None`` means all at once.
130 + holdback_chars: Simulate detokenizer holdback by holding back
131 + this many characters of decoded text between batches.
132 + finished_on_last: When True, pass ``finished=True`` on the last
133 + ``parse_delta()`` call, matching real server behavior.
134 + tools: Optional tool definitions to include on the request,
135 + matching the serving layer where tools set
136 + ``tool_choice`` to ``"auto"``.
137 +
138 + Returns:
139 + List of ``DeltaMessage`` results from each ``parse_delta()`` call.
140 + """
141 + if chunk_size is None:
142 + chunk_size = len(tokens)
143 +
144 + results: list[DeltaMessage | None] = []
145 + all_ids = [tid for tid, _ in tokens]
146 + all_texts = [text for _, text in tokens]
147 +
148 + request = _test_request(tools=tools)
149 +
150 + if holdback_chars <= 0:
151 + chunks = list(range(0, len(tokens), chunk_size))
152 + for i, start in enumerate(chunks):
153 + batch_end = min(start + chunk_size, len(tokens))
154 + batch_ids = all_ids[start:batch_end]
155 + delta_text = "".join(all_texts[start:batch_end])
156 + is_last = i == len(chunks) - 1
157 +
158 + result = parser.parse_delta(
159 + delta_text,
160 + batch_ids,
161 + request,
162 + prompt_token_ids=[] if start == 0 else None,
163 + finished=finished_on_last and is_last,
164 + )
165 + results.append(result)
166 + return results
167 +
168 + emitted_up_to = 0
169 + is_first = True
170 +
171 + for start in range(0, len(tokens), chunk_size):
172 + batch_end = min(start + chunk_size, len(tokens))
173 +
174 + if batch_end < len(tokens):
175 + held_chars = 0
176 + safe_end = batch_end
177 + while safe_end > emitted_up_to and held_chars < holdback_chars:
178 + safe_end -= 1
179 + held_chars += len(all_texts[safe_end])
180 + else:
181 + safe_end = batch_end
182 +
183 + if safe_end <= emitted_up_to:
184 + continue
185 +
186 + batch_ids = all_ids[emitted_up_to:safe_end]
187 + delta_text = "".join(all_texts[emitted_up_to:safe_end])
188 + emitted_up_to = safe_end
189 +
190 + is_last_chunk = batch_end >= len(tokens)
191 + result = parser.parse_delta(
192 + delta_text,
193 + batch_ids,
194 + request,
195 + prompt_token_ids=[] if is_first else None,
196 + finished=finished_on_last and is_last_chunk,
197 + )
198 + results.append(result)
199 + is_first = False
200 +
201 + if emitted_up_to < len(tokens):
202 + batch_ids = all_ids[emitted_up_to:]
203 + delta_text = "".join(all_texts[emitted_up_to:])
204 + result = parser.parse_delta(
205 + delta_text,
206 + batch_ids,
207 + request,
208 + prompt_token_ids=[] if is_first else None,
209 + finished=finished_on_last,
210 + )
211 + results.append(result)
212 +
213 + return results
214 +
215 +
216 +def replay_with_text_holdback(
217 + parser,
218 + tokens: list[tuple[int, str]],
219 + text_delay: int = 1,
220 + tools: list[dict] | None = None,
221 +) -> list[DeltaMessage | None]:
222 + """Replay token-by-token with text arriving *text_delay* steps late.
223 +
224 + Simulates the production detokenizer holdback where token IDs arrive
225 + immediately but decoded text is delayed. On the last token all
226 + remaining held-back text is flushed, matching real server behavior::
227 +
228 + step 0: ids=[tok0], text="" (held back)
229 + step 1: ids=[tok1], text=tok0_text (tok0 released)
230 + ...
231 + step N-1: ids=[tokN-1], text=remaining_texts (flush all)
232 +
233 + This exercises the TokenIDScanner deferred-terminal path that
234 + ``replay_streaming`` (which keeps text and IDs aligned) does not.
235 + """
236 + results: list[DeltaMessage | None] = []
237 + request = _test_request(tools=tools)
238 +
239 + n = len(tokens)
240 + held_texts: list[str] = []
241 +
242 + for i in range(n):
243 + token_id = tokens[i][0]
244 + held_texts.append(tokens[i][1])
245 +
246 + is_last = i == n - 1
247 + if is_last:
248 + delta_text = "".join(held_texts)
249 + held_texts.clear()
250 + elif len(held_texts) > text_delay:
251 + delta_text = held_texts.pop(0)
252 + else:
253 + delta_text = ""
254 +
255 + result = parser.parse_delta(
256 + delta_text,
257 + [token_id],
258 + request,
259 + prompt_token_ids=[] if i == 0 else None,
260 + finished=is_last,
261 + )
262 + results.append(result)
263 +
264 + return results
265 +
266 +
267 +def accumulate_deltas(
268 + deltas: Sequence[DeltaMessage | None],
269 +) -> dict:
270 + reasoning_parts: list[str] = []
271 + content_parts: list[str] = []
272 + tool_calls_by_idx: dict[int, dict] = {}
273 +
274 + for delta in deltas:
275 + if delta is None:
276 + continue
277 + if delta.reasoning:
278 + reasoning_parts.append(delta.reasoning)
279 + if delta.content:
280 + content_parts.append(delta.content)
281 + if delta.tool_calls:
282 + for tc in delta.tool_calls:
283 + if tc.function and tc.function.name:
284 + existing = tool_calls_by_idx.get(tc.index)
285 + if existing is None:
286 + tool_calls_by_idx[tc.index] = {
287 + "name": tc.function.name,
288 + "_args_parts": [tc.function.arguments or ""],
289 + }
290 + else:
291 + existing["_args_parts"].append(tc.function.arguments or "")
292 + elif tc.function and tc.function.arguments:
293 + existing = tool_calls_by_idx.get(tc.index)
294 + if existing is not None:
295 + existing["_args_parts"].append(tc.function.arguments)
296 +
297 + return {
298 + "reasoning": "".join(reasoning_parts),
299 + "content": "".join(content_parts),
300 + "tool_calls": [
301 + {"name": tc["name"], "arguments": "".join(tc["_args_parts"])}
302 + for tc in tool_calls_by_idx.values()
303 + ],
304 + }
305 +
306 +
307 +def collect_output(results: list[DeltaMessage | None]) -> ParseOutput:
308 + """Accumulate ``DeltaMessage`` results into a :class:`ParseOutput`."""
309 + result = accumulate_deltas(results)
310 + return ParseOutput(
311 + reasoning=result["reasoning"],
312 + content=result["content"],
313 + tool_calls=result["tool_calls"],
314 + )
315 +
316 +
317 +def assert_parse_output(actual: ParseOutput, sample: Sample) -> None:
318 + """Compare actual parse output against expected values from a sample."""
319 + if sample.expected_reasoning is not None:
320 + assert actual.reasoning == sample.expected_reasoning, (
321 + f"Reasoning mismatch:\n"
322 + f" expected: {sample.expected_reasoning!r}\n"
323 + f" actual: {actual.reasoning!r}"
324 + )
325 +
326 + if sample.expected_content is not None:
327 + assert actual.content == sample.expected_content, (
328 + f"Content mismatch:\n"
329 + f" expected: {sample.expected_content!r}\n"
330 + f" actual: {actual.content!r}"
331 + )
332 + if sample.expected_tool_calls is not None:
333 + assert len(actual.tool_calls) == len(sample.expected_tool_calls), (
334 + f"Tool call count mismatch: "
335 + f"expected {len(sample.expected_tool_calls)}, "
336 + f"got {len(actual.tool_calls)}"
337 + )
338 + for i, (expected_tc, actual_tc) in enumerate(
339 + zip(sample.expected_tool_calls, actual.tool_calls)
340 + ):
341 + assert actual_tc["name"] == expected_tc["name"], (
342 + f"Tool call {i} name mismatch: "
343 + f"expected {expected_tc['name']!r}, "
344 + f"got {actual_tc['name']!r}"
345 + )
346 + if "arguments" in expected_tc:
347 + expected_args = expected_tc["arguments"]
348 + actual_args_str = actual_tc.get("arguments", "{}")
349 + if isinstance(expected_args, dict):
350 + try:
351 + actual_args = json.loads(actual_args_str)
352 + except json.JSONDecodeError as e:
353 + raise AssertionError(
354 + f"Tool call {i} arguments not valid JSON: "
355 + f"{actual_args_str!r}"
356 + ) from e
357 + assert actual_args == expected_args, (
358 + f"Tool call {i} arguments mismatch:\n"
359 + f" expected: {expected_args}\n"
360 + f" actual: {actual_args}"
361 + )
362 +
363 +
364 +def assert_no_terminal_leakage(
365 + actual: ParseOutput,
366 + terminals: list[str],
367 + context: str = "",
368 +) -> None:
369 + """Assert that none of *terminals* appear in reasoning or content."""
370 + suffix = f" ({context})" if context else ""
371 + for terminal in terminals:
372 + assert terminal not in actual.reasoning, (
373 + f"{terminal!r} leaked into reasoning{suffix}"
374 + )
375 + assert terminal not in actual.content, (
376 + f"{terminal!r} leaked into content{suffix}"
377 + )
tests/parser/engine/streaming_helpers.py new
+137
@@ -0,0 +1,137 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Shared streaming simulation helpers for parser engine tests."""
4 +
5 +from __future__ import annotations
6 +
7 +from typing import Any
8 +
9 +from vllm.entrypoints.openai.engine.protocol import DeltaMessage
10 +
11 +
12 +def _build_token_id_map(parser) -> dict[str, int]:
13 + """Map special token text to token IDs from the parser's config."""
14 + token_id_map: dict[str, int] = {}
15 + cfg = getattr(parser, "parser_engine_config", None)
16 + vocab = getattr(parser, "vocab", None)
17 + if cfg is not None and vocab is not None:
18 + for text in (cfg.token_id_terminals or {}).values():
19 + tid = vocab.get(text)
20 + if tid is not None:
21 + token_id_map[text] = tid
22 + return token_id_map
23 +
24 +
25 +def simulate_tool_streaming(
26 + parser,
27 + request,
28 + chunks: list[str],
29 +) -> list[tuple[DeltaMessage | None, str]]:
30 + """Feed text chunks through ``extract_tool_calls_streaming()``."""
31 + token_id_map = _build_token_id_map(parser)
32 +
33 + results: list[tuple[Any, str]] = []
34 + previous_text = ""
35 + previous_token_ids: list[int] = []
36 +
37 + for chunk in chunks:
38 + current_text = previous_text + chunk
39 +
40 + delta_token_ids: list[int] = [
41 + tid for text, tid in token_id_map.items() if text in chunk
42 + ]
43 +
44 + current_token_ids = previous_token_ids + delta_token_ids
45 +
46 + delta = parser.extract_tool_calls_streaming(
47 + previous_text=previous_text,
48 + current_text=current_text,
49 + delta_text=chunk,
50 + previous_token_ids=tuple(previous_token_ids),
51 + current_token_ids=tuple(current_token_ids),
52 + delta_token_ids=tuple(delta_token_ids),
53 + request=request,
54 + )
55 + results.append((delta, current_text))
56 + previous_text = current_text
57 + previous_token_ids = list(current_token_ids)
58 +
59 + return results
60 +
61 +
62 +def collect_tool_arguments(
63 + results: list[tuple[DeltaMessage | None, str]],
64 +) -> str:
65 + """Concatenate all streamed argument fragments."""
66 + args_text = ""
67 + for delta, _ in results:
68 + if delta and delta.tool_calls:
69 + for tc in delta.tool_calls:
70 + if tc.function and tc.function.arguments:
71 + args_text += tc.function.arguments
72 + return args_text
73 +
74 +
75 +def collect_content(
76 + results: list[tuple[DeltaMessage | None, str]],
77 +) -> str:
78 + """Concatenate all streamed content parts."""
79 + parts: list[str] = []
80 + for delta, _ in results:
81 + if delta and delta.content:
82 + parts.append(delta.content)
83 + return "".join(parts)
84 +
85 +
86 +def collect_function_name(
87 + results: list[tuple[DeltaMessage | None, str]],
88 +) -> str | None:
89 + """Return first function name from deltas."""
90 + for delta, _ in results:
91 + if delta and delta.tool_calls:
92 + for tc in delta.tool_calls:
93 + if tc.function and tc.function.name:
94 + return tc.function.name
95 + return None
96 +
97 +
98 +def simulate_reasoning_streaming(
99 + parser,
100 + chunks: list[str],
101 + delta_token_ids_per_chunk: list[tuple[int, ...]] | None = None,
102 +) -> tuple[str, str]:
103 + """Feed chunks through ``extract_reasoning_streaming()``.
104 +
105 + Returns ``(reasoning_text, content_text)`` tuple.
106 + """
107 + token_id_map = (
108 + _build_token_id_map(parser) if delta_token_ids_per_chunk is None else {}
109 + )
110 +
111 + reasoning_parts: list[str] = []
112 + content_parts: list[str] = []
113 + prev_text = ""
114 + prev_ids: list[int] = []
115 + for i, chunk in enumerate(chunks):
116 + cur_text = prev_text + chunk
117 + if delta_token_ids_per_chunk is not None:
118 + d_ids = delta_token_ids_per_chunk[i]
119 + else:
120 + d_ids = tuple(tid for text, tid in token_id_map.items() if text in chunk)
121 + cur_ids = prev_ids + list(d_ids)
122 + delta = parser.extract_reasoning_streaming(
123 + previous_text=prev_text,
124 + current_text=cur_text,
125 + delta_text=chunk,
126 + previous_token_ids=tuple(prev_ids),
127 + current_token_ids=tuple(cur_ids),
128 + delta_token_ids=d_ids,
129 + )
130 + if delta:
131 + if delta.reasoning:
132 + reasoning_parts.append(delta.reasoning)
133 + if delta.content:
134 + content_parts.append(delta.content)
135 + prev_text = cur_text
136 + prev_ids = list(cur_ids)
137 + return "".join(reasoning_parts), "".join(content_parts)
tests/parser/engine/test_delegating_replay.py new
+82
@@ -0,0 +1,82 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Replay tests for DelegatingParser with engine adapters.
4 +
5 +Exercises DelegatingParser in engine-adapter mode to verify that delegated
6 +routing produces correct output across chunk sizes.
7 +See test_replay.py for tests that target engine parsers directly.
8 +"""
9 +
10 +from __future__ import annotations
11 +
12 +from functools import lru_cache
13 +
14 +import pytest
15 +from pydantic import TypeAdapter
16 +
17 +from tests.parser.engine.replay_harness import (
18 + assert_parse_output,
19 + collect_output,
20 + make_mock_tokenizer,
21 + replay_streaming,
22 +)
23 +from tests.parser.engine.trace_builder import build_samples
24 +from vllm.entrypoints.openai.chat_completion.protocol import (
25 + ChatCompletionToolsParam,
26 +)
27 +from vllm.parser.abstract_parser import Parser
28 +from vllm.parser.parser_manager import ParserManager
29 +
30 +_TOOLS_VALIDATOR = TypeAdapter(list[ChatCompletionToolsParam])
31 +
32 +_PAIRINGS: dict[str, tuple[str, str]] = {
33 + "engine": ("qwen3_coder", "qwen3"),
34 +}
35 +
36 +CHUNK_SIZES = [1, 2, 3, 5, 11, 23, None]
37 +
38 +
39 +@lru_cache
40 +def _get_delegating_parser_cls(pairings: str) -> type[Parser]:
41 + tool_name, reasoning_name = _PAIRINGS[pairings]
42 + parser_cls = ParserManager.get_parser(
43 + tool_parser_name=tool_name,
44 + reasoning_parser_name=reasoning_name,
45 + enable_auto_tools=True,
46 + )
47 + assert parser_cls is not None
48 + return parser_cls
49 +
50 +
51 +_all_samples = build_samples("qwen3")
52 +
53 +
54 +@pytest.mark.parametrize(
55 + "pairings",
56 + list(_PAIRINGS),
57 + ids=lambda p: f"mode={p}",
58 +)
59 +@pytest.mark.parametrize("chunk_size", CHUNK_SIZES, ids=lambda c: f"chunk={c}")
60 +@pytest.mark.parametrize("sample", _all_samples, ids=lambda s: s.id)
61 +def test_delegating_replay(sample, chunk_size, pairings):
62 + parser_cls = _get_delegating_parser_cls(pairings=pairings)
63 +
64 + tokenizer = make_mock_tokenizer(sample)
65 + validated_tools = (
66 + _TOOLS_VALIDATOR.validate_python(sample.tools) if sample.tools else None
67 + )
68 + parser = parser_cls(
69 + tokenizer,
70 + validated_tools,
71 + chat_template_kwargs=sample.chat_template_kwargs,
72 + )
73 +
74 + deltas = replay_streaming(
75 + parser,
76 + sample.tokens,
77 + chunk_size=chunk_size,
78 + finished_on_last=True,
79 + tools=sample.tools,
80 + )
81 + output = collect_output(deltas)
82 + assert_parse_output(output, sample)
tests/parser/engine/test_engine.py new
+846
@@ -0,0 +1,846 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Tests for the streaming parser engine core pipeline."""
4 +
5 +from unittest.mock import MagicMock
6 +
7 +import pytest
8 +
9 +from tests.parser.engine.conftest import make_mock_tokenizer
10 +from vllm.parser.engine.events import EventType, SemanticEvent
11 +from vllm.parser.engine.incremental_lexer import (
12 + LexerShape,
13 + TerminalDef,
14 + terminals_from_literals,
15 +)
16 +from vllm.parser.engine.parser_engine_config import (
17 + ParserEngineConfig,
18 + ParserState,
19 + Transition,
20 +)
21 +from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
22 +
23 +
24 +def _hermes_config() -> ParserEngineConfig:
25 + """Simple Hermes-style config: <tool_call>JSON</tool_call>."""
26 + return ParserEngineConfig(
27 + name="hermes_test",
28 + terminals={
29 + "TOOL_START": "<tool_call>",
30 + "TOOL_END": "</tool_call>",
31 + },
32 + token_id_terminals={
33 + "TOOL_START": "<tool_call>",
34 + "TOOL_END": "</tool_call>",
35 + },
36 + transitions={
37 + (ParserState.CONTENT, "TOOL_START"): Transition(
38 + ParserState.TOOL_ARGS,
39 + (EventType.TOOL_CALL_START,),
40 + ),
41 + (ParserState.TOOL_ARGS, "TOOL_END"): Transition(
42 + ParserState.CONTENT,
43 + (EventType.TOOL_CALL_END,),
44 + ),
45 + },
46 + content_events={
47 + ParserState.CONTENT: EventType.TEXT_CHUNK,
48 + ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
49 + },
50 + )
51 +
52 +
53 +def _think_config() -> ParserEngineConfig:
54 + """Simple think-tag reasoning config: <think>...</think>."""
55 + return ParserEngineConfig(
56 + name="think_test",
57 + terminals={
58 + "THINK_START": "<think>",
59 + "THINK_END": "</think>",
60 + },
61 + transitions={
62 + (ParserState.CONTENT, "THINK_START"): Transition(
63 + ParserState.REASONING,
64 + (EventType.REASONING_START,),
65 + ),
66 + (ParserState.REASONING, "THINK_END"): Transition(
67 + ParserState.CONTENT,
68 + (EventType.REASONING_END,),
69 + ),
70 + },
71 + )
72 +
73 +
74 +class TestNonStreaming:
75 + def test_plain_text(self):
76 + engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
77 + events = engine.parse_complete("Hello, world!")
78 + assert len(events) == 1
79 + assert events[0].type == EventType.TEXT_CHUNK
80 + assert events[0].value == "Hello, world!"
81 +
82 + def test_single_tool_call(self):
83 + engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
84 + text = (
85 + '<tool_call>{"name": "get_weather",'
86 + ' "arguments": {"city": "SF"}}'
87 + "</tool_call>"
88 + )
89 + events = engine.parse_complete(text)
90 +
91 + types = [e.type for e in events]
92 + assert EventType.TOOL_CALL_START in types
93 + assert EventType.TOOL_CALL_END in types
94 + assert EventType.ARG_VALUE_CHUNK in types
95 +
96 + arg_text = "".join(
97 + e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK
98 + )
99 + assert '"name": "get_weather"' in arg_text
100 + assert '"city": "SF"' in arg_text
101 +
102 + def test_text_then_tool_call(self):
103 + engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
104 + text = 'Sure!<tool_call>{"name": "add"}</tool_call>'
105 + events = engine.parse_complete(text)
106 +
107 + types = [e.type for e in events]
108 + assert types[0] == EventType.TEXT_CHUNK
109 + assert events[0].value == "Sure!"
110 + assert EventType.TOOL_CALL_START in types
111 + assert EventType.TOOL_CALL_END in types
112 +
113 + def test_multiple_tool_calls(self):
114 + engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
115 + text = (
116 + '<tool_call>{"name": "a"}</tool_call><tool_call>{"name": "b"}</tool_call>'
117 + )
118 + events = engine.parse_complete(text)
119 +
120 + starts = [e for e in events if e.type == EventType.TOOL_CALL_START]
121 + ends = [e for e in events if e.type == EventType.TOOL_CALL_END]
122 + assert len(starts) == 2
123 + assert len(ends) == 2
124 + assert starts[0].tool_index == 0
125 + assert starts[1].tool_index == 1
126 +
127 + def test_reasoning(self):
128 + engine = StreamingParserEngine(_think_config(), tokenizer=None)
129 + text = "<think>Let me think...</think>The answer is 42."
130 + events = engine.parse_complete(text)
131 +
132 + types = [e.type for e in events]
133 + assert types[0] == EventType.REASONING_START
134 + assert EventType.REASONING_CHUNK in types
135 + assert EventType.REASONING_END in types
136 + assert EventType.TEXT_CHUNK in types
137 +
138 + reasoning = "".join(
139 + e.value for e in events if e.type == EventType.REASONING_CHUNK
140 + )
141 + assert "Let me think..." in reasoning
142 +
143 + content = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
144 + assert "The answer is 42." in content
145 +
146 +
147 +class TestStreaming:
148 + @staticmethod
149 + def _feed_chars(
150 + engine: StreamingParserEngine,
151 + text: str,
152 + ) -> list[SemanticEvent]:
153 + """Feed text one character at a time."""
154 + all_events = []
155 + for ch in text:
156 + all_events.extend(engine.feed(ch, []))
157 + all_events.extend(engine.finish())
158 + return all_events
159 +
160 + @staticmethod
161 + def _feed_chunks(
162 + engine: StreamingParserEngine,
163 + text: str,
164 + chunk_size: int,
165 + ) -> list[SemanticEvent]:
166 + """Feed text in fixed-size chunks."""
167 + all_events = []
168 + for i in range(0, len(text), chunk_size):
169 + chunk = text[i : i + chunk_size]
170 + all_events.extend(engine.feed(chunk, []))
171 + all_events.extend(engine.finish())
172 + return all_events
173 +
174 + def test_char_by_char_tool_call(self):
175 + engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
176 + text = '<tool_call>{"name": "add", "arguments": {"a": 1}}</tool_call>'
177 + events = self._feed_chars(engine, text)
178 +
179 + types = [e.type for e in events]
180 + assert EventType.TOOL_CALL_START in types
181 + assert EventType.TOOL_CALL_END in types
182 + assert EventType.ARG_VALUE_CHUNK in types
183 +
184 + arg_text = "".join(
185 + e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK
186 + )
187 + assert '"name": "add"' in arg_text
188 +
189 + @pytest.mark.parametrize(
190 + "text",
191 + [
192 + '<tool_call>{"name": "get", "arguments": {"x": "hello"}}</tool_call>',
193 + '<tool_call>{"name": "f", "arguments": '
194 + '{"items": [1, [2, 3]], "obj": {"k": "v"}}}'
195 + "</tool_call>",
196 + ],
197 + ids=["flat_args", "nested_arrays"],
198 + )
199 + def test_chunk_sizes_produce_same_content(self, text):
200 + """Different chunk sizes must produce identical concatenated content."""
201 + results = {}
202 + for chunk_size in [1, 2, 3, 5, 7, len(text)]:
203 + engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
204 + events = self._feed_chunks(engine, text, chunk_size)
205 + arg_text = "".join(
206 + e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK
207 + )
208 + results[chunk_size] = arg_text
209 +
210 + values = list(results.values())
211 + for v in values[1:]:
212 + assert v == values[0], f"Mismatch: {results}"
213 +
214 + def test_prefix_buffering_prevents_premature_emit(self):
215 + """Text like '<tool_' should be buffered, not emitted as content."""
216 + engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
217 +
218 + events1 = engine.feed("<tool_", [])
219 + content_events = [e for e in events1 if e.type == EventType.TEXT_CHUNK]
220 + assert len(content_events) == 0, "Should buffer partial tag"
221 +
222 + events2 = engine.feed("call>", [])
223 + starts = [e for e in events2 if e.type == EventType.TOOL_CALL_START]
224 + assert len(starts) == 1
225 +
226 + def test_prefix_buffering_flush_on_mismatch(self):
227 + """Text like '<tool_box' should eventually flush as content."""
228 + engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
229 +
230 + events1 = engine.feed("<tool_", [])
231 + assert len([e for e in events1 if e.type == EventType.TEXT_CHUNK]) == 0
232 +
233 + events2 = engine.feed("box>rest", [])
234 + events2.extend(engine.finish())
235 + content = "".join(e.value for e in events2 if e.type == EventType.TEXT_CHUNK)
236 + assert content == "<tool_box>rest"
237 +
238 + def test_reasoning_streaming(self):
239 + engine = StreamingParserEngine(_think_config(), tokenizer=None)
240 + events = self._feed_chars(engine, "<think>hmm</think>answer")
241 +
242 + reasoning = "".join(
243 + e.value for e in events if e.type == EventType.REASONING_CHUNK
244 + )
245 + content = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
246 + assert "hmm" in reasoning
247 + assert "answer" in content
248 +
249 + def test_text_between_tool_calls(self):
250 + engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
251 + text = (
252 + 'Hi<tool_call>{"name":"a"}</tool_call>'
253 + 'mid<tool_call>{"name":"b"}</tool_call>end'
254 + )
255 + events = self._feed_chunks(engine, text, 3)
256 +
257 + texts = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
258 + assert "Hi" in texts
259 + assert "mid" in texts
260 + assert "end" in texts
261 +
262 + starts = [e for e in events if e.type == EventType.TOOL_CALL_START]
263 + assert len(starts) == 2
264 +
265 + def test_unmatched_close_brace_does_not_poison_depth(self):
266 + """A stray } in malformed JSON must not kill streaming for
267 + all subsequent content."""
268 + engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
269 + engine.feed("<tool_call>", [])
270 +
271 + malformed = '}{{"a": 1}}'
272 + events = self._feed_chars(engine, malformed + "</tool_call>")
273 +
274 + arg_chunks = [e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK]
275 + assert len(arg_chunks) > 1, (
276 + "Content after stray } should still stream incrementally"
277 + )
278 + arg_text = "".join(arg_chunks)
279 + assert '"a": 1' in arg_text
280 +
281 + def test_json_args_no_premature_close_brace(self):
282 + """Closing braces of the top-level JSON shouldn't be streamed
283 + until confirmed by the end tag."""
284 + engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
285 +
286 + engine.feed("<tool_call>", [])
287 + events = engine.feed('{"name": "f"}', [])
288 +
289 + arg_text = "".join(
290 + e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK
291 + )
292 + assert "}" not in arg_text, "Top-level } should be held back"
293 +
294 + events2 = engine.feed("</tool_call>", [])
295 + arg_text2 = "".join(
296 + e.value for e in events2 if e.type == EventType.ARG_VALUE_CHUNK
297 + )
298 + assert "}" in arg_text2, "} should flush on end tag"
299 +
300 +
301 +_START_ID = 50
302 +_END_ID = 51
303 +_TOOL_START_ID = 60
304 +_TOOL_END_ID = 61
305 +
306 +
307 +def _make_think_tokenizer():
308 + tok = MagicMock()
309 + tok.encode.return_value = [1, 2, 3]
310 + tok.get_vocab.return_value = {"<think>": _START_ID, "</think>": _END_ID}
311 + tok.decode.side_effect = lambda ids: {
312 + _START_ID: "<think>",
313 + _END_ID: "</think>",
314 + }.get(ids[0], f"tok{ids[0]}")
315 + return tok
316 +
317 +
318 +def _make_hermes_tokenizer():
319 + """Tokenizer that resolves tool_call tags to special IDs."""
320 + _special = {_TOOL_START_ID: "<tool_call>", _TOOL_END_ID: "</tool_call>"}
321 + tok = MagicMock()
322 + tok.encode.return_value = [1, 2, 3]
323 + tok.get_vocab.return_value = {
324 + "<tool_call>": _TOOL_START_ID,
325 + "</tool_call>": _TOOL_END_ID,
326 + }
327 + tok.decode.side_effect = lambda ids: "".join(
328 + _special.get(i, chr(i) if i < 128 else f"<{i}>") for i in ids
329 + )
330 + return tok
331 +
332 +
333 +class TestLexerBufferFlush:
334 + """Lexer buffer must be flushed before PreLexedTerminal transitions."""
335 +
336 + def test_buffered_prefix_emitted_in_current_state(self):
337 + """Text buffered by the lexer (e.g. '<') must be emitted as
338 + REASONING_CHUNK before THINK_END transitions to CONTENT."""
339 + engine = StreamingParserEngine(_think_config(), _make_think_tokenizer())
340 +
341 + events = engine.feed("<think>", [_START_ID])
342 + assert any(e.type == EventType.REASONING_START for e in events)
343 +
344 + events = engine.feed("reasoning text<", [])
345 + reasoning_text = "".join(
346 + e.value for e in events if e.type == EventType.REASONING_CHUNK
347 + )
348 + assert "reasoning text" in reasoning_text
349 +
350 + events = engine.feed("</think>", [_END_ID])
351 + event_types = [e.type for e in events]
352 + if EventType.REASONING_CHUNK in event_types:
353 + rc_idx = event_types.index(EventType.REASONING_CHUNK)
354 + re_idx = event_types.index(EventType.REASONING_END)
355 + assert rc_idx < re_idx, (
356 + "'<' must be emitted as REASONING_CHUNK before REASONING_END"
357 + )
358 + flushed = events[rc_idx].value
359 + assert "<" in flushed
360 +
361 + def test_empty_buffer_no_extra_events(self):
362 + """When the lexer buffer is empty, flushing is a no-op."""
363 + engine = StreamingParserEngine(_think_config(), _make_think_tokenizer())
364 +
365 + engine.feed("<think>", [_START_ID])
366 + engine.feed("clean text", [])
367 +
368 + events = engine.feed("</think>", [_END_ID])
369 + assert any(e.type == EventType.REASONING_END for e in events)
370 + chunk_events = [e for e in events if e.type == EventType.REASONING_CHUNK]
371 + assert all(e.value for e in chunk_events)
372 +
373 +
374 +class TestTokenIdFiltering:
375 + """When token IDs are available, lex-matched terminals that also
376 + have token_id_terminal entries should be demoted to content."""
377 +
378 + def test_lex_matched_terminal_demoted_after_token_ids_seen(self):
379 + """After receiving token IDs, text that matches a token-ID
380 + terminal should be treated as content, not trigger a transition."""
381 + engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer())
382 +
383 + # First feed with a non-special token ID to set _ever_had_token_ids
384 + engine.feed("prefix ", [1])
385 +
386 + # Now feed text containing <tool_call> as literal text
387 + events = engine.feed(
388 + "Use <tool_call> to invoke tools.</tool_call>", [2, 3, 4, 5]
389 + )
390 + events.extend(engine.finish())
391 +
392 + types = [e.type for e in events]
393 + assert EventType.TOOL_CALL_START not in types
394 + assert EventType.TEXT_CHUNK in types
395 +
396 + text = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
397 + assert "<tool_call>" in text
398 +
399 + def test_scanner_matched_terminal_bypasses_filter(self):
400 + """PreLexedTerminals from the scanner bypass the filter and
401 + still trigger state transitions."""
402 + engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer())
403 +
404 + events = engine.feed("<tool_call>", [_TOOL_START_ID])
405 + assert any(e.type == EventType.TOOL_CALL_START for e in events)
406 +
407 + events = engine.feed('{"name": "f"}', [2, 3])
408 + events.extend(engine.feed("</tool_call>", [_TOOL_END_ID]))
409 + events.extend(engine.finish())
410 + assert any(e.type == EventType.TOOL_CALL_END for e in events)
411 +
412 + def test_no_filtering_without_token_ids(self):
413 + """When no token IDs are ever provided (non-streaming),
414 + text matching still triggers transitions."""
415 + engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer())
416 +
417 + events = engine.feed('<tool_call>{"name": "f"}</tool_call>', [])
418 + events.extend(engine.finish())
419 +
420 + types = [e.type for e in events]
421 + assert EventType.TOOL_CALL_START in types
422 + assert EventType.TOOL_CALL_END in types
423 +
424 + def test_mixed_text_then_real_tool_call(self):
425 + """Text mentioning tool syntax followed by a real special-token
426 + tool call."""
427 + engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer())
428 +
429 + events1 = engine.feed("Mention <tool_call> in text. ", [1, 2, 3, 4])
430 + events2 = engine.feed("<tool_call>", [_TOOL_START_ID])
431 + events3 = engine.feed('{"name": "a"}', [5, 6])
432 + events4 = engine.feed("</tool_call>", [_TOOL_END_ID])
433 + events4.extend(engine.finish())
434 +
435 + all_events = events1 + events2 + events3 + events4
436 +
437 + content = "".join(e.value for e in all_events if e.type == EventType.TEXT_CHUNK)
438 + assert "<tool_call>" in content
439 +
440 + assert sum(1 for e in all_events if e.type == EventType.TOOL_CALL_START) == 1
441 + assert sum(1 for e in all_events if e.type == EventType.TOOL_CALL_END) == 1
442 +
443 +
444 +def _func_prefix_config() -> ParserEngineConfig:
445 + """Config mixing token-ID terminals (TOOL_START/END) with
446 + text-only terminals (FUNC_PREFIX) and fallback transitions."""
447 + return ParserEngineConfig(
448 + name="func_prefix_test",
449 + terminals={
450 + "TOOL_START": "<tool_call>",
451 + "TOOL_END": "</tool_call>",
452 + "FUNC_PREFIX": "<function=",
453 + "FUNC_END": "</function>",
454 + "CLOSE_ANGLE": ">",
455 + },
456 + token_id_terminals={
457 + "TOOL_START": "<tool_call>",
458 + "TOOL_END": "</tool_call>",
459 + },
460 + transitions={
461 + (ParserState.CONTENT, "TOOL_START"): Transition(
462 + ParserState.TOOL_PREAMBLE,
463 + (EventType.TOOL_CALL_START,),
464 + ),
465 + (ParserState.CONTENT, "FUNC_PREFIX"): Transition(
466 + ParserState.TOOL_NAME,
467 + (EventType.TOOL_CALL_START,),
468 + skip_in_token_id_mode=True,
469 + ),
470 + (ParserState.TOOL_PREAMBLE, "FUNC_PREFIX"): Transition(
471 + ParserState.TOOL_NAME,
472 + (),
473 + ),
474 + (ParserState.TOOL_NAME, "CLOSE_ANGLE"): Transition(
475 + ParserState.TOOL_ARGS,
476 + (),
477 + ),
478 + (ParserState.TOOL_ARGS, "FUNC_END"): Transition(
479 + ParserState.TOOL_BETWEEN,
480 + (EventType.TOOL_CALL_END,),
481 + ),
482 + (ParserState.TOOL_BETWEEN, "TOOL_END"): Transition(
483 + ParserState.CONTENT,
484 + (),
485 + ),
486 + (ParserState.TOOL_BETWEEN, "TOOL_START"): Transition(
487 + ParserState.TOOL_PREAMBLE,
488 + (EventType.TOOL_CALL_START,),
489 + ),
490 + (ParserState.TOOL_BETWEEN, "FUNC_PREFIX"): Transition(
491 + ParserState.TOOL_NAME,
492 + (EventType.TOOL_CALL_START,),
493 + skip_in_token_id_mode=True,
494 + ),
495 + },
496 + content_events={
497 + ParserState.CONTENT: EventType.TEXT_CHUNK,
498 + ParserState.TOOL_NAME: EventType.TOOL_NAME,
499 + ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
500 + },
501 + )
502 +
503 +
504 +def _make_func_prefix_tokenizer():
505 + return make_mock_tokenizer(
506 + {
507 + "<tool_call>": _TOOL_START_ID,
508 + "</tool_call>": _TOOL_END_ID,
509 + }
510 + )
511 +
512 +
513 +class TestTextOnlyFallbackFiltering:
514 + """When token IDs are available, transitions marked
515 + skip_in_token_id_mode should be skipped."""
516 +
517 + def test_func_prefix_in_prose_demoted_in_strict_mode(self):
518 + """<function=get_time> in prose should NOT trigger a tool call
519 + when strict mode is active."""
520 + engine = StreamingParserEngine(
521 + _func_prefix_config(), _make_func_prefix_tokenizer()
522 + )
523 + engine.feed("prefix ", [1])
524 +
525 + events = engine.feed("Use <function=get_time> to check.", [2, 3, 4, 5])
526 + events.extend(engine.finish())
527 +
528 + types = [e.type for e in events]
529 + assert EventType.TOOL_CALL_START not in types
530 + assert EventType.TEXT_CHUNK in types
531 + text = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
532 + assert "<function=" in text
533 +
534 + def test_normal_flow_after_tool_start_still_works(self):
535 + """TOOL_START (special token) -> FUNC_PREFIX (text) should
536 + still parse a tool call normally in strict mode."""
537 + engine = StreamingParserEngine(
538 + _func_prefix_config(), _make_func_prefix_tokenizer()
539 + )
540 +
541 + events1 = engine.feed("<tool_call>", [_TOOL_START_ID])
542 + assert any(e.type == EventType.TOOL_CALL_START for e in events1)
543 +
544 + events2 = engine.feed("<function=get_weather>", [2, 3])
545 + events3 = engine.feed("args", [4])
546 + events4 = engine.feed("</function>", [5, 6])
547 + events4.extend(engine.feed("</tool_call>", [_TOOL_END_ID]))
548 + events4.extend(engine.finish())
549 +
550 + all_events = events1 + events2 + events3 + events4
551 + assert sum(1 for e in all_events if e.type == EventType.TOOL_CALL_START) == 1
552 + assert sum(1 for e in all_events if e.type == EventType.TOOL_CALL_END) == 1
553 +
554 + def test_fallback_fires_without_token_ids(self):
555 + """When no token IDs are provided, fallback transitions should
556 + still fire normally."""
557 + engine = StreamingParserEngine(
558 + _func_prefix_config(), _make_func_prefix_tokenizer()
559 + )
560 +
561 + events = engine.feed("<function=get_time>args</function>", [])
562 + events.extend(engine.finish())
563 +
564 + types = [e.type for e in events]
565 + assert EventType.TOOL_CALL_START in types
566 + assert EventType.TOOL_CALL_END in types
567 +
568 + def test_tool_between_fallback_blocked_in_strict_mode(self):
569 + """The (TOOL_BETWEEN, FUNC_PREFIX) fallback should also be
570 + blocked in strict mode."""
571 + engine = StreamingParserEngine(
572 + _func_prefix_config(), _make_func_prefix_tokenizer()
573 + )
574 +
575 + engine.feed("<tool_call>", [_TOOL_START_ID])
576 + engine.feed("<function=a>", [2, 3])
577 + engine.feed("args", [4])
578 + engine.feed("</function>", [5, 6])
579 + engine.feed("</tool_call>", [_TOOL_END_ID])
580 +
581 + events = engine.feed("<function=b>more</function>", [7, 8, 9])
582 + events.extend(engine.finish())
583 +
584 + types = [e.type for e in events]
585 + assert EventType.TOOL_CALL_START not in types
586 +
587 +
588 +class TestNoUnusedTokenizerAttr:
589 + """StreamingParserEngine no longer stores a redundant _tokenizer."""
590 +
591 + def test_no_tokenizer_attribute(self):
592 + config = ParserEngineConfig(name="test")
593 + engine = StreamingParserEngine(config, tokenizer=None)
594 + assert not hasattr(engine, "_tokenizer")
595 +
596 +
597 +class TestArgsResetOnReentry:
598 + """When leaving TOOL_ARGS and later re-entering (e.g. two tool
599 + calls), the entering-TOOL_ARGS block resets args tracking. The
600 + redundant reset on exit was removed."""
601 +
602 + @staticmethod
603 + def _multi_tool_config() -> ParserEngineConfig:
604 + return ParserEngineConfig(
605 + name="multi_tool",
606 + terminals={
607 + "TOOL_START": "<tool_call>",
608 + "TOOL_END": "</tool_call>",
609 + "TOOL_SEP": "<tool_sep>",
610 + },
611 + transitions={
612 + (ParserState.CONTENT, "TOOL_START"): Transition(
613 + ParserState.TOOL_ARGS,
614 + (EventType.TOOL_CALL_START,),
615 + ),
616 + (ParserState.TOOL_ARGS, "TOOL_END"): Transition(
617 + ParserState.TOOL_BETWEEN,
618 + (EventType.TOOL_CALL_END,),
619 + ),
620 + (ParserState.TOOL_BETWEEN, "TOOL_SEP"): Transition(
621 + ParserState.TOOL_ARGS,
622 + (EventType.TOOL_CALL_START,),
623 + ),
624 + },
625 + content_events={
626 + ParserState.CONTENT: EventType.TEXT_CHUNK,
627 + ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
628 + },
629 + )
630 +
631 + def test_args_tracking_across_reentry(self):
632 + engine = StreamingParserEngine(self._multi_tool_config(), tokenizer=None)
633 +
634 + events = engine.feed(
635 + '<tool_call>{"city": "SF"}</tool_call>'
636 + "<tool_sep>"
637 + '{"name": "bar"}</tool_call>',
638 + [],
639 + )
640 +
641 + tool_starts = [e for e in events if e.type == EventType.TOOL_CALL_START]
642 + tool_ends = [e for e in events if e.type == EventType.TOOL_CALL_END]
643 + arg_chunks = [e for e in events if e.type == EventType.ARG_VALUE_CHUNK]
644 +
645 + assert len(tool_starts) == 2
646 + assert len(tool_ends) == 2
647 + assert tool_starts[0].tool_index == 0
648 + assert tool_starts[1].tool_index == 1
649 +
650 + first_args = "".join(e.value for e in arg_chunks if e.tool_index == 0)
651 + second_args = "".join(e.value for e in arg_chunks if e.tool_index == 1)
652 + assert '"city"' in first_args
653 + assert '"name"' in second_args
654 +
655 + def test_brace_depth_resets_on_reentry(self):
656 + """Verify _args_brace_depth resets when re-entering TOOL_ARGS."""
657 + engine = StreamingParserEngine(self._multi_tool_config(), tokenizer=None)
658 + engine.feed("<tool_call>", [])
659 + assert engine.state == ParserState.TOOL_ARGS
660 + assert engine._args_brace_depth == 0
661 +
662 + engine.feed('{"a": 1}', [])
663 + engine.feed("</tool_call>", [])
664 + assert engine.state == ParserState.TOOL_BETWEEN
665 +
666 + engine.feed("<tool_sep>", [])
667 + assert engine.state == ParserState.TOOL_ARGS
668 + assert engine._args_brace_depth == 0
669 + assert engine._args_in_string is False
670 + assert engine._args_escape_next is False
671 +
672 +
673 +class TestToolPreambleFinish:
674 + """finish() in TOOL_PREAMBLE state emits TOOL_CALL_END when a tool
675 + call was started (tool_index >= 0), but not when tool_index is -1."""
676 +
677 + @staticmethod
678 + def _preamble_with_tool_call_start_config() -> ParserEngineConfig:
679 + return ParserEngineConfig(
680 + name="preamble_tcs",
681 + terminals={"TOOL_START": "<tool_call>"},
682 + transitions={
683 + (ParserState.CONTENT, "TOOL_START"): Transition(
684 + ParserState.TOOL_PREAMBLE,
685 + (EventType.TOOL_CALL_START,),
686 + ),
687 + },
688 + content_events={ParserState.CONTENT: EventType.TEXT_CHUNK},
689 + )
690 +
691 + @staticmethod
692 + def _preamble_without_tool_call_start_config() -> ParserEngineConfig:
693 + return ParserEngineConfig(
694 + name="preamble_no_tcs",
695 + terminals={"TOOL_CALLS_START": "<tool_calls>"},
696 + transitions={
697 + (ParserState.CONTENT, "TOOL_CALLS_START"): Transition(
698 + ParserState.TOOL_PREAMBLE,
699 + (),
700 + ),
701 + },
702 + content_events={ParserState.CONTENT: EventType.TEXT_CHUNK},
703 + )
704 +
705 + def test_finish_emits_tool_call_end_with_tool_index(self):
706 + config = self._preamble_with_tool_call_start_config()
707 + engine = StreamingParserEngine(config, tokenizer=None)
708 +
709 + engine.feed("<tool_call>", [])
710 + assert engine.state == ParserState.TOOL_PREAMBLE
711 + assert engine.tool_index == 0
712 +
713 + finish_events = engine.finish()
714 + end_events = [e for e in finish_events if e.type == EventType.TOOL_CALL_END]
715 + assert len(end_events) == 1
716 + assert end_events[0].tool_index == 0
717 +
718 + def test_finish_no_tool_call_end_without_tool_index(self):
719 + config = self._preamble_without_tool_call_start_config()
720 + engine = StreamingParserEngine(config, tokenizer=None)
721 +
722 + engine.feed("<tool_calls>", [])
723 + assert engine.state == ParserState.TOOL_PREAMBLE
724 + assert engine.tool_index == -1
725 +
726 + finish_events = engine.finish()
727 + end_events = [e for e in finish_events if e.type == EventType.TOOL_CALL_END]
728 + assert len(end_events) == 0
729 + assert engine.state == ParserState.CONTENT
730 +
731 +
732 +class TestRegexTerminalInfraRemoved:
733 + """TerminalDef.priority, LexerShape.regex_terminals, and the regex
734 + matching loop were removed."""
735 +
736 + def test_terminal_def_no_priority(self):
737 + import regex as re
738 +
739 + td = TerminalDef(name="X", pattern=re.compile("x"))
740 + assert not hasattr(td, "priority")
741 +
742 + def test_lexer_shape_no_regex_terminals(self):
743 + shape = LexerShape([])
744 + assert not hasattr(shape, "regex_terminals")
745 +
746 + def test_terminals_from_literals_still_works(self):
747 + literals = {"TOOL_START": "<tool_call>", "TOOL_END": "</tool_call>"}
748 + defs = terminals_from_literals(literals)
749 + assert len(defs) == 2
750 + names = {d.name for d in defs}
751 + assert names == {"TOOL_START", "TOOL_END"}
752 + for d in defs:
753 + assert d.is_literal
754 + assert d.literal in ("<tool_call>", "</tool_call>")
755 +
756 +
757 +class TestMultiCharTerminalInArgs:
758 + """Regression: multi-char terminals falling through in TOOL_ARGS
759 + must be fed char-by-char via _feed_args_text, not _feed_args_char."""
760 +
761 + @staticmethod
762 + def _newline_config() -> ParserEngineConfig:
763 + return ParserEngineConfig(
764 + name="newline_test",
765 + terminals={
766 + "TOOL_START": "<tool_call>",
767 + "TOOL_END": "</tool_call>",
768 + "NEWLINE": "\n",
769 + },
770 + transitions={
771 + (ParserState.CONTENT, "TOOL_START"): Transition(
772 + ParserState.TOOL_ARGS,
773 + (EventType.TOOL_CALL_START,),
774 + ),
775 + (ParserState.TOOL_ARGS, "TOOL_END"): Transition(
776 + ParserState.CONTENT,
777 + (EventType.TOOL_CALL_END,),
778 + ),
779 + },
780 + content_events={
781 + ParserState.CONTENT: EventType.TEXT_CHUNK,
782 + ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
783 + },
784 + )
785 +
786 + def test_newline_in_args_parsed_correctly(self):
787 + engine = StreamingParserEngine(self._newline_config(), tokenizer=None)
788 + text = '<tool_call>{"name": "f",\n"arguments": {"a": 1}}</tool_call>'
789 + events = engine.parse_complete(text)
790 +
791 + arg_text = "".join(
792 + e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK
793 + )
794 + assert '"name": "f"' in arg_text
795 + assert '"arguments"' in arg_text
796 +
797 + def test_newline_in_args_streaming(self):
798 + engine = StreamingParserEngine(self._newline_config(), tokenizer=None)
799 + all_events = TestStreaming._feed_chars(
800 + engine, '<tool_call>{"name": "f",\n"a": 1}</tool_call>'
801 + )
802 +
803 + arg_text = "".join(
804 + e.value for e in all_events if e.type == EventType.ARG_VALUE_CHUNK
805 + )
806 + assert '"name": "f"' in arg_text
807 + assert '"a": 1' in arg_text
808 +
809 +
810 +class TestSkipToolParsing:
811 + """When skip_tool_parsing is set, tool tags become content."""
812 +
813 + def test_tool_tags_emitted_as_content(self):
814 + engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
815 + engine.skip_tool_parsing = True
816 +
817 + text = '<tool_call>{"name": "f"}</tool_call>'
818 + events = engine.parse_complete(text)
819 +
820 + types = [e.type for e in events]
821 + assert EventType.TOOL_CALL_START not in types
822 + assert EventType.TOOL_CALL_END not in types
823 +
824 + content = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
825 + assert "<tool_call>" in content
826 + assert "</tool_call>" in content
827 +
828 + def test_skip_tool_streaming(self):
829 + engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
830 + engine.skip_tool_parsing = True
831 +
832 + all_events = TestStreaming._feed_chars(
833 + engine, '<tool_call>{"name": "f"}</tool_call>'
834 + )
835 +
836 + types = [e.type for e in all_events]
837 + assert EventType.TOOL_CALL_START not in types
838 +
839 + content = "".join(e.value for e in all_events if e.type == EventType.TEXT_CHUNK)
840 + assert "<tool_call>" in content
841 +
842 + def test_reset_preserves_skip_tool_parsing(self):
843 + engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
844 + engine.skip_tool_parsing = True
845 + engine.reset()
846 + assert engine.skip_tool_parsing is True
tests/parser/engine/test_parser_engine.py new
+1356
@@ -0,0 +1,1356 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Tests for :class:`ParserEngine` — the glue layer between
4 +:class:`StreamingParserEngine` events and the serving layer's
5 +DeltaMessage / ExtractedToolCallInformation protocol.
6 +"""
7 +
8 +from __future__ import annotations
9 +
10 +import json
11 +from types import SimpleNamespace
12 +from unittest.mock import MagicMock
13 +
14 +import pytest
15 +import regex as re
16 +
17 +from tests.parser.engine.conftest import make_mock_tokenizer
18 +from vllm.entrypoints.openai.chat_completion.protocol import (
19 + ChatCompletionRequest,
20 + ChatCompletionToolsParam,
21 +)
22 +from vllm.entrypoints.openai.engine.protocol import (
23 + DeltaFunctionCall,
24 + DeltaToolCall,
25 + FunctionDefinition,
26 +)
27 +from vllm.parser.abstract_parser import DelegatingParser
28 +from vllm.parser.engine.adapters import make_adapters
29 +from vllm.parser.engine.events import EventType, SemanticEvent
30 +from vllm.parser.engine.parser_engine import ParserEngine
31 +from vllm.parser.engine.parser_engine_config import (
32 + ParserEngineConfig,
33 + ParserState,
34 + Transition,
35 +)
36 +
37 +# ── Shared test configs ──────────────────────────────────────────────
38 +
39 +_VOCAB: dict[str, int] = {
40 + "<think>": 200,
41 + "</think>": 201,
42 + "<tool_call>": 202,
43 + "</tool_call>": 203,
44 +}
45 +
46 +
47 +def _combined_config() -> ParserEngineConfig:
48 + """Config with reasoning tags and tool-call tags."""
49 + return ParserEngineConfig(
50 + name="combined_test",
51 + terminals={
52 + "THINK_START": "<think>",
53 + "THINK_END": "</think>",
54 + "TOOL_START": "<tool_call>",
55 + "TOOL_END": "</tool_call>",
56 + },
57 + token_id_terminals={
58 + "THINK_START": "<think>",
59 + "THINK_END": "</think>",
60 + "TOOL_START": "<tool_call>",
61 + "TOOL_END": "</tool_call>",
62 + },
63 + transitions={
64 + (ParserState.REASONING, "THINK_END"): Transition(
65 + ParserState.CONTENT,
66 + (EventType.REASONING_END,),
67 + ),
68 + (ParserState.CONTENT, "THINK_START"): Transition(
69 + ParserState.REASONING,
70 + (EventType.REASONING_START,),
71 + ),
72 + (ParserState.CONTENT, "TOOL_START"): Transition(
73 + ParserState.TOOL_ARGS,
74 + (EventType.TOOL_CALL_START,),
75 + ),
76 + (ParserState.TOOL_ARGS, "TOOL_END"): Transition(
77 + ParserState.CONTENT,
78 + (EventType.TOOL_CALL_END,),
79 + ),
80 + },
81 + initial_state=ParserState.REASONING,
82 + content_events={
83 + ParserState.CONTENT: EventType.TEXT_CHUNK,
84 + ParserState.REASONING: EventType.REASONING_CHUNK,
85 + ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
86 + },
87 + )
88 +
89 +
90 +def _hermes_config() -> ParserEngineConfig:
91 + """Tool-call-only config (no reasoning)."""
92 + return ParserEngineConfig(
93 + name="hermes_test",
94 + terminals={
95 + "TOOL_START": "<tool_call>",
96 + "TOOL_END": "</tool_call>",
97 + },
98 + token_id_terminals={
99 + "TOOL_START": "<tool_call>",
100 + "TOOL_END": "</tool_call>",
101 + },
102 + transitions={
103 + (ParserState.CONTENT, "TOOL_START"): Transition(
104 + ParserState.TOOL_ARGS,
105 + (EventType.TOOL_CALL_START,),
106 + ),
107 + (ParserState.TOOL_ARGS, "TOOL_END"): Transition(
108 + ParserState.CONTENT,
109 + (EventType.TOOL_CALL_END,),
110 + ),
111 + },
112 + content_events={
113 + ParserState.CONTENT: EventType.TEXT_CHUNK,
114 + ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
115 + },
116 + )
117 +
118 +
119 +def _make_engine(
120 + config: ParserEngineConfig | None = None,
121 + tools: list | None = None,
122 +) -> ParserEngine:
123 + tokenizer = make_mock_tokenizer(_VOCAB)
124 + cfg = config or _combined_config()
125 + return ParserEngine(
126 + tokenizer,
127 + tools=tools,
128 + parser_engine_config=cfg,
129 + )
130 +
131 +
132 +# ── TestEventsToDelta ────────────────────────────────────────────────
133 +
134 +
135 +class TestEventsToDelta:
136 + """Unit tests for ParserEngine._events_to_delta()."""
137 +
138 + def test_text_chunk_produces_content(self):
139 + engine = _make_engine()
140 + delta = engine._events_to_delta(
141 + [
142 + SemanticEvent(EventType.TEXT_CHUNK, "Hello world"),
143 + ]
144 + )
145 + assert delta is not None
146 + assert delta.content == "Hello world"
147 + assert not delta.tool_calls
148 +
149 + def test_reasoning_chunk_produces_reasoning(self):
150 + engine = _make_engine()
151 + delta = engine._events_to_delta(
152 + [
153 + SemanticEvent(EventType.REASONING_CHUNK, "Let me think"),
154 + ]
155 + )
156 + assert delta is not None
157 + assert delta.reasoning == "Let me think"
158 + assert delta.content is None
159 +
160 + def test_empty_events_returns_none(self):
161 + engine = _make_engine()
162 + delta = engine._events_to_delta([])
163 + assert delta is None
164 +
165 + def test_tool_call_produces_tool_call_delta(self):
166 + engine = _make_engine()
167 + events = [
168 + SemanticEvent(EventType.TOOL_CALL_START, tool_index=0),
169 + SemanticEvent(EventType.TOOL_NAME, "get_weather", tool_index=0),
170 + SemanticEvent(
171 + EventType.ARG_VALUE_CHUNK,
172 + '{"location": "NYC"}',
173 + tool_index=0,
174 + ),
175 + SemanticEvent(EventType.TOOL_CALL_END, tool_index=0),
176 + ]
177 + delta = engine._events_to_delta(events)
178 + assert delta is not None
179 + assert len(delta.tool_calls) > 0
180 + names = [
181 + tc.function.name
182 + for tc in delta.tool_calls
183 + if tc.function and tc.function.name
184 + ]
185 + assert "get_weather" in names
186 +
187 + def test_reasoning_end_sets_flag(self):
188 + engine = _make_engine()
189 + assert engine._reasoning_ended is False
190 + engine._events_to_delta([SemanticEvent(EventType.REASONING_END)])
191 + assert engine._reasoning_ended is True
192 +
193 + def test_mixed_content_and_reasoning(self):
194 + engine = _make_engine()
195 + events = [
196 + SemanticEvent(EventType.REASONING_CHUNK, "thinking..."),
197 + SemanticEvent(EventType.REASONING_END),
198 + SemanticEvent(EventType.TEXT_CHUNK, "answer"),
199 + ]
200 + delta = engine._events_to_delta(events)
201 + assert delta is not None
202 + assert delta.reasoning == "thinking..."
203 + assert delta.content == "answer"
204 +
205 + @pytest.mark.parametrize(
206 + "events,expected,excluded",
207 + [
208 + (
209 + [SemanticEvent(EventType.TEXT_CHUNK, "Hello world")],
210 + "content",
211 + ["tool_calls", "reasoning"],
212 + ),
213 + (
214 + [SemanticEvent(EventType.REASONING_CHUNK, "Let me think")],
215 + "reasoning",
216 + ["tool_calls", "content"],
217 + ),
218 + (
219 + [
220 + SemanticEvent(EventType.TOOL_CALL_START, tool_index=0),
221 + SemanticEvent(EventType.TOOL_NAME, "fn", tool_index=0),
222 + SemanticEvent(EventType.ARG_VALUE_CHUNK, '{"k":1}', tool_index=0),
223 + SemanticEvent(EventType.TOOL_CALL_END, tool_index=0),
224 + ],
225 + "tool_calls",
226 + ["content", "reasoning"],
227 + ),
228 + ],
229 + ids=["content_only", "reasoning_only", "tool_call_only"],
230 + )
231 + def test_delta_excludes_unset_fields(self, events, expected, excluded):
232 + engine = _make_engine()
233 + delta = engine._events_to_delta(events)
234 + assert delta is not None
235 + dumped = delta.model_dump(exclude_unset=True)
236 + assert expected in dumped
237 + for field in excluded:
238 + assert field not in dumped
239 +
240 + def test_kimi_k2_tool_call_id_includes_func_name(self):
241 + engine = _make_engine()
242 + engine._stream_state.tool_call_id_type = "kimi_k2"
243 + events = [
244 + SemanticEvent(EventType.TOOL_CALL_START, tool_index=0),
245 + SemanticEvent(EventType.TOOL_NAME, "get_weather", tool_index=0),
246 + SemanticEvent(
247 + EventType.ARG_VALUE_CHUNK,
248 + '{"city": "NYC"}',
249 + tool_index=0,
250 + ),
251 + SemanticEvent(EventType.TOOL_CALL_END, tool_index=0),
252 + ]
253 + delta = engine._events_to_delta(events)
254 + assert delta is not None
255 + assert len(delta.tool_calls) == 1
256 + assert delta.tool_calls[0].id == "functions.get_weather:0"
257 +
258 + def test_multiple_arg_chunks_same_batch_coalesced(self):
259 + """Multiple events for the same tool in one batch must produce
260 + at most one DeltaToolCall per index."""
261 + engine = _make_engine()
262 + events = [
263 + SemanticEvent(EventType.TOOL_CALL_START, tool_index=0),
264 + SemanticEvent(EventType.TOOL_NAME, "get_weather", tool_index=0),
265 + SemanticEvent(
266 + EventType.ARG_VALUE_CHUNK,
267 + '{"city": ',
268 + tool_index=0,
269 + ),
270 + SemanticEvent(
271 + EventType.ARG_VALUE_CHUNK,
272 + '"Tokyo"}',
273 + tool_index=0,
274 + ),
275 + SemanticEvent(EventType.TOOL_CALL_END, tool_index=0),
276 + ]
277 + delta = engine._events_to_delta(events)
278 + assert delta is not None
279 + indices = [tc.index for tc in delta.tool_calls]
280 + assert len(indices) == len(set(indices)), (
281 + f"Duplicate indices in tool_calls: {delta.tool_calls}"
282 + )
283 + assert delta.tool_calls[0].function.name == "get_weather"
284 + assert delta.tool_calls[0].id is not None
285 +
286 +
287 +# ── TestCoalesceToolCallDeltas ──────────────────────────────────────
288 +
289 +
290 +class TestCoalesceToolCallDeltas:
291 + """Unit tests for ParserEngine._coalesce_tool_call_deltas()."""
292 +
293 + def test_no_duplicates_unchanged(self):
294 + deltas = [
295 + DeltaToolCall(
296 + index=0,
297 + id="a",
298 + type="function",
299 + function=DeltaFunctionCall(name="f"),
300 + ),
301 + DeltaToolCall(
302 + index=1,
303 + function=DeltaFunctionCall(arguments="{}"),
304 + ),
305 + ]
306 + result = ParserEngine._coalesce_tool_call_deltas(deltas)
307 + assert len(result) == 2
308 + assert result[0].index == 0
309 + assert result[1].index == 1
310 +
311 + def test_name_and_args_same_index_merged(self):
312 + deltas = [
313 + DeltaToolCall(
314 + index=0,
315 + id="call_1",
316 + type="function",
317 + function=DeltaFunctionCall(name="get_weather"),
318 + ),
319 + DeltaToolCall(
320 + index=0,
321 + function=DeltaFunctionCall(arguments='{"city":'),
322 + ),
323 + DeltaToolCall(
324 + index=0,
325 + function=DeltaFunctionCall(arguments='"Tokyo"}'),
326 + ),
327 + ]
328 + result = ParserEngine._coalesce_tool_call_deltas(deltas)
329 + assert len(result) == 1
330 + assert result[0].index == 0
331 + assert result[0].id == "call_1"
332 + assert result[0].type == "function"
333 + assert result[0].function.name == "get_weather"
334 + assert result[0].function.arguments == '{"city":"Tokyo"}'
335 +
336 + def test_empty_list(self):
337 + assert ParserEngine._coalesce_tool_call_deltas([]) == []
338 +
339 + def test_single_element(self):
340 + tc = DeltaToolCall(
341 + index=0,
342 + function=DeltaFunctionCall(name="f"),
343 + )
344 + result = ParserEngine._coalesce_tool_call_deltas([tc])
345 + assert result == [tc]
346 +
347 + def test_partial_duplicates(self):
348 + deltas = [
349 + DeltaToolCall(
350 + index=0,
351 + id="a",
352 + type="function",
353 + function=DeltaFunctionCall(name="f1"),
354 + ),
355 + DeltaToolCall(
356 + index=1,
357 + id="b",
358 + type="function",
359 + function=DeltaFunctionCall(name="f2"),
360 + ),
361 + DeltaToolCall(
362 + index=0,
363 + function=DeltaFunctionCall(arguments='{"x":1}'),
364 + ),
365 + ]
366 + result = ParserEngine._coalesce_tool_call_deltas(deltas)
367 + assert len(result) == 2
368 + assert result[0].index == 0
369 + assert result[0].function.name == "f1"
370 + assert result[0].function.arguments == '{"x":1}'
371 + assert result[1].index == 1
372 +
373 + def test_id_type_from_later_entry(self):
374 + deltas = [
375 + DeltaToolCall(
376 + index=0,
377 + function=DeltaFunctionCall(arguments='{"a":1}'),
378 + ),
379 + DeltaToolCall(
380 + index=0,
381 + id="call_1",
382 + type="function",
383 + function=DeltaFunctionCall(name="f"),
384 + ),
385 + ]
386 + result = ParserEngine._coalesce_tool_call_deltas(deltas)
387 + assert len(result) == 1
388 + assert result[0].id == "call_1"
389 + assert result[0].type == "function"
390 + assert result[0].function.name == "f"
391 + assert result[0].function.arguments == '{"a":1}'
392 +
393 +
394 +# ── TestContentWhitespaceHandling ────────────────────────────────────
395 +
396 +
397 +class TestContentWhitespaceHandling:
398 + """Unit tests for whitespace deferral / dropping in _events_to_delta."""
399 +
400 + def test_whitespace_only_deferred_until_next_tick(self):
401 + engine = _make_engine()
402 + d1 = engine._events_to_delta(
403 + [SemanticEvent(EventType.TEXT_CHUNK, " \n")],
404 + )
405 + assert d1 is None
406 + d2 = engine._events_to_delta(
407 + [SemanticEvent(EventType.TEXT_CHUNK, "hello")],
408 + )
409 + assert d2 is not None
410 + assert d2.content == " \nhello"
411 +
412 + def test_whitespace_only_emitted_on_finished(self):
413 + engine = _make_engine()
414 + d = engine._events_to_delta(
415 + [SemanticEvent(EventType.TEXT_CHUNK, " \n")],
416 + finished=True,
417 + )
418 + assert d is not None
419 + assert d.content == " \n"
420 +
421 + def test_whitespace_dropped_before_tool_call(self):
422 + engine = _make_engine()
423 + engine._events_to_delta(
424 + [
425 + SemanticEvent(EventType.TEXT_CHUNK, " \n"),
426 + SemanticEvent(EventType.TOOL_CALL_START, tool_index=0),
427 + ]
428 + )
429 + d = engine._events_to_delta(
430 + [
431 + SemanticEvent(EventType.TOOL_NAME, "f", tool_index=0),
432 + SemanticEvent(
433 + EventType.ARG_VALUE_CHUNK,
434 + '{"a":1}',
435 + tool_index=0,
436 + ),
437 + SemanticEvent(EventType.TOOL_CALL_END, tool_index=0),
438 + ]
439 + )
440 + assert d is not None
441 + assert d.content is None
442 + assert d.tool_calls
443 +
444 + def test_real_content_before_tool_preserved(self):
445 + engine = _make_engine()
446 + d = engine._events_to_delta(
447 + [
448 + SemanticEvent(EventType.TEXT_CHUNK, "prefix"),
449 + SemanticEvent(EventType.TOOL_CALL_START, tool_index=0),
450 + ]
451 + )
452 + assert d is not None
453 + assert d.content == "prefix"
454 +
455 + def test_whitespace_after_nonws_content_preserved(self):
456 + engine = _make_engine()
457 + engine._events_to_delta(
458 + [SemanticEvent(EventType.TEXT_CHUNK, "hello")],
459 + )
460 + d = engine._events_to_delta(
461 + [SemanticEvent(EventType.TEXT_CHUNK, " \n")],
462 + )
463 + assert d is not None
464 + assert d.content == " \n"
465 +
466 + def test_whitespace_after_nonws_not_dropped_with_tools(self):
467 + engine = _make_engine()
468 + engine._events_to_delta(
469 + [SemanticEvent(EventType.TEXT_CHUNK, "hello")],
470 + )
471 + d = engine._events_to_delta(
472 + [
473 + SemanticEvent(EventType.TEXT_CHUNK, " \n"),
474 + SemanticEvent(EventType.TOOL_CALL_START, tool_index=0),
475 + ]
476 + )
477 + assert d is not None
478 + assert d.content == " \n"
479 +
480 +
481 +# ── TestPostToolContentDeferral ──────────────────────────────────────
482 +
483 +
484 +class TestPostToolContentDeferral:
485 + """Regression: content after TOOL_CALL_END in the same batch must not
486 + produce a mixed DeltaMessage(content=..., tool_calls=...) — that causes
487 + split_delta to reorder content before tool_calls, breaking the Responses
488 + API state machine."""
489 +
490 + def test_text_after_tool_end_deferred(self):
491 + engine = _make_engine()
492 + events = [
493 + SemanticEvent(EventType.TOOL_CALL_START, tool_index=0),
494 + SemanticEvent(EventType.TOOL_NAME, "get_weather", tool_index=0),
495 + SemanticEvent(EventType.ARG_VALUE_CHUNK, '{"city":"NYC"}', tool_index=0),
496 + SemanticEvent(EventType.TOOL_CALL_END, tool_index=0),
497 + SemanticEvent(EventType.TEXT_CHUNK, "\nHere is the result"),
498 + ]
499 + delta = engine._events_to_delta(events)
500 + assert delta is not None
501 + assert delta.tool_calls
502 + assert delta.content is None
503 +
504 + deferred = engine._events_to_delta([])
505 + assert deferred is not None
506 + assert deferred.content == "\nHere is the result"
507 + assert not deferred.tool_calls
508 +
509 + def test_text_after_tool_deferred_even_when_finished(self):
510 + engine = _make_engine()
511 + events = [
512 + SemanticEvent(EventType.TOOL_CALL_START, tool_index=0),
513 + SemanticEvent(EventType.TOOL_NAME, "f", tool_index=0),
514 + SemanticEvent(EventType.ARG_VALUE_CHUNK, "{}", tool_index=0),
515 + SemanticEvent(EventType.TOOL_CALL_END, tool_index=0),
516 + SemanticEvent(EventType.TEXT_CHUNK, "done"),
517 + ]
518 + delta = engine._events_to_delta(events, finished=True)
519 + assert delta is not None
520 + assert delta.tool_calls
521 + assert delta.content is None
522 +
523 + def test_text_before_tool_not_deferred(self):
524 + engine = _make_engine()
525 + engine._content_has_nonws = True
526 + events = [
527 + SemanticEvent(EventType.TEXT_CHUNK, "hello"),
528 + SemanticEvent(EventType.TOOL_CALL_START, tool_index=0),
529 + SemanticEvent(EventType.TOOL_NAME, "f", tool_index=0),
530 + SemanticEvent(EventType.ARG_VALUE_CHUNK, "{}", tool_index=0),
531 + SemanticEvent(EventType.TOOL_CALL_END, tool_index=0),
532 + ]
533 + delta = engine._events_to_delta(events)
534 + assert delta is not None
535 + assert delta.content == "hello"
536 + assert delta.tool_calls
537 +
538 + def test_deferred_content_not_flushed_during_arg_continuation(self):
539 + """Deferred content from batch N must not mix with arg-continuation
540 + tool events in batch N+1 — that creates a DeltaMessage with both
541 + content and nameless tool_calls, which crashes the Responses API
542 + state machine (name=None → Pydantic ValidationError)."""
543 + engine = _make_engine()
544 + engine._content_has_nonws = True
545 +
546 + batch1 = [
547 + SemanticEvent(EventType.TOOL_CALL_START, tool_index=0),
548 + SemanticEvent(EventType.TOOL_NAME, "get_weather", tool_index=0),
549 + SemanticEvent(EventType.ARG_VALUE_CHUNK, '{"city":', tool_index=0),
550 + SemanticEvent(EventType.TEXT_CHUNK, "\n"),
551 + ]
552 + delta1 = engine._events_to_delta(batch1)
553 + assert delta1 is not None
554 + assert delta1.tool_calls
555 + assert delta1.content is None
556 +
557 + batch2 = [
558 + SemanticEvent(EventType.ARG_VALUE_CHUNK, '"NYC"}', tool_index=0),
559 + ]
560 + delta2 = engine._events_to_delta(batch2)
561 + assert delta2 is not None
562 + assert delta2.tool_calls
563 + assert delta2.content is None
564 +
565 + flush = engine._events_to_delta([])
566 + assert flush is not None
567 + assert flush.content == "\n"
568 + assert not flush.tool_calls
569 +
570 +
571 +# ── TestFixArgTypes ──────────────────────────────────────────────────
572 +
573 +
574 +def _make_tool(name: str, properties: dict) -> ChatCompletionToolsParam:
575 + return ChatCompletionToolsParam(
576 + type="function",
577 + function=FunctionDefinition(
578 + name=name,
579 + parameters={"type": "object", "properties": properties},
580 + ),
581 + )
582 +
583 +
584 +class TestFixArgTypes:
585 + """Tests for ParserEngine._fix_arg_types()."""
586 +
587 + def test_string_param_reverted_from_int(self):
588 + tool = _make_tool("f", {"zipcode": {"type": "string"}})
589 + engine = _make_engine(tools=[tool])
590 + result = engine._fix_arg_types('{"zipcode": 12345}', "f")
591 + assert '"zipcode": "12345"' in result
592 +
593 + def test_string_param_reverted_from_bool(self):
594 + tool = _make_tool("f", {"flag": {"type": "string"}})
595 + engine = _make_engine(tools=[tool])
596 + result = engine._fix_arg_types('{"flag": true}', "f")
597 + assert '"flag": "true"' in result
598 +
599 + def test_string_param_reverted_from_null(self):
600 + tool = _make_tool("f", {"val": {"type": "string"}})
601 + engine = _make_engine(tools=[tool])
602 + result = engine._fix_arg_types('{"val": null}', "f")
603 + assert '"val": "null"' in result
604 +
605 + def test_int_param_not_changed(self):
606 + tool = _make_tool("f", {"count": {"type": "integer"}})
607 + engine = _make_engine(tools=[tool])
608 + result = engine._fix_arg_types('{"count": 42}', "f")
609 + assert '"count": 42' in result
610 +
611 + def test_no_tools_returns_unchanged(self):
612 + engine = _make_engine(tools=None)
613 + original = '{"a": 1}'
614 + assert engine._fix_arg_types(original, "f") == original
615 +
616 + def test_unknown_function_returns_unchanged(self):
617 + tool = _make_tool("known", {"x": {"type": "string"}})
618 + engine = _make_engine(tools=[tool])
619 + original = '{"x": 1}'
620 + assert engine._fix_arg_types(original, "unknown") == original
621 +
622 + def test_invalid_json_returns_unchanged(self):
623 + tool = _make_tool("f", {"x": {"type": "string"}})
624 + engine = _make_engine(tools=[tool])
625 + original = "not json"
626 + assert engine._fix_arg_types(original, "f") == original
627 +
628 + def test_string_value_not_touched(self):
629 + tool = _make_tool("f", {"name": {"type": "string"}})
630 + engine = _make_engine(tools=[tool])
631 + original = '{"name": "Alice"}'
632 + assert engine._fix_arg_types(original, "f") == original
633 +
634 +
635 +# ── TestBuildExtractedResult ─────────────────────────────────────────
636 +
637 +
638 +class TestBuildExtractedResult:
639 + """Tests for ParserEngine._build_extracted_result()."""
640 +
641 + def test_no_tool_calls(self):
642 + engine = _make_engine()
643 + result = engine._build_extracted_result()
644 + assert result.tools_called is False
645 + assert result.tool_calls == []
646 +
647 + def test_single_tool_call(self):
648 + engine = _make_engine(_hermes_config())
649 + text = '<tool_call>{"name": "f", "arguments": {"a": 1}}</tool_call>'
650 + events = engine._engine.feed(text, [])
651 + events.extend(engine._engine.finish())
652 + delta = engine._events_to_delta(events)
653 + result = engine._build_extracted_result(delta)
654 + assert result.tools_called is True
655 + assert len(result.tool_calls) == 1
656 + assert result.tool_calls[0].function.name == "f"
657 +
658 + def test_content_passthrough(self):
659 + engine = _make_engine(_hermes_config())
660 + text = "Hello world"
661 + events = engine._engine.feed(text, [])
662 + events.extend(engine._engine.finish())
663 + delta = engine._events_to_delta(events, finished=True)
664 + result = engine._build_extracted_result(delta)
665 + assert result.tools_called is False
666 + assert result.content == "Hello world"
667 +
668 +
669 +# ── TestEngineBasedPath ──────────────────────────────────────────────
670 +
671 +
672 +class TestEngineBasedPath:
673 + """Tests for the _engine_based accumulation behavior in
674 + DelegatingParser.parse_delta."""
675 +
676 + def test_engine_based_true_when_both_parsers_engine(self):
677 + r = SimpleNamespace(engine_based_streaming=True)
678 + t = SimpleNamespace(engine_based_streaming=True)
679 + engine_based = r.engine_based_streaming and t.engine_based_streaming
680 + assert engine_based is True
681 +
682 + def test_engine_based_false_when_reasoning_parser_not_engine(self):
683 + r = SimpleNamespace(engine_based_streaming=False)
684 + t = SimpleNamespace(engine_based_streaming=True)
685 + engine_based = r.engine_based_streaming and t.engine_based_streaming
686 + assert engine_based is False
687 +
688 + def test_parse_delta_streaming(self, mock_request):
689 + """Engine's parse_delta returns content from streaming events."""
690 + engine = _make_engine(_hermes_config())
691 + engine._streaming_initialized = True
692 + result = engine.parse_delta(
693 + "Hello",
694 + [],
695 + mock_request,
696 + finished=False,
697 + )
698 + assert result is not None
699 + assert result.content == "Hello"
700 +
701 + def test_parse_delta_tool_call(self, mock_request):
702 + """Engine's parse_delta handles tool calls in streaming."""
703 + engine = _make_engine(_hermes_config())
704 + engine._streaming_initialized = True
705 + result = engine.parse_delta(
706 + '<tool_call>{"name": "f", "arguments": {}}</tool_call>',
707 + [],
708 + mock_request,
709 + finished=True,
710 + )
711 + assert result is not None
712 + assert len(result.tool_calls) > 0
713 +
714 +
715 +# ── TestParseTokenIdPassthrough ────────────────────────────────────
716 +
717 +
718 +class TestParseTokenIdPassthrough:
719 + """parse() must forward model_output_token_ids to _single_pass_parse
720 + so that token-ID-based strict terminal matching is active."""
721 +
722 + def test_literal_tool_tag_in_content_preserved_with_token_ids(self, mock_request):
723 + engine = _make_engine(_hermes_config())
724 + text = (
725 + "Use <tool_call> to call tools."
726 + '<tool_call>{"name": "f", "arguments": {"a": 1}}</tool_call>'
727 + )
728 + token_ids = [
729 + 65,
730 + 66,
731 + 67,
732 + 68,
733 + 69,
734 + 70,
735 + 71, # "Use <tool_call> to call tools."
736 + 202, # real <tool_call>
737 + 72,
738 + 73,
739 + 74, # '{"name": "f", ...}'
740 + 203, # real </tool_call>
741 + ]
742 +
743 + _, content, tool_calls = engine.parse(
744 + text, mock_request, model_output_token_ids=token_ids
745 + )
746 +
747 + assert content is not None
748 + assert "<tool_call>" in content
749 + assert tool_calls is not None
750 + assert len(tool_calls) == 1
751 + assert tool_calls[0].name == "f"
752 +
753 + def test_parse_with_token_ids_basic(self, mock_request):
754 + engine = _make_engine(_hermes_config())
755 + text = '<tool_call>{"name": "h", "arguments": {"x": 1}}</tool_call>'
756 + token_ids = [202, 65, 66, 67, 203]
757 +
758 + _, content, tool_calls = engine.parse(
759 + text, mock_request, model_output_token_ids=token_ids
760 + )
761 +
762 + assert tool_calls is not None
763 + assert len(tool_calls) == 1
764 + assert tool_calls[0].name == "h"
765 +
766 + def test_parse_without_token_ids_backward_compat(self, mock_request):
767 + engine = _make_engine(_hermes_config())
768 + text = '<tool_call>{"name": "g", "arguments": {}}</tool_call>'
769 +
770 + _, content, tool_calls = engine.parse(text, mock_request)
771 +
772 + assert tool_calls is not None
773 + assert len(tool_calls) == 1
774 + assert tool_calls[0].name == "g"
775 +
776 +
777 +# ── TestAdapterFinishOnStreamEnd ────────────────────────────────────
778 +
779 +
780 +class _CombinedTestEngine(ParserEngine):
781 + def __init__(self, tokenizer, tools=None, **kwargs):
782 + super().__init__(
783 + tokenizer, tools, parser_engine_config=_combined_config(), **kwargs
784 + )
785 +
786 +
787 +_CombinedReasoningAdapter, _CombinedToolAdapter = make_adapters(_CombinedTestEngine)
788 +
789 +
790 +class _CombinedDelegating(DelegatingParser):
791 + reasoning_parser_cls = _CombinedReasoningAdapter
792 + tool_parser_cls = _CombinedToolAdapter
793 +
794 +
795 +def _make_delegating_request():
796 + req = MagicMock(spec=ChatCompletionRequest)
797 + req.tools = []
798 + req.tool_choice = "auto"
799 + return req
800 +
801 +
802 +class TestAdapterFinishOnStreamEnd:
803 + """Engine adapters must flush buffered text when streaming ends.
804 +
805 + When a DelegatingParser wraps engine adapters, the underlying
806 + StreamingParserEngine.finish() must be called on the last
807 + parse_delta(finished=True) so that lexer-buffered text (terminal
808 + prefixes) and scanner-deferred terminals are not silently lost.
809 + """
810 +
811 + def test_lexer_buffer_flushed_on_finished(self):
812 + """Text buffered as a potential terminal prefix must be emitted
813 + as content when the stream ends."""
814 + tokenizer = make_mock_tokenizer(_VOCAB)
815 + parser = _CombinedDelegating(tokenizer)
816 + request = _make_delegating_request()
817 +
818 + # Feed reasoning then content with a trailing '<' that looks like
819 + # the start of a terminal ('<think>' or '<tool_call>').
820 + parser.parse_delta("</think>", [201], request, finished=False)
821 + delta = parser.parse_delta("Hello world<", [], request, finished=True)
822 + # The '<' must NOT be silently dropped.
823 + assert delta is not None
824 + assert delta.content is not None
825 + assert "<" in delta.content, (
826 + "Trailing '<' lost: lexer buffer was not flushed on finish"
827 + )
828 +
829 + def test_args_buffer_flushed_on_finished(self):
830 + """Pending arg buffer text must be emitted when stream ends
831 + mid-tool-call (closing brace held back in buffer)."""
832 + tokenizer = make_mock_tokenizer(_VOCAB)
833 + parser = _CombinedDelegating(tokenizer)
834 + request = _make_delegating_request()
835 +
836 + parser.parse_delta("</think>", [201], request, finished=False)
837 + parser.parse_delta("<tool_call>", [202], request, finished=False)
838 + parser.parse_delta('{"name": "f"}', [], request, finished=False)
839 + # The closing } is held back in args buffer, waiting for
840 + # a TOOL_END terminal. Stream ends without one — finish()
841 + # must flush the buffer.
842 + delta = parser.parse_delta("", [], request, finished=True)
843 + assert delta is not None, (
844 + "Engine finish should produce a delta with flushed args/end"
845 + )
846 +
847 +
848 +# ── TestReasoningOnlyDelegatingParser ─────────────────────────────
849 +
850 +
851 +class _ReasoningOnlyDelegating(DelegatingParser):
852 + """DelegatingParser with reasoning adapter but NO tool adapter."""
853 +
854 + reasoning_parser_cls = _CombinedReasoningAdapter
855 + tool_parser_cls = None
856 +
857 +
858 +class TestReasoningOnlyEndTokenLeak:
859 + """When there is no tool parser, the content passthrough must not
860 + re-emit the end-of-reasoning marker (e.g. ``</think>``) as content.
861 +
862 + Regression test for the scenario where ``</think>`` arrives as a
863 + single-token delta: the engine correctly consumes it (emitting
864 + REASONING_END with no content), but the content passthrough
865 + fired because ``delta_message is None`` and reasoning had just ended.
866 + """
867 +
868 + def test_think_end_not_leaked_as_content(self):
869 + tokenizer = make_mock_tokenizer(_VOCAB)
870 + parser = _ReasoningOnlyDelegating(tokenizer)
871 + request = _make_delegating_request()
872 +
873 + # Feed reasoning text.
874 + d1 = parser.parse_delta(
875 + "I am thinking",
876 + [],
877 + request,
878 + finished=False,
879 + )
880 + assert d1 is not None
881 + assert d1.reasoning is not None
882 + assert d1.content is None
883 +
884 + # Feed </think> as a single-token delta.
885 + d2 = parser.parse_delta(
886 + "</think>",
887 + [201],
888 + request,
889 + finished=False,
890 + )
891 + # The end-of-reasoning marker must NOT appear as content.
892 + if d2 is not None:
893 + assert d2.content is None, f"</think> leaked as content: {d2.content!r}"
894 +
895 + # Feed content after reasoning.
896 + d3 = parser.parse_delta(
897 + "\n\nHello!",
898 + [],
899 + request,
900 + finished=False,
901 + )
902 + assert d3 is not None
903 + assert d3.content is not None
904 + assert "</think>" not in d3.content
905 +
906 + def test_streaming_content_matches_non_streaming(self):
907 + """Concatenated streaming content must match extract_reasoning."""
908 + tokenizer = make_mock_tokenizer(_VOCAB)
909 + # No <think> in input: the combined config starts in REASONING
910 + # state, so all text before </think> is reasoning.
911 + full_text = "reasoning</think>\n\nHello!"
912 +
913 + # Non-streaming extraction.
914 + parser_ns = _ReasoningOnlyDelegating(tokenizer)
915 + request = _make_delegating_request()
916 + reasoning, content = parser_ns.extract_reasoning(full_text, request)
917 + assert reasoning == "reasoning"
918 + assert content == "\n\nHello!"
919 +
920 + # Streaming extraction — simulate per-token deltas.
921 + parser_s = _ReasoningOnlyDelegating(tokenizer)
922 + deltas = [
923 + ("reasoning", []),
924 + ("</think>", [201]),
925 + ("\n\n", []),
926 + ("Hello!", []),
927 + ]
928 + content_parts: list[str] = []
929 + for text, ids in deltas:
930 + dm = parser_s.parse_delta(text, ids, request, finished=False)
931 + if dm is not None and dm.content:
932 + content_parts.append(dm.content)
933 + dm = parser_s.parse_delta("", [], request, finished=True)
934 + if dm is not None and dm.content:
935 + content_parts.append(dm.content)
936 +
937 + streaming_content = "".join(content_parts)
938 + assert streaming_content == content, (
939 + f"Streaming content {streaming_content!r} "
940 + f"does not match non-streaming {content!r}"
941 + )
942 +
943 + def test_multi_token_delta_preserves_content_after_think_end(self):
944 + """Content after </think> in the same delta must not be lost."""
945 + tokenizer = make_mock_tokenizer(_VOCAB)
946 + parser = _ReasoningOnlyDelegating(tokenizer)
947 + request = _make_delegating_request()
948 +
949 + # Feed reasoning text.
950 + d1 = parser.parse_delta(
951 + "thinking",
952 + [],
953 + request,
954 + finished=False,
955 + )
956 + assert d1 is not None
957 + assert d1.reasoning is not None
958 +
959 + # Feed </think> and content in the same delta (e.g. speculative
960 + # decoding accepting multiple tokens at once). Token IDs must
961 + # cover all text so the scanner can split correctly.
962 + # chr(10)='\n', chr(72)='H', chr(105)='i', chr(33)='!'
963 + d2 = parser.parse_delta(
964 + "</think>\n\nHi!",
965 + [201, 10, 10, 72, 105, 33],
966 + request,
967 + finished=False,
968 + )
969 + assert d2 is not None, "Content after </think> in multi-token delta was lost"
970 + assert d2.content is not None, (
971 + "Content after </think> in multi-token delta was nullified"
972 + )
973 + assert "</think>" not in d2.content
974 + assert "Hi!" in d2.content
975 +
976 +
977 +# ── TestToolAdapterForwardsKwargs ──────────────────────────────────
978 +
979 +
980 +class TestToolAdapterForwardsKwargs:
981 + """ParserEngineToolAdapter.__init__ must forward **kwargs to the
982 + parser engine class so chat_template_kwargs reach model parsers."""
983 +
984 + @pytest.mark.parametrize(
985 + "enable_thinking,expected_state",
986 + [
987 + (False, ParserState.CONTENT),
988 + (True, ParserState.REASONING),
989 + ],
990 + )
991 + def test_kwargs_forwarded_to_parser_engine(self, enable_thinking, expected_state):
992 + from vllm.parser.qwen3 import Qwen3Parser
993 +
994 + vocab = {"<think>": 100, "</think>": 101}
995 + tokenizer = make_mock_tokenizer(vocab)
996 +
997 + _, ToolAdapter = make_adapters(Qwen3Parser)
998 + adapter = ToolAdapter(
999 + tokenizer,
1000 + tools=None,
1001 + chat_template_kwargs={"enable_thinking": enable_thinking},
1002 + )
1003 + engine = adapter._parser_engine
1004 + assert engine.parser_engine_config.initial_state == expected_state
1005 +
1006 +
1007 +# ── TestExtractContentIdsNoEmptyReturn ─────────────────────────────
1008 +
1009 +
1010 +class TestExtractContentIdsNoEmptyReturn:
1011 + """extract_content_ids must return input_ids (not []) when there is
1012 + no THINK_END token ID and _reasoning_ended is True."""
1013 +
1014 + _NO_THINK_CONFIG = ParserEngineConfig(name="no_think_end", token_id_terminals={})
1015 +
1016 + @pytest.mark.parametrize("input_ids", [[1, 2, 3], []])
1017 + def test_returns_input_ids_without_think_end(self, input_ids):
1018 + engine = _make_engine(self._NO_THINK_CONFIG)
1019 + assert engine._reasoning_end_token_id is None
1020 + engine._reasoning_ended = True
1021 + assert engine.extract_content_ids(input_ids) == input_ids
1022 +
1023 +
1024 +# ── TestValuePostprocessorRemoved ──────────────────────────────────
1025 +
1026 +
1027 +class TestValuePostprocessorRemoved:
1028 + """ParserEngineConfig no longer has a value_postprocessor field."""
1029 +
1030 + def test_no_value_postprocessor_field(self):
1031 + config = ParserEngineConfig(name="test")
1032 + assert not hasattr(config, "value_postprocessor")
1033 +
1034 + def test_constructor_rejects_value_postprocessor(self):
1035 + with pytest.raises(TypeError):
1036 + ParserEngineConfig(
1037 + name="test",
1038 + value_postprocessor=lambda x: x, # type: ignore[call-arg]
1039 + )
1040 +
1041 +
1042 +# ── TestArgDeltaWithConverter ─────────────────────────────────────
1043 +
1044 +
1045 +_KV_RE = re.compile(r"(\w+)=(\S+)")
1046 +
1047 +
1048 +def _kv_converter(raw_args: str, partial: bool) -> str:
1049 + params: dict[str, str] = {}
1050 + for m in _KV_RE.finditer(raw_args):
1051 + params[m.group(1)] = m.group(2)
1052 + return json.dumps(params, ensure_ascii=False)
1053 +
1054 +
1055 +def _converter_config(
1056 + converter=_kv_converter,
1057 + name: str = "converter_test",
1058 +) -> ParserEngineConfig:
1059 + return ParserEngineConfig(
1060 + name=name,
1061 + terminals={
1062 + "TOOL_START": "<tool_call>",
1063 + "TOOL_END": "</tool_call>",
1064 + },
1065 + token_id_terminals={
1066 + "TOOL_START": "<tool_call>",
1067 + "TOOL_END": "</tool_call>",
1068 + },
1069 + transitions={
1070 + (ParserState.CONTENT, "TOOL_START"): Transition(
1071 + ParserState.TOOL_ARGS,
1072 + (EventType.TOOL_CALL_START,),
1073 + ),
1074 + (ParserState.TOOL_ARGS, "TOOL_END"): Transition(
1075 + ParserState.CONTENT,
1076 + (EventType.TOOL_CALL_END,),
1077 + ),
1078 + },
1079 + content_events={
1080 + ParserState.CONTENT: EventType.TEXT_CHUNK,
1081 + ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
1082 + },
1083 + arg_converter=converter,
1084 + stream_arg_deltas=True,
1085 + )
1086 +
1087 +
1088 +def _collect_arg_deltas(deltas: list) -> str:
1089 + parts: list[str] = []
1090 + for d in deltas:
1091 + if d is None:
1092 + continue
1093 + for tc in d.tool_calls or []:
1094 + if tc.function and tc.function.arguments:
1095 + parts.append(tc.function.arguments)
1096 + return "".join(parts)
1097 +
1098 +
1099 +def _run_streaming_tool(engine, name: str, chunks: list[str]) -> dict:
1100 + deltas = []
1101 + deltas.append(
1102 + engine._events_to_delta(
1103 + [SemanticEvent(EventType.TOOL_CALL_START, tool_index=0)]
1104 + )
1105 + )
1106 + deltas.append(
1107 + engine._events_to_delta(
1108 + [SemanticEvent(EventType.TOOL_NAME, name, tool_index=0)]
1109 + )
1110 + )
1111 + for chunk in chunks:
1112 + deltas.append(
1113 + engine._events_to_delta(
1114 + [SemanticEvent(EventType.ARG_VALUE_CHUNK, chunk, tool_index=0)]
1115 + )
1116 + )
1117 + deltas.append(
1118 + engine._events_to_delta([SemanticEvent(EventType.TOOL_CALL_END, tool_index=0)])
1119 + )
1120 + return json.loads(_collect_arg_deltas(deltas))
1121 +
1122 +
1123 +class TestArgDeltaWithConverter:
1124 + """Exercise _compute_arg_delta with arg_converter + stream_arg_deltas.
1125 +
1126 + The startswith guard on line 814 of parser_engine.py validates that
1127 + converted JSON grows prefix-monotonically across streaming ticks.
1128 + These tests exercise that path with a synthetic config.
1129 + """
1130 +
1131 + def test_streaming_arg_deltas_prefix_monotonic(self):
1132 + engine = _make_engine(_converter_config())
1133 + deltas = []
1134 +
1135 + deltas.append(
1136 + engine._events_to_delta(
1137 + [SemanticEvent(EventType.TOOL_CALL_START, tool_index=0)],
1138 + )
1139 + )
1140 + deltas.append(
1141 + engine._events_to_delta(
1142 + [SemanticEvent(EventType.TOOL_NAME, "f", tool_index=0)],
1143 + )
1144 + )
1145 + deltas.append(
1146 + engine._events_to_delta(
1147 + [
1148 + SemanticEvent(
1149 + EventType.ARG_VALUE_CHUNK,
1150 + "a=hello ",
1151 + tool_index=0,
1152 + )
1153 + ],
1154 + )
1155 + )
1156 + deltas.append(
1157 + engine._events_to_delta(
1158 + [
1159 + SemanticEvent(
1160 + EventType.ARG_VALUE_CHUNK,
1161 + "b=world ",
1162 + tool_index=0,
1163 + )
1164 + ],
1165 + )
1166 + )
1167 + deltas.append(
1168 + engine._events_to_delta(
1169 + [
1170 + SemanticEvent(
1171 + EventType.ARG_VALUE_CHUNK,
1172 + "c=ok",
1173 + tool_index=0,
1174 + )
1175 + ],
1176 + )
1177 + )
1178 + deltas.append(
1179 + engine._events_to_delta(
1180 + [SemanticEvent(EventType.TOOL_CALL_END, tool_index=0)],
1181 + )
1182 + )
1183 +
1184 + all_args = _collect_arg_deltas(deltas)
1185 + assert json.loads(all_args) == {
1186 + "a": "hello",
1187 + "b": "world",
1188 + "c": "ok",
1189 + }
1190 +
1191 + def test_streaming_arg_deltas_with_type_coercion(self):
1192 + tool = _make_tool(
1193 + "f",
1194 + {
1195 + "count": {"type": "integer"},
1196 + "name": {"type": "string"},
1197 + },
1198 + )
1199 + engine = _make_engine(_converter_config(), tools=[tool])
1200 + deltas = []
1201 +
1202 + deltas.append(
1203 + engine._events_to_delta(
1204 + [SemanticEvent(EventType.TOOL_CALL_START, tool_index=0)],
1205 + )
1206 + )
1207 + deltas.append(
1208 + engine._events_to_delta(
1209 + [SemanticEvent(EventType.TOOL_NAME, "f", tool_index=0)],
1210 + )
1211 + )
1212 + deltas.append(
1213 + engine._events_to_delta(
1214 + [
1215 + SemanticEvent(
1216 + EventType.ARG_VALUE_CHUNK,
1217 + "count=5 ",
1218 + tool_index=0,
1219 + )
1220 + ],
1221 + )
1222 + )
1223 + deltas.append(
1224 + engine._events_to_delta(
1225 + [
1226 + SemanticEvent(
1227 + EventType.ARG_VALUE_CHUNK,
1228 + "name=test",
1229 + tool_index=0,
1230 + )
1231 + ],
1232 + )
1233 + )
1234 + deltas.append(
1235 + engine._events_to_delta(
1236 + [SemanticEvent(EventType.TOOL_CALL_END, tool_index=0)],
1237 + )
1238 + )
1239 +
1240 + all_args = _collect_arg_deltas(deltas)
1241 + parsed = json.loads(all_args)
1242 + assert parsed == {"count": 5, "name": "test"}
1243 + assert isinstance(parsed["count"], int)
1244 +
1245 +
1246 +# ── TestSafeArgPrefix ────────────────────────────────────────────
1247 +
1248 +
1249 +class TestSafeArgPrefix:
1250 + """Unit tests for ParserEngine._safe_arg_prefix."""
1251 +
1252 + @pytest.mark.parametrize(
1253 + "json_str, expected",
1254 + [
1255 + ('{"a": 1}', '{"a": '),
1256 + ('{"a": 1, "b": 2}', '{"a": 1, "b": '),
1257 + ('{"a": "hello", "b": "world"}', '{"a": "hello", "b": '),
1258 + ('{"obj": {"x": 1}, "b": 2}', '{"obj": {"x": 1}, "b": '),
1259 + ('{"url": "http://x:80", "b": 1}', '{"url": "http://x:80", "b": '),
1260 + ('{"a": 1', '{"a": '),
1261 + ("{}", ""),
1262 + ("{", ""),
1263 + ("", ""),
1264 + ('{"k":1}', '{"k":'),
1265 + ('{"k": 1, "v":2}', '{"k": 1, "v":'),
1266 + ],
1267 + )
1268 + def test_safe_arg_prefix(self, json_str, expected):
1269 + assert ParserEngine._safe_arg_prefix(json_str) == expected
1270 +
1271 +
1272 +# ── Coercion instability regression tests ────────────────────────
1273 +
1274 +
1275 +def _growing_kv_converter(raw_args: str, partial: bool) -> str:
1276 + """Converter that produces growing bare values (no delimiter)."""
1277 + params: dict[str, str] = {}
1278 + for part in raw_args.split(" "):
1279 + if "=" in part:
1280 + k, v = part.split("=", 1)
1281 + params[k] = v
1282 + return json.dumps(params, ensure_ascii=False)
1283 +
1284 +
1285 +class TestCoercionInstabilityRegression:
1286 + """Regression tests for _fix_arg_types coercion instability.
1287 +
1288 + These tests exercise scenarios where a trailing value's coercion
1289 + status changes between ticks (e.g. "4" coerces to int but "4e"
1290 + does not). Before the _safe_arg_prefix fix, these would violate
1291 + the startswith prefix invariant and permanently drop deltas.
1292 + """
1293 +
1294 + def test_coercion_flip_does_not_corrupt_stream(self):
1295 + tool = _make_tool(
1296 + "f",
1297 + {
1298 + "count": {"type": "integer"},
1299 + "flag": {"type": "string"},
1300 + },
1301 + )
1302 + engine = _make_engine(_converter_config(), tools=[tool])
1303 + parsed = _run_streaming_tool(
1304 + engine,
1305 + "f",
1306 + ["count=42 ", "flag=ok"],
1307 + )
1308 + assert parsed == {"count": 42, "flag": "ok"}
1309 + assert isinstance(parsed["count"], int)
1310 +
1311 + def test_bool_partial_value_coercion_is_safe(self):
1312 + """Boolean value building char by char must not break prefix."""
1313 + tool = _make_tool(
1314 + "f",
1315 + {
1316 + "name": {"type": "string"},
1317 + "flag": {"type": "boolean"},
1318 + },
1319 + )
1320 + cfg = _converter_config(_growing_kv_converter)
1321 + engine = _make_engine(cfg, tools=[tool])
1322 + parsed = _run_streaming_tool(
1323 + engine,
1324 + "f",
1325 + ["name=hello ", "flag=t", "r", "u", "e"],
1326 + )
1327 + assert parsed == {"name": "hello", "flag": True}
1328 + assert isinstance(parsed["flag"], bool)
1329 +
1330 + def test_int_partial_value_flip_is_safe(self):
1331 + """Integer that becomes non-coercible must not break prefix.
1332 +
1333 + A dummy first arg is needed so the name emission consumes the
1334 + first ARG_VALUE_CHUNK, ensuring _compute_arg_delta runs for the
1335 + chunk where val="4" coerces to int 4. On the next chunk val
1336 + grows to "4e" which is NOT a valid int, flipping the coercion.
1337 + """
1338 + tool = _make_tool(
1339 + "f",
1340 + {
1341 + "dummy": {"type": "string"},
1342 + "val": {"type": "integer"},
1343 + "extra": {"type": "string"},
1344 + },
1345 + )
1346 + cfg = _converter_config(_growing_kv_converter)
1347 + engine = _make_engine(cfg, tools=[tool])
1348 + parsed = _run_streaming_tool(
1349 + engine,
1350 + "f",
1351 + ["dummy=x ", "val=4", "e ", "extra=ok"],
1352 + )
1353 + assert parsed["dummy"] == "x"
1354 + assert parsed["val"] == "4e"
1355 + assert isinstance(parsed["val"], str)
1356 + assert parsed["extra"] == "ok"
tests/parser/engine/test_qwen3.py new
+1095
@@ -0,0 +1,1095 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Tests for the engine-based Qwen3 tool call parser.
4 +
5 +These validate that the engine-driven parser correctly handles
6 +Qwen3 XML-style tool calls.
7 +"""
8 +
9 +import json
10 +from unittest.mock import MagicMock
11 +
12 +import pytest
13 +
14 +from tests.parser.engine.conftest import make_mock_tokenizer
15 +from tests.parser.engine.streaming_helpers import (
16 + collect_content,
17 + collect_function_name,
18 + collect_tool_arguments,
19 + simulate_tool_streaming,
20 +)
21 +from vllm.parser.engine.parser_engine import ParserEngine
22 +from vllm.parser.qwen3 import (
23 + TOOL_CALL_END,
24 + TOOL_CALL_START,
25 + qwen3_config,
26 +)
27 +
28 +
29 +@pytest.fixture
30 +def mock_tokenizer():
31 + return make_mock_tokenizer(
32 + {
33 + TOOL_CALL_START: 100,
34 + TOOL_CALL_END: 101,
35 + }
36 + )
37 +
38 +
39 +@pytest.fixture
40 +def parser(mock_tokenizer):
41 + return ParserEngine(
42 + mock_tokenizer,
43 + parser_engine_config=qwen3_config(thinking=False),
44 + )
45 +
46 +
47 +class TestNonStreaming:
48 + def test_no_tool_calls(self, parser, mock_request):
49 + result = parser.extract_tool_calls(
50 + "This is a regular response without any tool calls.",
51 + mock_request,
52 + )
53 + assert result.tools_called is False
54 + assert result.tool_calls == []
55 + assert result.content == ("This is a regular response without any tool calls.")
56 +
57 + def test_single_tool_call(self, parser, mock_request):
58 + text = (
59 + "<tool_call>\n"
60 + "<function=get_weather>\n"
61 + "<parameter=city>Tokyo</parameter>\n"
62 + "</function>\n"
63 + "</tool_call>"
64 + )
65 + result = parser.extract_tool_calls(text, mock_request)
66 +
67 + assert result.tools_called is True
68 + assert len(result.tool_calls) == 1
69 + assert result.tool_calls[0].function.name == "get_weather"
70 + args = json.loads(result.tool_calls[0].function.arguments)
71 + assert args == {"city": "Tokyo"}
72 +
73 + def test_parallel_tool_calls(self, parser, mock_request):
74 + text = (
75 + "<tool_call>\n"
76 + "<function=get_weather>\n"
77 + "<parameter=city>Tokyo</parameter>\n"
78 + "</function>\n"
79 + "</tool_call>"
80 + "<tool_call>\n"
81 + "<function=get_time>\n"
82 + "<parameter=timezone>Asia/Tokyo</parameter>\n"
83 + "</function>\n"
84 + "</tool_call>"
85 + )
86 + result = parser.extract_tool_calls(text, mock_request)
87 +
88 + assert result.tools_called is True
89 + assert len(result.tool_calls) == 2
90 + assert result.tool_calls[0].function.name == "get_weather"
91 + assert result.tool_calls[1].function.name == "get_time"
92 +
93 + args0 = json.loads(result.tool_calls[0].function.arguments)
94 + assert args0 == {"city": "Tokyo"}
95 + args1 = json.loads(result.tool_calls[1].function.arguments)
96 + assert args1 == {"timezone": "Asia/Tokyo"}
97 +
98 + def test_various_data_types(self, parser, mock_request):
99 + text = (
100 + "<tool_call>\n<function=test_function>\n"
101 + "<parameter=string_field>hello</parameter>\n"
102 + "<parameter=int_field>42</parameter>\n"
103 + "<parameter=float_field>3.14</parameter>\n"
104 + "<parameter=bool_field>true</parameter>\n"
105 + "<parameter=null_field>null</parameter>\n"
106 + '<parameter=array_field>["a", "b", "c"]</parameter>\n'
107 + '<parameter=object_field>{"nested": "value"}</parameter>\n'
108 + "</function>\n</tool_call>"
109 + )
110 + result = parser.extract_tool_calls(text, mock_request)
111 +
112 + assert result.tools_called is True
113 + args = json.loads(result.tool_calls[0].function.arguments)
114 + assert args["string_field"] == "hello"
115 + assert args["int_field"] == "42"
116 + assert args["float_field"] == "3.14"
117 + assert args["bool_field"] == "true"
118 + assert args["null_field"] == "null"
119 + assert args["array_field"] == '["a", "b", "c"]'
120 + assert args["object_field"] == '{"nested": "value"}'
121 +
122 + def test_empty_arguments(self, parser, mock_request):
123 + text = "<tool_call>\n<function=refresh>\n</function>\n</tool_call>"
124 + result = parser.extract_tool_calls(text, mock_request)
125 +
126 + assert result.tools_called is True
127 + assert result.tool_calls[0].function.name == "refresh"
128 + args = json.loads(result.tool_calls[0].function.arguments)
129 + assert args == {}
130 +
131 + def test_surrounding_text(self, parser, mock_request):
132 + text = (
133 + "Let me check the weather for you.\n\n"
134 + "<tool_call>\n<function=get_weather>\n"
135 + "<parameter=city>Tokyo</parameter>\n"
136 + "</function>\n</tool_call>\n\n"
137 + "I will get that information."
138 + )
139 + result = parser.extract_tool_calls(text, mock_request)
140 +
141 + assert result.tools_called is True
142 + assert result.content is not None
143 + assert "Let me check the weather" in result.content
144 + assert result.tool_calls[0].function.name == "get_weather"
145 +
146 + def test_escaped_strings(self, parser, mock_request):
147 + text = (
148 + "<tool_call>\n<function=test_function>\n"
149 + '<parameter=quoted>He said "hello"</parameter>\n'
150 + "<parameter=path>C:\\Users\\file.txt</parameter>\n"
151 + "<parameter=newline>line1\nline2</parameter>\n"
152 + "</function>\n</tool_call>"
153 + )
154 + result = parser.extract_tool_calls(text, mock_request)
155 +
156 + assert result.tools_called is True
157 + args = json.loads(result.tool_calls[0].function.arguments)
158 + assert args["quoted"] == 'He said "hello"'
159 + assert args["path"] == "C:\\Users\\file.txt"
160 + assert args["newline"] == "line1\nline2"
161 +
162 + def test_multiple_parameters(self, parser, mock_request):
163 + text = (
164 + "<tool_call>\n<function=search>\n"
165 + "<parameter=query>vllm parsing</parameter>\n"
166 + "<parameter=limit>10</parameter>\n"
167 + "<parameter=exact_match>false</parameter>\n"
168 + "</function>\n</tool_call>"
169 + )
170 + result = parser.extract_tool_calls(text, mock_request)
171 +
172 + assert result.tools_called is True
173 + args = json.loads(result.tool_calls[0].function.arguments)
174 + assert args == {
175 + "query": "vllm parsing",
176 + "limit": "10",
177 + "exact_match": "false",
178 + }
179 +
180 + def test_multiline_param_values(self, parser, mock_request):
181 + """Parameter values spanning multiple lines."""
182 + text = (
183 + "<tool_call>\n"
184 + "<function=Bash>\n"
185 + "<parameter=command>\n"
186 + "ls -la /tmp\n"
187 + "</parameter>\n"
188 + "<parameter=description>\n"
189 + "List files in /tmp directory\n"
190 + "</parameter>\n"
191 + "</function>\n"
192 + "</tool_call>"
193 + )
194 + result = parser.extract_tool_calls(text, mock_request)
195 +
196 + assert result.tools_called is True
197 + assert len(result.tool_calls) == 1
198 + assert result.tool_calls[0].function.name == "Bash"
199 + args = json.loads(result.tool_calls[0].function.arguments)
200 + assert args["command"] == "ls -la /tmp"
201 + assert args["description"] == "List files in /tmp directory"
202 +
203 + def test_multiline_two_tool_calls(self, parser, mock_request):
204 + """Two tool calls with multi-line parameter values (bug report)."""
205 + text = (
206 + "<tool_call>\n"
207 + "<function=Bash>\n"
208 + "<parameter=command>\n"
209 + "find /workspace -name '*.py' | head -20\n"
210 + "</parameter>\n"
211 + "<parameter=description>\n"
212 + "Find Python files\n"
213 + "</parameter>\n"
214 + "</function>\n"
215 + "</tool_call>"
216 + "<tool_call>\n"
217 + "<function=Read>\n"
218 + "<parameter=file_path>\n"
219 + "/workspace/main.py\n"
220 + "</parameter>\n"
221 + "</function>\n"
222 + "</tool_call>"
223 + )
224 + result = parser.extract_tool_calls(text, mock_request)
225 +
226 + assert result.tools_called is True
227 + assert len(result.tool_calls) == 2
228 + assert result.tool_calls[0].function.name == "Bash"
229 + assert result.tool_calls[1].function.name == "Read"
230 + args0 = json.loads(result.tool_calls[0].function.arguments)
231 + assert "find /workspace" in args0["command"]
232 + assert "Find Python files" in args0["description"]
233 + args1 = json.loads(result.tool_calls[1].function.arguments)
234 + assert "/workspace/main.py" in args1["file_path"]
235 +
236 + def test_consecutive_tool_calls_without_tool_end(self, parser, mock_request):
237 + text = (
238 + "<tool_call>\n"
239 + "<function=get_weather>\n"
240 + "<parameter=city>Tokyo</parameter>\n"
241 + "</function>\n"
242 + "<tool_call>\n"
243 + "<function=get_weather>\n"
244 + "<parameter=city>Paris</parameter>\n"
245 + "</function>\n"
246 + "</tool_call>"
247 + )
248 + result = parser.extract_tool_calls(text, mock_request)
249 +
250 + assert result.tools_called is True
251 + assert len(result.tool_calls) == 2
252 + args0 = json.loads(result.tool_calls[0].function.arguments)
253 + assert args0 == {"city": "Tokyo"}
254 + args1 = json.loads(result.tool_calls[1].function.arguments)
255 + assert args1 == {"city": "Paris"}
256 +
257 + def test_nested_json_array_parameter(self, parser, mock_request):
258 + text = (
259 + "<tool_call>\n"
260 + "<function=AskUserQuestion>\n"
261 + "<parameter=questions>"
262 + '[{"question": "Pick a color",'
263 + ' "multiSelect": false, "answer": null}]'
264 + "</parameter>\n"
265 + "</function>\n"
266 + "</tool_call>"
267 + )
268 + result = parser.extract_tool_calls(text, mock_request)
269 +
270 + assert result.tools_called is True
271 + args = json.loads(result.tool_calls[0].function.arguments)
272 + assert args == {
273 + "questions": '[{"question": "Pick a color",'
274 + ' "multiSelect": false, "answer": null}]',
275 + }
276 +
277 +
278 +class TestStreaming:
279 + def test_basic_streaming(self, parser, mock_request):
280 + chunks = [
281 + "<tool_call>\n",
282 + "<function=get_weather>\n",
283 + "<parameter=city>Tokyo",
284 + "</parameter>\n",
285 + "</function>\n",
286 + "</tool_call>",
287 + ]
288 +
289 + results = simulate_tool_streaming(parser, mock_request, chunks)
290 +
291 + name = collect_function_name(results)
292 + assert name == "get_weather"
293 +
294 + args_text = collect_tool_arguments(results)
295 + assert args_text
296 + parsed = json.loads(args_text)
297 + assert parsed == {"city": "Tokyo"}
298 +
299 + def test_streaming_multi_param(self, parser, mock_request):
300 + chunks = [
301 + "<tool_call>\n",
302 + "<function=get_weather>\n",
303 + "<parameter=city>Tokyo</parameter>\n",
304 + "<parameter=unit>celsius</parameter>\n",
305 + "</function>\n",
306 + "</tool_call>",
307 + ]
308 +
309 + results = simulate_tool_streaming(parser, mock_request, chunks)
310 +
311 + name = collect_function_name(results)
312 + assert name == "get_weather"
313 +
314 + args_text = collect_tool_arguments(results)
315 + assert args_text
316 + parsed = json.loads(args_text)
317 + assert parsed == {"city": "Tokyo", "unit": "celsius"}
318 +
319 + def test_streaming_args_arrive_incrementally(self, parser, mock_request):
320 + """Arguments must stream as intermediate deltas, not batch at
321 + tool-end."""
322 + chunks = [
323 + "<tool_call>\n",
324 + "<function=get_weather>\n",
325 + "<parameter=city>Tokyo</parameter>\n",
326 + "<parameter=unit>celsius</parameter>\n",
327 + "<parameter=days>5</parameter>\n",
328 + "</function>\n",
329 + "</tool_call>",
330 + ]
331 +
332 + results = simulate_tool_streaming(parser, mock_request, chunks)
333 +
334 + arg_deltas: list[str] = []
335 + for delta, _ in results:
336 + if delta and delta.tool_calls:
337 + for tc in delta.tool_calls:
338 + if tc.function and tc.function.arguments:
339 + arg_deltas.append(tc.function.arguments)
340 +
341 + assert len(arg_deltas) > 1, (
342 + f"Expected arguments across multiple deltas, got {len(arg_deltas)}: "
343 + f"{arg_deltas}"
344 + )
345 + concatenated = "".join(arg_deltas)
346 + parsed = json.loads(concatenated)
347 + assert parsed == {"city": "Tokyo", "unit": "celsius", "days": "5"}
348 +
349 + def test_streaming_text_before_tool(self, parser, mock_request):
350 + chunks = [
351 + "Let me check ",
352 + "the weather. ",
353 + "<tool_call>\n",
354 + "<function=get_weather>\n",
355 + "<parameter=city>Tokyo</parameter>\n",
356 + "</function>\n",
357 + "</tool_call>",
358 + ]
359 +
360 + results = simulate_tool_streaming(parser, mock_request, chunks)
361 + assert collect_content(results).strip().startswith("Let me check")
362 +
363 + def test_streaming_empty_args(self, parser, mock_request):
364 + chunks = [
365 + "<tool_call>\n",
366 + "<function=refresh>\n",
367 + "</function>\n",
368 + "</tool_call>",
369 + ]
370 +
371 + results = simulate_tool_streaming(parser, mock_request, chunks)
372 +
373 + name = collect_function_name(results)
374 + assert name == "refresh"
375 +
376 + def test_streaming_split_parameter_tag(self, parser, mock_request):
377 + """Parameter tag split across chunks."""
378 + chunks = [
379 + "<tool_call>\n",
380 + "<function=test>\n",
381 + "<parameter=",
382 + "name>Alice",
383 + "</parameter>\n",
384 + "</function>\n",
385 + "</tool_call>",
386 + ]
387 +
388 + results = simulate_tool_streaming(parser, mock_request, chunks)
389 +
390 + name = collect_function_name(results)
391 + assert name == "test"
392 +
393 + args_text = collect_tool_arguments(results)
394 + assert args_text
395 + parsed = json.loads(args_text)
396 + assert parsed["name"] == "Alice"
397 +
398 + def test_streaming_numeric_values(self, parser, mock_request):
399 + chunks = [
400 + "<tool_call>\n",
401 + "<function=set_config>\n",
402 + "<parameter=count>42</parameter>\n",
403 + "<parameter=active>true</parameter>\n",
404 + "</function>\n",
405 + "</tool_call>",
406 + ]
407 +
408 + results = simulate_tool_streaming(parser, mock_request, chunks)
409 + args_text = collect_tool_arguments(results)
410 + if args_text:
411 + parsed = json.loads(args_text)
412 + assert parsed["count"] == "42"
413 + assert parsed["active"] == "true"
414 +
415 + def test_streaming_parallel_calls(self, parser, mock_request):
416 + chunks = [
417 + "<tool_call>\n",
418 + "<function=get_weather>\n",
419 + "<parameter=city>Tokyo</parameter>\n",
420 + "</function>\n",
421 + "</tool_call>",
422 + "<tool_call>\n",
423 + "<function=get_time>\n",
424 + "<parameter=tz>JST</parameter>\n",
425 + "</function>\n",
426 + "</tool_call>",
427 + ]
428 +
429 + results = simulate_tool_streaming(parser, mock_request, chunks)
430 +
431 + names = []
432 + for delta, _ in results:
433 + if delta and delta.tool_calls:
434 + for tc in delta.tool_calls:
435 + if tc.function and tc.function.name:
436 + names.append(tc.function.name)
437 +
438 + assert "get_weather" in names
439 + assert "get_time" in names
440 +
441 + def test_streaming_value_split_across_chunks(self, parser, mock_request):
442 + """Parameter value split across multiple chunks."""
443 + chunks = [
444 + "<tool_call>\n",
445 + "<function=search>\n",
446 + "<parameter=query>hello ",
447 + "world",
448 + " test</parameter>\n",
449 + "</function>\n",
450 + "</tool_call>",
451 + ]
452 +
453 + results = simulate_tool_streaming(parser, mock_request, chunks)
454 +
455 + args_text = collect_tool_arguments(results)
456 + assert args_text
457 + parsed = json.loads(args_text)
458 + assert parsed["query"] == "hello world test"
459 +
460 + def test_streaming_split_tool_call_tag(self, parser, mock_request):
461 + """<tool_call> arrives as a single special token; the rest of
462 + the content is split into fine-grained chunks."""
463 + chunks = [
464 + "<tool_call>\n",
465 + "<function=test>\n",
466 + "<parameter=x>1",
467 + "</parameter>\n",
468 + "</function>\n",
469 + "</tool_call>",
470 + ]
471 + results = simulate_tool_streaming(parser, mock_request, chunks)
472 +
473 + name = collect_function_name(results)
474 + assert name == "test"
475 +
476 + args_text = collect_tool_arguments(results)
477 + assert args_text
478 + parsed = json.loads(args_text)
479 + assert parsed["x"] == "1"
480 +
481 + def test_char_by_char_streaming(self, mock_request):
482 + """Feed text character-by-character to test lexer robustness.
483 +
484 + Uses a tokenizer without special token IDs because char-by-char
485 + delivery only occurs when the tokenizer splits the tag across
486 + multiple sub-word tokens (i.e., no dedicated special token).
487 + """
488 + tokenizer = MagicMock()
489 + tokenizer.encode.return_value = [1, 2, 3]
490 + tokenizer.get_vocab.return_value = {}
491 + tokenizer.decode.side_effect = lambda ids: "".join(
492 + chr(i) if i < 128 else f"<{i}>" for i in ids
493 + )
494 + no_tid_parser = ParserEngine(
495 + tokenizer, parser_engine_config=qwen3_config(thinking=False)
496 + )
497 +
498 + full_text = (
499 + "<tool_call>\n"
500 + "<function=echo>\n"
501 + "<parameter=msg>hi</parameter>\n"
502 + "</function>\n"
503 + "</tool_call>"
504 + )
505 + chunks = list(full_text)
506 + results = simulate_tool_streaming(no_tid_parser, mock_request, chunks)
507 +
508 + name = collect_function_name(results)
509 + assert name == "echo"
510 +
511 + args_text = collect_tool_arguments(results)
512 + assert args_text
513 + parsed = json.loads(args_text)
514 + assert parsed == {"msg": "hi"}
515 +
516 + def test_streaming_multiline_param_values(self, parser, mock_request):
517 + """Multi-line parameter values in streaming mode."""
518 + chunks = [
519 + "<tool_call>\n",
520 + "<function=Bash>\n",
521 + "<parameter=command>\n",
522 + "ls -la /tmp\n",
523 + "</parameter>\n",
524 + "<parameter=description>\n",
525 + "List files\n",
526 + "</parameter>\n",
527 + "</function>\n",
528 + "</tool_call>",
529 + ]
530 + results = simulate_tool_streaming(parser, mock_request, chunks)
531 +
532 + name = collect_function_name(results)
533 + assert name == "Bash"
534 +
535 + args_text = collect_tool_arguments(results)
536 + assert args_text
537 + parsed = json.loads(args_text)
538 + assert "ls -la /tmp" in parsed["command"]
539 + assert "List files" in parsed["description"]
540 +
541 + def test_streaming_multiline_two_tool_calls(self, parser, mock_request):
542 + """Two tool calls with multi-line values — matches bug report."""
543 + chunks = [
544 + "<tool_call>\n",
545 + "<function=Bash>\n",
546 + "<parameter=command>\n",
547 + "find /workspace -name '*.py' | head -20\n",
548 + "</parameter>\n",
549 + "<parameter=description>\n",
550 + "Find Python files\n",
551 + "</parameter>\n",
552 + "</function>\n",
553 + "</tool_call>",
554 + "<tool_call>\n",
555 + "<function=Read>\n",
556 + "<parameter=file_path>\n",
557 + "/workspace/main.py\n",
558 + "</parameter>\n",
559 + "</function>\n",
560 + "</tool_call>",
561 + ]
562 + results = simulate_tool_streaming(parser, mock_request, chunks)
563 +
564 + names = []
565 + for delta, _ in results:
566 + if delta and delta.tool_calls:
567 + for tc in delta.tool_calls:
568 + if tc.function and tc.function.name:
569 + names.append(tc.function.name)
570 +
571 + assert "Bash" in names
572 + assert "Read" in names
573 +
574 +
575 +class TestArgConverter:
576 + """Direct tests for the Qwen3 arg_converter with multi-line values."""
577 +
578 + def test_multiline_param_values(self):
579 + from vllm.parser.qwen3 import (
580 + _qwen3_arg_converter,
581 + )
582 +
583 + raw = (
584 + "<parameter=command>\n"
585 + "ls -la /tmp\n"
586 + "</parameter>\n"
587 + "<parameter=description>\n"
588 + "List files\n"
589 + "</parameter>\n"
590 + )
591 + result = json.loads(_qwen3_arg_converter(raw, partial=False))
592 + assert result["command"] == "ls -la /tmp"
593 + assert result["description"] == "List files"
594 +
595 + def test_two_multiline_params(self):
596 + from vllm.parser.qwen3 import (
597 + _qwen3_arg_converter,
598 + )
599 +
600 + raw = (
601 + "<parameter=a>\nfoo\nbar\n</parameter>\n"
602 + "<parameter=b>\nbaz\nqux\n</parameter>\n"
603 + )
604 + result = json.loads(_qwen3_arg_converter(raw, partial=False))
605 + assert result["a"] == "foo\nbar"
606 + assert result["b"] == "baz\nqux"
607 +
608 + def test_partial_multiline(self):
609 + from vllm.parser.qwen3 import (
610 + _qwen3_arg_converter,
611 + )
612 +
613 + raw = "<parameter=command>\nls -la</parameter>\n<parameter=desc>\npartial value"
614 + result = json.loads(_qwen3_arg_converter(raw, partial=True))
615 + assert result["command"] == "ls -la"
616 + assert result["desc"] == "\npartial value"
617 +
618 +
619 +class TestSchemaAwareTypeCoercion:
620 + """Verify that _fix_arg_types corrects miscoerced values using the
621 + tool schema."""
622 +
623 + @pytest.fixture
624 + def tools(self):
625 + from vllm.entrypoints.openai.chat_completion.protocol import (
626 + ChatCompletionToolsParam,
627 + )
628 +
629 + return [
630 + ChatCompletionToolsParam(
631 + type="function",
632 + function={
633 + "name": "TaskUpdate",
634 + "parameters": {
635 + "type": "object",
636 + "properties": {
637 + "taskId": {"type": "string"},
638 + "count": {"type": "integer"},
639 + "ratio": {"type": "number"},
640 + "flag": {"type": "string"},
641 + },
642 + },
643 + },
644 + )
645 + ]
646 +
647 + @pytest.fixture
648 + def parser_with_tools(self, mock_tokenizer, tools):
649 + return ParserEngine(
650 + mock_tokenizer,
651 + tools=tools,
652 + parser_engine_config=qwen3_config(thinking=False),
653 + )
654 +
655 + def test_string_param_not_coerced_to_int(self, parser_with_tools, mock_request):
656 + text = (
657 + "<tool_call>\n"
658 + "<function=TaskUpdate>\n"
659 + "<parameter=taskId>1</parameter>\n"
660 + "</function>\n"
661 + "</tool_call>"
662 + )
663 + result = parser_with_tools.extract_tool_calls(text, mock_request)
664 + assert result.tools_called
665 + args = json.loads(result.tool_calls[0].function.arguments)
666 + assert args["taskId"] == "1"
667 + assert isinstance(args["taskId"], str)
668 +
669 + def test_string_param_not_coerced_to_bool(self, parser_with_tools, mock_request):
670 + text = (
671 + "<tool_call>\n"
672 + "<function=TaskUpdate>\n"
673 + "<parameter=flag>true</parameter>\n"
674 + "</function>\n"
675 + "</tool_call>"
676 + )
677 + result = parser_with_tools.extract_tool_calls(text, mock_request)
678 + args = json.loads(result.tool_calls[0].function.arguments)
679 + assert args["flag"] == "true"
680 + assert isinstance(args["flag"], str)
681 +
682 + def test_int_param_still_coerced(self, parser_with_tools, mock_request):
683 + text = (
684 + "<tool_call>\n"
685 + "<function=TaskUpdate>\n"
686 + "<parameter=count>42</parameter>\n"
687 + "</function>\n"
688 + "</tool_call>"
689 + )
690 + result = parser_with_tools.extract_tool_calls(text, mock_request)
691 + args = json.loads(result.tool_calls[0].function.arguments)
692 + assert args["count"] == 42
693 + assert isinstance(args["count"], int)
694 +
695 + def test_no_tools_keeps_strings(self, parser, mock_request):
696 + text = (
697 + "<tool_call>\n"
698 + "<function=TaskUpdate>\n"
699 + "<parameter=taskId>1</parameter>\n"
700 + "</function>\n"
701 + "</tool_call>"
702 + )
703 + result = parser.extract_tool_calls(text, mock_request)
704 + args = json.loads(result.tool_calls[0].function.arguments)
705 + assert args["taskId"] == "1"
706 + assert isinstance(args["taskId"], str)
707 +
708 + def test_streaming_string_param_not_coerced(self, parser_with_tools, mock_request):
709 + chunks = [
710 + "<tool_call>\n",
711 + "<function=TaskUpdate>\n",
712 + "<parameter=taskId>1</parameter>\n",
713 + "</function>\n",
714 + "</tool_call>",
715 + ]
716 + results = simulate_tool_streaming(parser_with_tools, mock_request, chunks)
717 + args_str = collect_tool_arguments(results)
718 + args = json.loads(args_str)
719 + assert args["taskId"] == "1"
720 + assert isinstance(args["taskId"], str)
721 +
722 +
723 +class TestAnyOfTypeCoercion:
724 + """Verify that _fix_arg_types handles union types (anyOf/oneOf)."""
725 +
726 + @pytest.fixture
727 + def tools_with_anyof(self):
728 + from vllm.entrypoints.openai.chat_completion.protocol import (
729 + ChatCompletionToolsParam,
730 + )
731 +
732 + return [
733 + ChatCompletionToolsParam(
734 + type="function",
735 + function={
736 + "name": "set_config",
737 + "parameters": {
738 + "type": "object",
739 + "properties": {
740 + "port": {
741 + "anyOf": [
742 + {"type": "string"},
743 + {"type": "null"},
744 + ],
745 + },
746 + "count": {"type": "integer"},
747 + },
748 + },
749 + },
750 + )
751 + ]
752 +
753 + @pytest.fixture
754 + def parser_with_anyof(self, mock_tokenizer, tools_with_anyof):
755 + return ParserEngine(
756 + mock_tokenizer,
757 + tools=tools_with_anyof,
758 + parser_engine_config=qwen3_config(thinking=False),
759 + )
760 +
761 + def test_anyof_string_param_not_coerced(self, parser_with_anyof, mock_request):
762 + """A param with anyOf including 'string' must not be coerced
763 + to integer."""
764 + text = (
765 + "<tool_call>\n"
766 + "<function=set_config>\n"
767 + "<parameter=port>8080</parameter>\n"
768 + "</function>\n"
769 + "</tool_call>"
770 + )
771 + result = parser_with_anyof.extract_tool_calls(text, mock_request)
772 + assert result.tools_called
773 + args = json.loads(result.tool_calls[0].function.arguments)
774 + assert args["port"] == "8080"
775 +
776 +
777 +class TestSchemaCoercionBoolNumberNull:
778 + """Verify that _fix_arg_types coerces string values to non-string
779 + schema types using coerce_to_schema_type."""
780 +
781 + @pytest.fixture
782 + def tools(self):
783 + from vllm.entrypoints.openai.chat_completion.protocol import (
784 + ChatCompletionToolsParam,
785 + )
786 +
787 + return [
788 + ChatCompletionToolsParam(
789 + type="function",
790 + function={
791 + "name": "configure",
792 + "parameters": {
793 + "type": "object",
794 + "properties": {
795 + "enabled": {"type": "boolean"},
796 + "ratio": {"type": "number"},
797 + "count": {"type": "integer"},
798 + "value": {"type": ["integer", "null"]},
799 + "label": {"type": "string"},
800 + },
801 + },
802 + },
803 + )
804 + ]
805 +
806 + @pytest.fixture
807 + def parser_with_tools(self, mock_tokenizer, tools):
808 + return ParserEngine(
809 + mock_tokenizer,
810 + tools=tools,
811 + parser_engine_config=qwen3_config(thinking=False),
812 + )
813 +
814 + def test_bool_param_coerced(self, parser_with_tools, mock_request):
815 + text = (
816 + "<tool_call>\n"
817 + "<function=configure>\n"
818 + "<parameter=enabled>true</parameter>\n"
819 + "</function>\n"
820 + "</tool_call>"
821 + )
822 + result = parser_with_tools.extract_tool_calls(text, mock_request)
823 + args = json.loads(result.tool_calls[0].function.arguments)
824 + assert args["enabled"] is True
825 + assert isinstance(args["enabled"], bool)
826 +
827 + def test_number_param_whole_normalized(self, parser_with_tools, mock_request):
828 + text = (
829 + "<tool_call>\n"
830 + "<function=configure>\n"
831 + "<parameter=ratio>5.0</parameter>\n"
832 + "</function>\n"
833 + "</tool_call>"
834 + )
835 + result = parser_with_tools.extract_tool_calls(text, mock_request)
836 + args = json.loads(result.tool_calls[0].function.arguments)
837 + assert args["ratio"] == 5
838 + assert isinstance(args["ratio"], int)
839 +
840 + def test_number_param_fractional(self, parser_with_tools, mock_request):
841 + text = (
842 + "<tool_call>\n"
843 + "<function=configure>\n"
844 + "<parameter=ratio>3.14</parameter>\n"
845 + "</function>\n"
846 + "</tool_call>"
847 + )
848 + result = parser_with_tools.extract_tool_calls(text, mock_request)
849 + args = json.loads(result.tool_calls[0].function.arguments)
850 + assert args["ratio"] == pytest.approx(3.14)
851 + assert isinstance(args["ratio"], float)
852 +
853 + def test_null_coerced_when_in_schema(self, parser_with_tools, mock_request):
854 + text = (
855 + "<tool_call>\n"
856 + "<function=configure>\n"
857 + "<parameter=value>null</parameter>\n"
858 + "</function>\n"
859 + "</tool_call>"
860 + )
861 + result = parser_with_tools.extract_tool_calls(text, mock_request)
862 + args = json.loads(result.tool_calls[0].function.arguments)
863 + assert args["value"] is None
864 +
865 + def test_null_stays_string_without_null_schema(
866 + self, parser_with_tools, mock_request
867 + ):
868 + text = (
869 + "<tool_call>\n"
870 + "<function=configure>\n"
871 + "<parameter=label>null</parameter>\n"
872 + "</function>\n"
873 + "</tool_call>"
874 + )
875 + result = parser_with_tools.extract_tool_calls(text, mock_request)
876 + args = json.loads(result.tool_calls[0].function.arguments)
877 + assert args["label"] == "null"
878 + assert isinstance(args["label"], str)
879 +
880 + def test_streaming_bool_param_coerced(self, parser_with_tools, mock_request):
881 + chunks = [
882 + "<tool_call>\n",
883 + "<function=configure>\n",
884 + "<parameter=enabled>true</parameter>\n",
885 + "</function>\n",
886 + "</tool_call>",
887 + ]
888 + results = simulate_tool_streaming(parser_with_tools, mock_request, chunks)
889 + args_str = collect_tool_arguments(results)
890 + args = json.loads(args_str)
891 + assert args["enabled"] is True
892 + assert isinstance(args["enabled"], bool)
893 +
894 + def test_streaming_number_param_coerced(self, parser_with_tools, mock_request):
895 + chunks = [
896 + "<tool_call>\n",
897 + "<function=configure>\n",
898 + "<parameter=ratio>3.14</parameter>\n",
899 + "</function>\n",
900 + "</tool_call>",
901 + ]
902 + results = simulate_tool_streaming(parser_with_tools, mock_request, chunks)
903 + args_str = collect_tool_arguments(results)
904 + args = json.loads(args_str)
905 + assert args["ratio"] == pytest.approx(3.14)
906 + assert isinstance(args["ratio"], float)
907 +
908 + def test_streaming_matches_non_streaming_comprehensive(
909 + self, parser_with_tools, mock_request
910 + ):
911 + text = (
912 + "<tool_call>\n"
913 + "<function=configure>\n"
914 + "<parameter=enabled>true</parameter>\n"
915 + "<parameter=ratio>5.0</parameter>\n"
916 + "<parameter=count>42</parameter>\n"
917 + "<parameter=value>null</parameter>\n"
918 + "<parameter=label>hello</parameter>\n"
919 + "</function>\n"
920 + "</tool_call>"
921 + )
922 + non_stream = parser_with_tools.extract_tool_calls(text, mock_request)
923 + ns_args = json.loads(non_stream.tool_calls[0].function.arguments)
924 +
925 + chunks = [line + "\n" for line in text.split("\n") if line]
926 + results = simulate_tool_streaming(parser_with_tools, mock_request, chunks)
927 + s_args = json.loads(collect_tool_arguments(results))
928 +
929 + assert s_args == ns_args
930 + assert ns_args == {
931 + "enabled": True,
932 + "ratio": 5,
933 + "count": 42,
934 + "value": None,
935 + "label": "hello",
936 + }
937 +
938 +
939 +class TestNestedSchemaCoercion:
940 + """Verify that _fix_arg_types recurses into nested objects and arrays."""
941 +
942 + @pytest.fixture
943 + def tools(self):
944 + from vllm.entrypoints.openai.chat_completion.protocol import (
945 + ChatCompletionToolsParam,
946 + )
947 +
948 + return [
949 + ChatCompletionToolsParam(
950 + type="function",
951 + function={
952 + "name": "search",
953 + "parameters": {
954 + "type": "object",
955 + "properties": {
956 + "query": {"type": "string"},
957 + "filters": {
958 + "type": "object",
959 + "properties": {
960 + "language": {"type": "string"},
961 + "min_stars": {"type": "integer"},
962 + },
963 + },
964 + "tags": {
965 + "type": "array",
966 + "items": {"type": "string"},
967 + },
968 + "limits": {
969 + "type": "array",
970 + "items": {"type": "integer"},
971 + },
972 + "verbose": {"type": "boolean"},
973 + },
974 + },
975 + },
976 + ),
977 + ChatCompletionToolsParam(
978 + type="function",
979 + function={
980 + "name": "AskUserQuestion",
981 + "parameters": {
982 + "type": "object",
983 + "properties": {
984 + "questions": {
985 + "type": "array",
986 + "items": {
987 + "type": "object",
988 + "properties": {
989 + "question": {"type": "string"},
990 + "multiSelect": {
991 + "type": "boolean",
992 + },
993 + "answer": {
994 + "type": ["string", "null"],
995 + },
996 + },
997 + },
998 + },
999 + },
1000 + },
1001 + },
1002 + ),
1003 + ]
1004 +
1005 + @pytest.fixture
1006 + def parser_with_tools(self, mock_tokenizer, tools):
1007 + return ParserEngine(
1008 + mock_tokenizer,
1009 + tools=tools,
1010 + parser_engine_config=qwen3_config(thinking=False),
1011 + )
1012 +
1013 + def test_nested_object_coerced(self, parser_with_tools, mock_request):
1014 + text = (
1015 + "<tool_call>\n"
1016 + "<function=search>\n"
1017 + '<parameter=filters>{"language": "python",'
1018 + ' "min_stars": 100}</parameter>\n'
1019 + "</function>\n"
1020 + "</tool_call>"
1021 + )
1022 + result = parser_with_tools.extract_tool_calls(text, mock_request)
1023 + args = json.loads(result.tool_calls[0].function.arguments)
1024 + assert args["filters"] == {"language": "python", "min_stars": 100}
1025 + assert isinstance(args["filters"]["min_stars"], int)
1026 +
1027 + def test_nested_array_items_coerced(self, parser_with_tools, mock_request):
1028 + text = (
1029 + "<tool_call>\n"
1030 + "<function=search>\n"
1031 + "<parameter=limits>[10, 20, 30]</parameter>\n"
1032 + "</function>\n"
1033 + "</tool_call>"
1034 + )
1035 + result = parser_with_tools.extract_tool_calls(text, mock_request)
1036 + args = json.loads(result.tool_calls[0].function.arguments)
1037 + assert args["limits"] == [10, 20, 30]
1038 + assert all(isinstance(v, int) for v in args["limits"])
1039 +
1040 + def test_nested_string_array_not_coerced(self, parser_with_tools, mock_request):
1041 + text = (
1042 + "<tool_call>\n"
1043 + "<function=search>\n"
1044 + '<parameter=tags>["ml", "42"]</parameter>\n'
1045 + "</function>\n"
1046 + "</tool_call>"
1047 + )
1048 + result = parser_with_tools.extract_tool_calls(text, mock_request)
1049 + args = json.loads(result.tool_calls[0].function.arguments)
1050 + assert args["tags"] == ["ml", "42"]
1051 + assert all(isinstance(v, str) for v in args["tags"])
1052 +
1053 + def test_array_of_objects_with_bool_and_null_coerced(
1054 + self, parser_with_tools, mock_request
1055 + ):
1056 + text = (
1057 + "<tool_call>\n"
1058 + "<function=AskUserQuestion>\n"
1059 + "<parameter=questions>"
1060 + '[{"question": "Pick a color",'
1061 + ' "multiSelect": false, "answer": null}]'
1062 + "</parameter>\n"
1063 + "</function>\n"
1064 + "</tool_call>"
1065 + )
1066 + result = parser_with_tools.extract_tool_calls(text, mock_request)
1067 + args = json.loads(result.tool_calls[0].function.arguments)
1068 + questions = args["questions"]
1069 + assert isinstance(questions, list)
1070 + assert len(questions) == 1
1071 + assert questions[0]["question"] == "Pick a color"
1072 + assert questions[0]["multiSelect"] is False
1073 + assert questions[0]["answer"] is None
1074 +
1075 + def test_streaming_array_of_objects_with_bool_and_null_coerced(
1076 + self, parser_with_tools, mock_request
1077 + ):
1078 + chunks = [
1079 + "<tool_call>\n",
1080 + "<function=AskUserQuestion>\n",
1081 + '<parameter=questions>[{"question": "Pick a color",',
1082 + ' "multiSelect": false, "answer": null}]',
1083 + "</parameter>\n",
1084 + "</function>\n",
1085 + "</tool_call>",
1086 + ]
1087 + results = simulate_tool_streaming(parser_with_tools, mock_request, chunks)
1088 + args_str = collect_tool_arguments(results)
1089 + args = json.loads(args_str)
1090 + questions = args["questions"]
1091 + assert isinstance(questions, list)
1092 + assert len(questions) == 1
1093 + assert questions[0]["question"] == "Pick a color"
1094 + assert questions[0]["multiSelect"] is False
1095 + assert questions[0]["answer"] is None
tests/parser/engine/test_qwen3_reasoning.py new
+549
@@ -0,0 +1,549 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Tests for the engine-based Qwen3 reasoning parser.
4 +
5 +Validates that ``Qwen3Parser`` correctly handles
6 +``<think>``/``</think>`` reasoning with Qwen3-specific extensions:
7 +- ``<tool_call>`` as implicit reasoning end (terminal + token ID)
8 +- Stripping ``<think>`` from generated output (old template compat)
9 +- No terminal text (``</think>``, ``<tool_call>``) leaks into output
10 +"""
11 +
12 +import dataclasses
13 +
14 +import pytest
15 +
16 +from tests.parser.engine.conftest import make_mock_tokenizer
17 +from tests.parser.engine.streaming_helpers import simulate_reasoning_streaming
18 +from vllm.parser.engine.parser_engine_config import ParserState
19 +from vllm.parser.qwen3 import Qwen3Parser, qwen3_config
20 +
21 +_THINK_START_ID = 50
22 +_THINK_END_ID = 51
23 +_TOOL_CALL_ID = 60
24 +_TOOL_CALL_END_ID = 61
25 +_TEXT_ID = 100
26 +
27 +_QWEN3_VOCAB = {
28 + "<think>": _THINK_START_ID,
29 + "</think>": _THINK_END_ID,
30 + "<tool_call>": _TOOL_CALL_ID,
31 + "</tool_call>": _TOOL_CALL_END_ID,
32 +}
33 +
34 +
35 +@pytest.fixture
36 +def mock_tokenizer():
37 + return make_mock_tokenizer(_QWEN3_VOCAB)
38 +
39 +
40 +@pytest.fixture
41 +def parser(mock_tokenizer):
42 + return Qwen3Parser(mock_tokenizer)
43 +
44 +
45 +class TestNonStreaming:
46 + def test_reasoning_then_content(self, parser):
47 + text = "<think>Let me analyze.</think>The answer is 42."
48 + reasoning, content = parser.extract_reasoning(text, None)
49 + assert reasoning == "Let me analyze."
50 + assert content == "The answer is 42."
51 +
52 + def test_no_start_token_in_output(self, parser):
53 + """Qwen3.5+ style: <think> in prompt, only </think> in output."""
54 + text = "Let me think about this.</think>The answer is 42."
55 + reasoning, content = parser.extract_reasoning(text, None)
56 + assert reasoning == "Let me think about this."
57 + assert content == "The answer is 42."
58 +
59 + def test_reasoning_only(self, parser):
60 + text = "<think>Still thinking...</think>"
61 + reasoning, content = parser.extract_reasoning(text, None)
62 + assert reasoning == "Still thinking..."
63 + assert content is None
64 +
65 + def test_no_end_tag_all_reasoning(self, parser):
66 + """No </think> means truncated output — everything is reasoning."""
67 + text = "Hello, no reasoning here."
68 + reasoning, content = parser.extract_reasoning(text, None)
69 + assert reasoning == "Hello, no reasoning here."
70 + assert content is None
71 +
72 + def test_multiline_reasoning(self, parser):
73 + text = (
74 + "<think>Step 1: parse.\nStep 2: compute.\nStep 3: output.</think>Result: 7."
75 + )
76 + reasoning, content = parser.extract_reasoning(text, None)
77 + assert "Step 1" in reasoning
78 + assert "Step 3" in reasoning
79 + assert content == "Result: 7."
80 +
81 + def test_tool_call_implicit_end(self, parser):
82 + """<tool_call> without </think> acts as implicit reasoning end."""
83 + text = (
84 + "<think>I need to read the file.\n\n"
85 + "<tool_call>\n<function=bash>\n"
86 + "<parameter=cmd>ls</parameter>\n"
87 + "</function>\n</tool_call>"
88 + )
89 + reasoning, content = parser.extract_reasoning(text, None)
90 + assert reasoning == "I need to read the file.\n\n"
91 + assert "</think>" not in reasoning
92 + assert "<tool_call>" not in reasoning
93 +
94 + def test_tool_call_implicit_end_no_think(self, parser):
95 + """<tool_call> as implicit end, no <think> in output."""
96 + text = (
97 + "I need to read the file.\n\n"
98 + "<tool_call>\n<function=bash>\n"
99 + "<parameter=cmd>ls</parameter>\n"
100 + "</function>\n</tool_call>"
101 + )
102 + reasoning, content = parser.extract_reasoning(text, None)
103 + assert reasoning == "I need to read the file.\n\n"
104 + assert "<tool_call>" not in reasoning
105 +
106 + def test_live_scenario_think_end_before_tool_call(self, parser):
107 + """Real model output: </think> immediately before <tool_call>.
108 +
109 + Regression test for the bug where </think> and <parameter=...>
110 + leaked into reasoning content.
111 + """
112 + text = (
113 + "The user wants to see what files are in the current directory"
114 + " and their contents. Let me start by listing the directory."
115 + "</think><tool_call><function=read>"
116 + "<parameter=filePath>/Users/test/demo</parameter>"
117 + "</function></tool_call>"
118 + )
119 + reasoning, content = parser.extract_reasoning(text, None)
120 + expected_reasoning = (
121 + "The user wants to see what files are in the current directory"
122 + " and their contents. Let me start by listing the directory."
123 + )
124 + assert reasoning == expected_reasoning
125 + assert "</think>" not in reasoning
126 + assert "<tool_call>" not in reasoning
127 + assert "<parameter=" not in reasoning
128 +
129 + def test_no_terminal_text_in_reasoning(self, parser):
130 + """Terminal text must never appear in reasoning output."""
131 + text = "Reasoning here.</think>Content here."
132 + reasoning, content = parser.extract_reasoning(text, None)
133 + assert "</think>" not in (reasoning or "")
134 + assert "<think>" not in (reasoning or "")
135 +
136 + def test_no_terminal_text_in_content(self, parser):
137 + """Terminal text must never appear in content output."""
138 + text = "Reasoning here.</think>Content here."
139 + reasoning, content = parser.extract_reasoning(text, None)
140 + assert "</think>" not in (content or "")
141 + assert "<think>" not in (content or "")
142 +
143 + def test_duplicate_think_end_absorbed(self, parser):
144 + """Duplicate </think> in CONTENT state must not leak."""
145 + text = "Reasoning here.</think>Content here.</think>More content."
146 + reasoning, content = parser.extract_reasoning(text, None)
147 + assert reasoning == "Reasoning here."
148 + assert content == "Content here.More content."
149 +
150 +
151 +class TestIsReasoningEnd:
152 + def test_think_end_token(self, parser):
153 + assert parser.is_reasoning_end([_THINK_START_ID, 1, _THINK_END_ID])
154 +
155 + def test_no_end_token(self, parser):
156 + assert not parser.is_reasoning_end([_THINK_START_ID, 1, 2])
157 +
158 + def test_start_after_end_means_not_ended(self, parser):
159 + assert not parser.is_reasoning_end([_THINK_END_ID, _THINK_START_ID, 1])
160 +
161 + def test_tool_call_as_implicit_end(self, parser):
162 + """Unpaired <tool_call> is implicit reasoning end."""
163 + assert parser.is_reasoning_end([_THINK_START_ID, 1, _TOOL_CALL_ID])
164 +
165 + def test_paired_tool_call_not_end(self, parser):
166 + """Paired <tool_call>...</tool_call> (from template) is NOT end."""
167 + assert not parser.is_reasoning_end(
168 + [_THINK_START_ID, 1, _TOOL_CALL_ID, 2, _TOOL_CALL_END_ID]
169 + )
170 +
171 + def test_tool_call_after_think_end(self, parser):
172 + """<tool_call> after </think> — already ended."""
173 + assert parser.is_reasoning_end(
174 + [_THINK_START_ID, 1, _THINK_END_ID, _TOOL_CALL_ID]
175 + )
176 +
177 + def test_empty_ids(self, parser):
178 + assert not parser.is_reasoning_end([])
179 +
180 +
181 +class TestStreaming:
182 + def test_basic_streaming(self, parser):
183 + reasoning, content = simulate_reasoning_streaming(
184 + parser,
185 + ["<think>", "thinking", " hard", "</think>", "done"],
186 + [
187 + (_THINK_START_ID,),
188 + (1,),
189 + (2,),
190 + (_THINK_END_ID,),
191 + (3,),
192 + ],
193 + )
194 + assert reasoning == "thinking hard"
195 + assert content == "done"
196 +
197 + def test_streaming_no_start_token(self, parser):
198 + """Qwen3.5 style: no <think> in output, just reasoning then </think>."""
199 + reasoning, content = simulate_reasoning_streaming(
200 + parser,
201 + ["reasoning ", "text", "</think>", "content"],
202 + [
203 + (1,),
204 + (2,),
205 + (_THINK_END_ID,),
206 + (3,),
207 + ],
208 + )
209 + assert reasoning == "reasoning text"
210 + assert content == "content"
211 +
212 + def test_streaming_start_token_stripped(self, parser):
213 + """<think> in output (old template) should be stripped."""
214 + reasoning, content = simulate_reasoning_streaming(
215 + parser,
216 + ["<think>reasoning", "</think>", "content"],
217 + [
218 + (_THINK_START_ID, 1),
219 + (_THINK_END_ID,),
220 + (2,),
221 + ],
222 + )
223 + assert reasoning == "reasoning"
224 + assert content == "content"
225 +
226 + def test_streaming_tool_call_implicit_end(self, parser):
227 + """<tool_call> ends reasoning implicitly during streaming."""
228 + reasoning, content = simulate_reasoning_streaming(
229 + parser,
230 + ["I need to check.", "<tool_call>", "\n<function=test>"],
231 + [
232 + (1,),
233 + (_TOOL_CALL_ID,),
234 + (2,),
235 + ],
236 + )
237 + assert reasoning == "I need to check."
238 + assert "<tool_call>" not in reasoning
239 + assert "</think>" not in reasoning
240 + assert content is not None
241 +
242 + def test_streaming_content_after_think_end(self, parser):
243 + """Content deltas after </think> are routed as content."""
244 + reasoning, content = simulate_reasoning_streaming(
245 + parser,
246 + ["reasoning", "</think>", "content1", " content2"],
247 + [
248 + (1,),
249 + (_THINK_END_ID,),
250 + (2,),
251 + (3,),
252 + ],
253 + )
254 + assert reasoning == "reasoning"
255 + assert content == "content1 content2"
256 +
257 + def test_streaming_content_after_tool_call(self, parser):
258 + """Content deltas after <tool_call> are routed as content."""
259 + reasoning, content = simulate_reasoning_streaming(
260 + parser,
261 + ["thinking", "<tool_call>", "<function=f>"],
262 + [
263 + (1,),
264 + (_TOOL_CALL_ID,),
265 + (2,),
266 + ],
267 + )
268 + assert reasoning == "thinking"
269 + assert "<tool_call>" not in reasoning
270 + assert content is not None
271 +
272 + def test_streaming_end_grouped_with_content(self, parser):
273 + """</think> grouped with following content in one delta."""
274 + reasoning, content = simulate_reasoning_streaming(
275 + parser,
276 + ["reasoning", "</think>the answer"],
277 + [
278 + (1,),
279 + (_THINK_END_ID, 2),
280 + ],
281 + )
282 + assert reasoning == "reasoning"
283 + assert content == "the answer"
284 +
285 + def test_streaming_think_and_end_in_one_delta(self, parser):
286 + """<think> and </think> in the same delta."""
287 + reasoning, content = simulate_reasoning_streaming(
288 + parser,
289 + ["<think>reasoning</think>"],
290 + [
291 + (_THINK_START_ID, 1, _THINK_END_ID),
292 + ],
293 + )
294 + assert reasoning == "reasoning"
295 + assert content == ""
296 +
297 + def test_streaming_pure_content_no_think(self, parser):
298 + """No think tokens at all — everything is reasoning (truncated)."""
299 + reasoning, content = simulate_reasoning_streaming(
300 + parser,
301 + ["hello ", "world"],
302 + [
303 + (1,),
304 + (2,),
305 + ],
306 + )
307 + assert reasoning == "hello world"
308 + assert content == ""
309 +
310 + def test_streaming_think_end_and_tool_call_same_delta(self, parser):
311 + """</think> and <tool_call> in the same delta — no leakage.
312 +
313 + Regression test: the old override split at <tool_call> without
314 + stripping </think>, causing </think> to leak into reasoning.
315 + """
316 + reasoning, content = simulate_reasoning_streaming(
317 + parser,
318 + [
319 + "Let me list the directory.",
320 + "</think><tool_call>",
321 + "<function=read>",
322 + "<parameter=filePath>/tmp</parameter>",
323 + ],
324 + [
325 + (1,),
326 + (_THINK_END_ID, _TOOL_CALL_ID),
327 + (2,),
328 + (3,),
329 + ],
330 + )
331 + assert reasoning == "Let me list the directory."
332 + assert "</think>" not in reasoning
333 + assert "<tool_call>" not in reasoning
334 + assert "<parameter=" not in reasoning
335 + assert content is not None
336 +
337 + def test_streaming_no_terminal_text_leaks(self, parser):
338 + """Terminal text must never appear in reasoning or content."""
339 + reasoning, content = simulate_reasoning_streaming(
340 + parser,
341 + ["reasoning", "</think>", "content"],
342 + [
343 + (1,),
344 + (_THINK_END_ID,),
345 + (2,),
346 + ],
347 + )
348 + assert "</think>" not in reasoning
349 + assert "</think>" not in content
350 + assert "<think>" not in reasoning
351 +
352 + def test_streaming_duplicate_think_end_absorbed(self, parser):
353 + """Duplicate </think> token in CONTENT state must not leak."""
354 + reasoning, content = simulate_reasoning_streaming(
355 + parser,
356 + ["reasoning", "</think>", "content", "</think>", "more"],
357 + [
358 + (1,),
359 + (_THINK_END_ID,),
360 + (2,),
361 + (_THINK_END_ID,),
362 + (3,),
363 + ],
364 + )
365 + assert reasoning == "reasoning"
366 + assert content == "contentmore"
367 +
368 +
369 +class TestTrailingWhitespaceStripping:
370 + """When strip_trailing_reasoning_whitespace is True,
371 + trailing whitespace before </think> must be stripped.
372 +
373 + Models often generate trailing newlines before </think>, and these
374 + accumulate across multi-turn conversations via a feedback loop.
375 + """
376 +
377 + @pytest.fixture
378 + def parser_with_strip(self):
379 + cfg = dataclasses.replace(
380 + qwen3_config(),
381 + strip_trailing_reasoning_whitespace=True,
382 + )
383 + return Qwen3Parser(make_mock_tokenizer(_QWEN3_VOCAB), parser_engine_config=cfg)
384 +
385 + def test_non_streaming_trailing_newline(self, parser_with_strip):
386 + text = "Reasoning here.\n</think>Content."
387 + reasoning, content = parser_with_strip.extract_reasoning(text, None)
388 + assert reasoning == "Reasoning here."
389 + assert content == "Content."
390 +
391 + def test_non_streaming_multiple_trailing_newlines(self, parser_with_strip):
392 + text = "Reasoning here.\n\n\n</think>Content."
393 + reasoning, content = parser_with_strip.extract_reasoning(text, None)
394 + assert reasoning == "Reasoning here."
395 + assert content == "Content."
396 +
397 + def test_non_streaming_internal_newlines_preserved(self, parser_with_strip):
398 + text = "Step 1.\n\nStep 2.\n\nStep 3.</think>Answer."
399 + reasoning, content = parser_with_strip.extract_reasoning(text, None)
400 + assert reasoning == "Step 1.\n\nStep 2.\n\nStep 3."
401 + assert content == "Answer."
402 +
403 + def test_non_streaming_only_newlines_becomes_none(self, parser_with_strip):
404 + text = "\n\n\n</think>Content."
405 + reasoning, content = parser_with_strip.extract_reasoning(text, None)
406 + assert reasoning is None
407 + assert content == "Content."
408 +
409 + def test_streaming_trailing_newline_stripped(self, parser_with_strip):
410 + reasoning, content = simulate_reasoning_streaming(
411 + parser_with_strip,
412 + ["thinking.\n", "</think>", "done"],
413 + [
414 + (1,),
415 + (_THINK_END_ID,),
416 + (2,),
417 + ],
418 + )
419 + assert reasoning == "thinking."
420 + assert content == "done"
421 +
422 + def test_streaming_multiple_trailing_newlines_stripped(self, parser_with_strip):
423 + reasoning, content = simulate_reasoning_streaming(
424 + parser_with_strip,
425 + ["thinking.\n", "\n", "\n", "</think>", "done"],
426 + [
427 + (1,),
428 + (2,),
429 + (3,),
430 + (_THINK_END_ID,),
431 + (4,),
432 + ],
433 + )
434 + assert reasoning == "thinking."
435 + assert content == "done"
436 +
437 + def test_streaming_internal_newlines_preserved(self, parser_with_strip):
438 + reasoning, content = simulate_reasoning_streaming(
439 + parser_with_strip,
440 + ["Step 1.\n", "\nStep 2.\n", "</think>", "Answer"],
441 + [
442 + (1,),
443 + (2,),
444 + (_THINK_END_ID,),
445 + (3,),
446 + ],
447 + )
448 + assert reasoning == "Step 1.\n\nStep 2."
449 + assert content == "Answer"
450 +
451 + def test_streaming_trailing_newlines_before_tool_call(self, parser_with_strip):
452 + """Trailing newlines before implicit <tool_call> end are stripped."""
453 + reasoning, content = simulate_reasoning_streaming(
454 + parser_with_strip,
455 + ["I'll check.\n\n", "<tool_call>", "<function=test>"],
456 + [
457 + (1,),
458 + (_TOOL_CALL_ID,),
459 + (2,),
460 + ],
461 + )
462 + assert reasoning == "I'll check."
463 + assert "<tool_call>" not in reasoning
464 +
465 +
466 +class TestWhitespaceStrippingDisabled:
467 + """When strip_trailing_reasoning_whitespace is False,
468 + trailing whitespace in reasoning must be preserved."""
469 +
470 + @pytest.fixture
471 + def parser_no_strip(self):
472 + cfg = dataclasses.replace(
473 + qwen3_config(),
474 + strip_trailing_reasoning_whitespace=False,
475 + )
476 + return Qwen3Parser(make_mock_tokenizer(_QWEN3_VOCAB), parser_engine_config=cfg)
477 +
478 + def test_non_streaming_preserves_trailing_newline(self, parser_no_strip):
479 + text = "Reasoning here.\n</think>Content."
480 + reasoning, content = parser_no_strip.extract_reasoning(text, None)
481 + assert reasoning == "Reasoning here.\n"
482 + assert content == "Content."
483 +
484 + def test_streaming_preserves_trailing_newlines(self, parser_no_strip):
485 + reasoning, content = simulate_reasoning_streaming(
486 + parser_no_strip,
487 + ["thinking.\n", "\n", "</think>", "done"],
488 + [
489 + (1,),
490 + (2,),
491 + (_THINK_END_ID,),
492 + (3,),
493 + ],
494 + )
495 + assert reasoning == "thinking.\n\n"
496 + assert content == "done"
497 +
498 +
499 +class TestThinkingDisabled:
500 + """When ``enable_thinking=False``, the chat template pre-fills a closed
501 + ``<think>\\n\\n</think>\\n\\n`` block. The model output starts in content
502 + state, so the parser's initial state must be CONTENT — not REASONING.
503 + """
504 +
505 + def test_thinking_disabled_initial_state_is_content(self, mock_tokenizer):
506 + p = Qwen3Parser(
507 + mock_tokenizer,
508 + chat_template_kwargs={"enable_thinking": False},
509 + )
510 + assert p.parser_engine_config.initial_state == ParserState.CONTENT
511 +
512 + def test_thinking_enabled_initial_state_is_reasoning(self, mock_tokenizer):
513 + p = Qwen3Parser(
514 + mock_tokenizer,
515 + chat_template_kwargs={"enable_thinking": True},
516 + )
517 + assert p.parser_engine_config.initial_state == ParserState.REASONING
518 +
519 + def test_default_initial_state_is_reasoning(self, mock_tokenizer):
520 + p = Qwen3Parser(mock_tokenizer)
521 + assert p.parser_engine_config.initial_state == ParserState.REASONING
522 +
523 + def test_thinking_disabled_streaming_content_only(self, mock_tokenizer):
524 + """Plain text with thinking disabled must stream as content, not
525 + reasoning. Before the fix, the REASONING initial state caused all
526 + output to be emitted as reasoning chunks."""
527 + p = Qwen3Parser(
528 + mock_tokenizer,
529 + chat_template_kwargs={"enable_thinking": False},
530 + )
531 + reasoning, content = simulate_reasoning_streaming(
532 + p,
533 + ["The answer", " is 42."],
534 + [
535 + (_TEXT_ID,),
536 + (_TEXT_ID,),
537 + ],
538 + )
539 + assert content == "The answer is 42."
540 + assert reasoning == ""
541 +
542 + def test_thinking_disabled_non_streaming(self, mock_tokenizer):
543 + p = Qwen3Parser(
544 + mock_tokenizer,
545 + chat_template_kwargs={"enable_thinking": False},
546 + )
547 + reasoning, content = p.extract_reasoning("The answer is 42.", None)
548 + assert reasoning is None
549 + assert content == "The answer is 42."
tests/parser/engine/test_replay.py new
+189
@@ -0,0 +1,189 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Replay tests for engine parsers (holdback, skip-tool-parsing, adapters).
4 +
5 +Replays dynamically built token sequences at different chunk sizes and
6 +holdback depths to verify chunk-size invariance and terminal-token hygiene.
7 +"""
8 +
9 +from __future__ import annotations
10 +
11 +import pytest
12 +
13 +from tests.parser.engine.replay_harness import (
14 + _test_request,
15 + assert_no_terminal_leakage,
16 + assert_parse_output,
17 + collect_output,
18 + make_mock_tokenizer,
19 + replay_streaming,
20 +)
21 +from tests.parser.engine.trace_builder import build_samples
22 +from vllm.parser.abstract_parser import Parser
23 +from vllm.parser.engine.registered_adapters import (
24 + Qwen3Parser,
25 +)
26 +
27 +_ENGINE_PARSERS: dict[str, type[Parser]] = {
28 + "qwen3_engine": Qwen3Parser,
29 +}
30 +
31 +_qwen3_samples = build_samples("qwen3")
32 +
33 +_QWEN3_TERMINALS = [
34 + "<think>",
35 + "</think>",
36 + "<tool_call>",
37 + "</tool_call>",
38 + "<function=",
39 + "</function>",
40 +]
41 +
42 +HOLDBACK_CONFIGS = [6, 12, 24]
43 +
44 +
45 +@pytest.mark.parametrize("holdback", HOLDBACK_CONFIGS, ids=lambda h: f"holdback{h}")
46 +@pytest.mark.parametrize("chunk_size", [5, 10], ids=lambda c: f"chunk{c}")
47 +@pytest.mark.parametrize("sample", _qwen3_samples, ids=lambda s: s.id)
48 +class TestQwen3ReplayWithHoldback:
49 + """Replay Qwen3 with simulated detokenizer holdback."""
50 +
51 + def test_replay(self, sample, chunk_size, holdback):
52 + tokenizer = make_mock_tokenizer(sample)
53 + parser = Qwen3Parser(tokenizer, sample.tools)
54 + deltas = replay_streaming(
55 + parser,
56 + sample.tokens,
57 + chunk_size=chunk_size,
58 + holdback_chars=holdback,
59 + )
60 + output = collect_output(deltas)
61 +
62 + assert_parse_output(output, sample)
63 + assert_no_terminal_leakage(
64 + output,
65 + _QWEN3_TERMINALS,
66 + context=f"chunk_size={chunk_size}, holdback={holdback}",
67 + )
68 +
69 +
70 +_TOOL_CALL_SAMPLES = [
71 + (Qwen3Parser, s)
72 + for s in _qwen3_samples
73 + if s.expected_tool_calls and s.expected_reasoning
74 +]
75 +
76 +
77 +def _suppressed_expectations(sample) -> tuple[str, str]:
78 + """Compute expected (reasoning, content) when tools are suppressed.
79 +
80 + When an explicit reasoning-end delimiter (``</think>``, ``<channel|>``)
81 + is present, reasoning ends there and the tool call block becomes content.
82 + When reasoning ends implicitly (the tool-start token triggers both
83 + REASONING_END and TOOL_CALL_START), reasoning still ends at the tool
84 + start and the raw tool call block becomes content text — only the
85 + structured tool parsing is suppressed, not the reasoning boundary.
86 + """
87 + full_text = "".join(text for _, text in sample.tokens)
88 + reasoning = sample.expected_reasoning
89 + idx = full_text.find(reasoning)
90 + if idx < 0:
91 + return (full_text, "")
92 + after_reasoning = full_text[idx + len(reasoning) :]
93 + for delim in ("</think>", "<channel|>"):
94 + pos = after_reasoning.find(delim)
95 + if pos >= 0:
96 + return (reasoning, after_reasoning[pos + len(delim) :])
97 + for delim in ("<tool_call>",):
98 + pos = after_reasoning.find(delim)
99 + if pos >= 0:
100 + return (reasoning, after_reasoning[pos:])
101 + return (full_text, "")
102 +
103 +
104 +_DUMMY_TOOLS = [
105 + {
106 + "type": "function",
107 + "function": {"name": "stub", "parameters": {"type": "object"}},
108 + }
109 +]
110 +
111 +
112 +@pytest.mark.parametrize("chunk_size", [1, 5, None], ids=lambda c: f"chunk{c}")
113 +@pytest.mark.parametrize(
114 + "parser_cls,sample",
115 + _TOOL_CALL_SAMPLES,
116 + ids=lambda v: v.id if hasattr(v, "id") else v.__name__,
117 +)
118 +class TestSkipToolParsingReplay:
119 + """Replay with skip_tool_parsing=True (tool_choice='none').
120 +
121 + Verifies that reasoning is extracted normally and the raw tool call
122 + block appears as content text with no tool calls parsed.
123 + """
124 +
125 + def test_replay(self, parser_cls, sample, chunk_size):
126 + tokenizer = make_mock_tokenizer(sample)
127 + kwargs = {}
128 + if sample.chat_template_kwargs:
129 + kwargs["chat_template_kwargs"] = sample.chat_template_kwargs
130 + parser = parser_cls(tokenizer, **kwargs)
131 +
132 + request = _test_request()
133 + request.tool_choice = "none"
134 + request.tools = _DUMMY_TOOLS
135 +
136 + all_ids = [tid for tid, _ in sample.tokens]
137 + all_texts = [text for _, text in sample.tokens]
138 + if chunk_size is None:
139 + chunk_size = len(all_ids)
140 +
141 + results = []
142 + chunks = list(range(0, len(all_ids), chunk_size))
143 + for i, start in enumerate(chunks):
144 + end = min(start + chunk_size, len(all_ids))
145 + is_last = i == len(chunks) - 1
146 + result = parser.parse_delta(
147 + "".join(all_texts[start:end]),
148 + all_ids[start:end],
149 + request,
150 + prompt_token_ids=[] if start == 0 else None,
151 + finished=is_last,
152 + )
153 + results.append(result)
154 +
155 + output = collect_output(results)
156 +
157 + expected_reasoning, expected_content = _suppressed_expectations(sample)
158 +
159 + assert output.reasoning == expected_reasoning, (
160 + f"Reasoning mismatch:\n"
161 + f" expected: {expected_reasoning!r}\n"
162 + f" actual: {output.reasoning!r}"
163 + )
164 + assert output.tool_calls == [], (
165 + f"Expected no tool calls but got {output.tool_calls}"
166 + )
167 + assert output.content == expected_content, (
168 + f"Content mismatch:\n"
169 + f" expected: {expected_content!r}\n"
170 + f" actual: {output.content!r}"
171 + )
172 +
173 +
174 +class TestAdapterReferences:
175 + """Verify make_adapters sets reasoning/tool parser class refs on parser engine
176 + parser classes so the serving layer finds them and calls adjust_request."""
177 +
178 + @pytest.mark.parametrize(
179 + "parser_name",
180 + list(_ENGINE_PARSERS.keys()),
181 + )
182 + def test_adapter_cls_refs_set(self, parser_name):
183 + parser_cls = _ENGINE_PARSERS[parser_name]
184 + assert parser_cls.reasoning_parser_cls is not None, (
185 + f"{parser_name}: reasoning_parser_cls is None"
186 + )
187 + assert parser_cls.tool_parser_cls is not None, (
188 + f"{parser_name}: tool_parser_cls is None"
189 + )
tests/parser/engine/test_token_id_scanner.py new
+631
@@ -0,0 +1,631 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Tests for TokenIDScanner, focusing on hold-back text recovery.
4 +
5 +Uses gemma4_config for all end-to-end engine tests, covering
6 +reasoning channels, tool calls, and combined flows."""
7 +
8 +from unittest.mock import MagicMock
9 +
10 +import pytest
11 +
12 +from vllm.parser.engine.events import EventType
13 +from vllm.parser.engine.token_id_scanner import (
14 + PreLexedTerminal,
15 + TextChunk,
16 + TokenIDScanner,
17 +)
18 +
19 +CHANNEL_START = "<|channel>"
20 +CHANNEL_END = "<channel|>"
21 +CHANNEL_START_ID = 100
22 +CHANNEL_END_ID = 101
23 +REGULAR_TOKEN_ID = 200
24 +TOOL_START = "<tool_call>"
25 +TOOL_END = "</tool_call>"
26 +TOOL_START_ID = 110
27 +TOOL_END_ID = 111
28 +
29 +
30 +@pytest.fixture
31 +def tokenizer():
32 + tok = MagicMock()
33 + tok.get_vocab.return_value = {
34 + CHANNEL_START: CHANNEL_START_ID,
35 + CHANNEL_END: CHANNEL_END_ID,
36 + }
37 + tok.decode.side_effect = lambda ids: {
38 + CHANNEL_START_ID: CHANNEL_START,
39 + CHANNEL_END_ID: CHANNEL_END,
40 + REGULAR_TOKEN_ID: "regular",
41 + }.get(ids[0], f"<unk:{ids[0]}>")
42 + return tok
43 +
44 +
45 +@pytest.fixture
46 +def scanner(tokenizer):
47 + return TokenIDScanner(
48 + token_id_to_terminal={
49 + CHANNEL_START_ID: "THINK_START",
50 + CHANNEL_END_ID: "THINK_END",
51 + },
52 + tokenizer=tokenizer,
53 + )
54 +
55 +
56 +class TestJoinDecodedTextReturnsStr:
57 + """_join_decoded_text now returns str unconditionally (was
58 + str | None when an isinstance guard made a branch unreachable)."""
59 +
60 + @pytest.fixture
61 + def bare_scanner(self):
62 + return TokenIDScanner({}, tokenizer=None, drop_token_ids=set())
63 +
64 + def test_mixed_items(self, bare_scanner):
65 + items = [
66 + TextChunk("hello "),
67 + PreLexedTerminal("TOOL_START", 42, "<tool_call>"),
68 + TextChunk(" world"),
69 + ]
70 + result = bare_scanner._join_decoded_text(items)
71 + assert isinstance(result, str)
72 + assert result == "hello <tool_call> world"
73 +
74 + def test_empty_list(self, bare_scanner):
75 + result = bare_scanner._join_decoded_text([])
76 + assert isinstance(result, str)
77 + assert result == ""
78 +
79 + def test_only_text_chunks(self, bare_scanner):
80 + result = bare_scanner._join_decoded_text([TextChunk("abc"), TextChunk("def")])
81 + assert result == "abcdef"
82 +
83 +
84 +class TestHoldbackTextRecovery:
85 + def test_holdback_text_with_special_token_text_absent(self, scanner):
86 + """delta_text has hold-back text but the special token's text is
87 + NOT in delta_text (held back by the detokenizer). Terminal is
88 + deferred until the text arrives in a subsequent delta."""
89 + result = scanner.scan(
90 + delta_text="processed is appropriate.",
91 + delta_token_ids=[CHANNEL_END_ID],
92 + )
93 +
94 + assert len(result) == 0
95 +
96 + # Second scan: terminal text arrives (detokenizer flushes).
97 + # Deferred terminal resolves with holdback text before it.
98 + result2 = scanner.scan(
99 + delta_text="<channel|>Understood.",
100 + delta_token_ids=[20, 21],
101 + )
102 + pre_lexed = [r for r in result2 if isinstance(r, PreLexedTerminal)]
103 + assert len(pre_lexed) == 1
104 + assert pre_lexed[0].terminal == "THINK_END"
105 + texts = [r.text for r in result2 if isinstance(r, TextChunk)]
106 + combined = "".join(texts)
107 + assert "processed is appropriate." in combined
108 + assert "Understood." in combined
109 +
110 + def test_holdback_text_with_special_token_text_present(self, scanner):
111 + """delta_text includes hold-back text AND the special token text."""
112 + result = scanner.scan(
113 + delta_text="holdback text<channel|>",
114 + delta_token_ids=[CHANNEL_END_ID],
115 + )
116 +
117 + assert len(result) == 2
118 + assert isinstance(result[0], TextChunk)
119 + assert result[0].text == "holdback text"
120 + assert isinstance(result[1], PreLexedTerminal)
121 + assert result[1].terminal == "THINK_END"
122 +
123 + def test_no_holdback_text(self, scanner):
124 + """delta_text is exactly the special token text — no hold-back."""
125 + result = scanner.scan(
126 + delta_text="<channel|>",
127 + delta_token_ids=[CHANNEL_END_ID],
128 + )
129 +
130 + assert len(result) == 1
131 + assert isinstance(result[0], PreLexedTerminal)
132 + assert result[0].terminal == "THINK_END"
133 +
134 + def test_empty_delta_text(self, scanner):
135 + """delta_text is empty — terminal deferred until text arrives."""
136 + result = scanner.scan(
137 + delta_text="",
138 + delta_token_ids=[CHANNEL_END_ID],
139 + )
140 +
141 + assert len(result) == 0
142 +
143 + flushed = scanner.flush_pending()
144 + assert len(flushed) == 1
145 + assert isinstance(flushed[0], PreLexedTerminal)
146 + assert flushed[0].terminal == "THINK_END"
147 +
148 + def test_empty_delta_text_drops_individual_decode_text(self, tokenizer):
149 + """delta_text="" with multiple tokens including special: all
150 + results deferred — individually-decoded TextChunks are unreliable
151 + and PreLexedTerminals wait for text confirmation."""
152 + tool_start_id = 400
153 + tok_a = 201
154 + tok_b = 202
155 + tokenizer.decode.side_effect = lambda ids: {
156 + tool_start_id: "<|tool_call>",
157 + tok_a: "call:",
158 + tok_b: "get_weather",
159 + }.get(ids[0], "?")
160 +
161 + scanner = TokenIDScanner(
162 + token_id_to_terminal={tool_start_id: "TOOL_START"},
163 + tokenizer=tokenizer,
164 + )
165 +
166 + result = scanner.scan(
167 + delta_text="",
168 + delta_token_ids=[tool_start_id, tok_a, tok_b],
169 + )
170 +
171 + assert len(result) == 0
172 +
173 + flushed = scanner.flush_pending()
174 + assert len(flushed) == 1
175 + assert isinstance(flushed[0], PreLexedTerminal)
176 + assert flushed[0].terminal == "TOOL_START"
177 +
178 + def test_holdback_before_start_tag(self, scanner):
179 + """Hold-back text before a reasoning start tag."""
180 + result = scanner.scan(
181 + delta_text="prefix text<|channel>",
182 + delta_token_ids=[CHANNEL_START_ID],
183 + )
184 +
185 + assert len(result) == 2
186 + assert isinstance(result[0], TextChunk)
187 + assert result[0].text == "prefix text"
188 + assert isinstance(result[1], PreLexedTerminal)
189 + assert result[1].terminal == "THINK_START"
190 +
191 + def test_multi_token_batch_special_in_middle(self, scanner, tokenizer):
192 + """Stream-interval > 1: batch has regular tokens + special token.
193 + delta_text differs from individual decodes (context-dependent)."""
194 + tok_a = 201
195 + tok_b = 202
196 + tokenizer.decode.side_effect = lambda ids: {
197 + tok_a: "wordA",
198 + tok_b: "wordB",
199 + CHANNEL_END_ID: CHANNEL_END,
200 + }.get(ids[0], "?")
201 +
202 + scanner_multi = TokenIDScanner(
203 + token_id_to_terminal={CHANNEL_END_ID: "THINK_END"},
204 + tokenizer=tokenizer,
205 + )
206 +
207 + result = scanner_multi.scan(
208 + delta_text="holdback wordA<channel|> wordB",
209 + delta_token_ids=[tok_a, CHANNEL_END_ID, tok_b],
210 + )
211 +
212 + texts = [r.text for r in result if isinstance(r, TextChunk)]
213 + terminals = [r.terminal for r in result if isinstance(r, PreLexedTerminal)]
214 + assert "THINK_END" in terminals
215 + assert "holdback wordA" in "".join(texts)
216 +
217 + def test_multi_token_batch_special_token_text_absent(self, scanner, tokenizer):
218 + """Stream-interval > 1: batch has regular + special token, but
219 + delta_text doesn't contain the special token text at all
220 + (held back by detokenizer along with trailing regular tokens).
221 + Terminal is deferred until text arrives."""
222 + tok_a = 201
223 + tok_b = 202
224 + tokenizer.decode.side_effect = lambda ids: {
225 + tok_a: "alpha",
226 + tok_b: "beta",
227 + CHANNEL_END_ID: CHANNEL_END,
228 + }.get(ids[0], "?")
229 +
230 + scanner_multi = TokenIDScanner(
231 + token_id_to_terminal={CHANNEL_END_ID: "THINK_END"},
232 + tokenizer=tokenizer,
233 + )
234 +
235 + result = scanner_multi.scan(
236 + delta_text="holdback alpha",
237 + delta_token_ids=[tok_a, CHANNEL_END_ID, tok_b],
238 + )
239 +
240 + assert len(result) == 0
241 +
242 + # Next delta: terminal text arrives (detokenizer flushes).
243 + # Deferred terminal resolves with holdback text before it.
244 + result2 = scanner_multi.scan(
245 + delta_text="<channel|> more text",
246 + delta_token_ids=[300],
247 + )
248 + pre_lexed = [r for r in result2 if isinstance(r, PreLexedTerminal)]
249 + assert len(pre_lexed) == 1
250 + assert pre_lexed[0].terminal == "THINK_END"
251 + text_chunks = [r for r in result2 if isinstance(r, TextChunk)]
252 + combined = "".join(t.text for t in text_chunks)
253 + assert "holdback alpha" in combined
254 + assert "more text" in combined
255 +
256 + def test_holdback_with_content_after_special_token(self, tokenizer):
257 + """delta_text has hold-back + special token + content after,
258 + with corresponding token IDs for all parts."""
259 + tok_content = 210
260 + tokenizer.decode.side_effect = lambda ids: {
261 + CHANNEL_END_ID: CHANNEL_END,
262 + tok_content: "content start",
263 + }.get(ids[0], "?")
264 +
265 + scanner = TokenIDScanner(
266 + token_id_to_terminal={CHANNEL_END_ID: "THINK_END"},
267 + tokenizer=tokenizer,
268 + )
269 +
270 + result = scanner.scan(
271 + delta_text="reasoning end.<channel|>content start",
272 + delta_token_ids=[CHANNEL_END_ID, tok_content],
273 + )
274 +
275 + pre_lexed = [r for r in result if isinstance(r, PreLexedTerminal)]
276 + assert len(pre_lexed) == 1
277 + assert pre_lexed[0].terminal == "THINK_END"
278 +
279 + text_chunks = [r for r in result if isinstance(r, TextChunk)]
280 + combined = "".join(t.text for t in text_chunks)
281 + assert "reasoning end." in combined
282 +
283 +
284 +class TestDropTokens:
285 + def test_drop_token_with_holdback(self, tokenizer):
286 + """Drop tokens stripped from delta_text, hold-back text preserved.
287 + Terminal is deferred when its text is absent from delta_text."""
288 + drop_id = 300
289 + tokenizer.decode.side_effect = lambda ids: {
290 + CHANNEL_END_ID: CHANNEL_END,
291 + drop_id: "<eos>",
292 + }.get(ids[0], "?")
293 +
294 + scanner = TokenIDScanner(
295 + token_id_to_terminal={CHANNEL_END_ID: "THINK_END"},
296 + tokenizer=tokenizer,
297 + drop_token_ids={drop_id},
298 + )
299 +
300 + result = scanner.scan(
301 + delta_text="holdback<eos>",
302 + delta_token_ids=[drop_id, CHANNEL_END_ID],
303 + )
304 +
305 + assert len(result) == 0
306 +
307 + # Terminal text arrives in next delta; deferred terminal resolves.
308 + result2 = scanner.scan(
309 + delta_text="<channel|>content",
310 + delta_token_ids=[20],
311 + )
312 + pre_lexed = [r for r in result2 if isinstance(r, PreLexedTerminal)]
313 + assert len(pre_lexed) == 1
314 + assert pre_lexed[0].terminal == "THINK_END"
315 + texts = [r.text for r in result2 if isinstance(r, TextChunk)]
316 + combined = "".join(texts)
317 + assert "holdback" in combined
318 + assert "<eos>" not in combined
319 +
320 + assert len(scanner.flush_pending()) == 0
321 +
322 +
323 +class TestEndToEndReasoningHoldback:
324 + """End-to-end tests through the full parser engine simulating
325 + stream-interval > 1 and detokenizer hold-back, using
326 + gemma4_config."""
327 +
328 + def test_reasoning_content_not_truncated(self):
329 + from vllm.parser.engine.parser_engine_config import (
330 + ParserEngineConfig,
331 + ParserState,
332 + Transition,
333 + )
334 + from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
335 +
336 + config = ParserEngineConfig(
337 + name="test_channel",
338 + initial_state=ParserState.CONTENT,
339 + terminals={
340 + "THINK_START": CHANNEL_START,
341 + "THINK_END": CHANNEL_END,
342 + },
343 + token_id_terminals={
344 + "THINK_START": CHANNEL_START,
345 + "THINK_END": CHANNEL_END,
346 + },
347 + transitions={
348 + (ParserState.CONTENT, "THINK_START"): Transition(
349 + ParserState.REASONING,
350 + (EventType.REASONING_START,),
351 + ),
352 + (ParserState.REASONING, "THINK_END"): Transition(
353 + ParserState.CONTENT,
354 + (EventType.REASONING_END,),
355 + ),
356 + },
357 + )
358 + tok = MagicMock()
359 + vocab = {
360 + CHANNEL_START: CHANNEL_START_ID,
361 + CHANNEL_END: CHANNEL_END_ID,
362 + }
363 + tok.get_vocab.return_value = vocab
364 + tok.decode.side_effect = lambda ids: {
365 + CHANNEL_START_ID: CHANNEL_START,
366 + CHANNEL_END_ID: CHANNEL_END,
367 + }.get(ids[0], f"tok{ids[0]}")
368 +
369 + engine = StreamingParserEngine(config, tok)
370 + all_events = []
371 +
372 + # Delta 1: channel start token (text includes start tag)
373 + all_events.extend(engine.feed(CHANNEL_START, [CHANNEL_START_ID]))
374 +
375 + # Delta 2: reasoning text (normal content, no special tokens)
376 + all_events.extend(
377 + engine.feed(
378 + "thought\nThe request was received and ",
379 + [10, 11, 12, 13, 14],
380 + )
381 + )
382 +
383 + # Delta 3: MORE reasoning text, the detokenizer held some back.
384 + # Then channel end token arrives in token_ids, but its text
385 + # is NOT in delta_text (held back by detokenizer).
386 + # delta_text = previously held-back reasoning text only.
387 + all_events.extend(
388 + engine.feed(
389 + "processed is appropriate.",
390 + [CHANNEL_END_ID],
391 + )
392 + )
393 +
394 + # Delta 4: detokenizer flushes held-back channel end text
395 + # plus new content tokens.
396 + all_events.extend(
397 + engine.feed(
398 + "<channel|>Understood.",
399 + [20, 21],
400 + )
401 + )
402 +
403 + all_events.extend(engine.finish())
404 +
405 + reasoning_text = "".join(
406 + e.value for e in all_events if e.type == EventType.REASONING_CHUNK
407 + )
408 + content_text = "".join(
409 + e.value for e in all_events if e.type == EventType.TEXT_CHUNK
410 + )
411 +
412 + assert "processed is appropriate." in reasoning_text
413 + assert "Understood." in content_text
414 +
415 + def test_backtick_content_not_truncated(self):
416 + """Reproduces the hostname backtick truncation case."""
417 + from vllm.parser.engine.parser_engine_config import (
418 + ParserEngineConfig,
419 + ParserState,
420 + Transition,
421 + )
422 + from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
423 +
424 + config = ParserEngineConfig(
425 + name="test_channel",
426 + initial_state=ParserState.CONTENT,
427 + terminals={
428 + "THINK_START": CHANNEL_START,
429 + "THINK_END": CHANNEL_END,
430 + },
431 + token_id_terminals={
432 + "THINK_START": CHANNEL_START,
433 + "THINK_END": CHANNEL_END,
434 + },
435 + transitions={
436 + (ParserState.CONTENT, "THINK_START"): Transition(
437 + ParserState.REASONING,
438 + (EventType.REASONING_START,),
439 + ),
440 + (ParserState.REASONING, "THINK_END"): Transition(
441 + ParserState.CONTENT,
442 + (EventType.REASONING_END,),
443 + ),
444 + },
445 + )
446 + tok = MagicMock()
447 + vocab = {
448 + CHANNEL_START: CHANNEL_START_ID,
449 + CHANNEL_END: CHANNEL_END_ID,
450 + }
451 + tok.get_vocab.return_value = vocab
452 + tok.decode.side_effect = lambda ids: {
453 + CHANNEL_START_ID: CHANNEL_START,
454 + CHANNEL_END_ID: CHANNEL_END,
455 + }.get(ids[0], f"tok{ids[0]}")
456 +
457 + engine = StreamingParserEngine(config, tok)
458 + all_events = []
459 +
460 + all_events.extend(engine.feed(CHANNEL_START, [CHANNEL_START_ID]))
461 + all_events.extend(
462 + engine.feed(
463 + "thought\n1/10 completed. Next: ",
464 + [10, 11, 12, 13],
465 + )
466 + )
467 +
468 + # Hold-back text includes backtick content; channel end text
469 + # absent from delta_text.
470 + all_events.extend(
471 + engine.feed(
472 + "`hostname`.\n",
473 + [CHANNEL_END_ID],
474 + )
475 + )
476 +
477 + # Next delta flushes channel end + tool call start
478 + all_events.extend(
479 + engine.feed(
480 + "<channel|>tool output",
481 + [20, 21],
482 + )
483 + )
484 +
485 + all_events.extend(engine.finish())
486 +
487 + reasoning_text = "".join(
488 + e.value for e in all_events if e.type == EventType.REASONING_CHUNK
489 + )
490 +
491 + assert "`hostname`." in reasoning_text
492 +
493 +
494 +class TestRebuildFromAnchorsLiteralLookalike:
495 + """When delta_text contains a literal mention of a special token's
496 + text before the real special token, _rebuild_from_anchors must
497 + anchor at the real occurrence, not the literal one."""
498 +
499 + @pytest.fixture
500 + def tool_scanner(self):
501 + tok = MagicMock()
502 + tok.get_vocab.return_value = {
503 + TOOL_START: TOOL_START_ID,
504 + TOOL_END: TOOL_END_ID,
505 + }
506 + tok.decode.side_effect = lambda ids: {
507 + TOOL_START_ID: TOOL_START,
508 + TOOL_END_ID: TOOL_END,
509 + }.get(ids[0], f"t{ids[0]}")
510 + return TokenIDScanner(
511 + {TOOL_START_ID: "TOOL_START", TOOL_END_ID: "TOOL_END"},
512 + tok,
513 + )
514 +
515 + def test_literal_before_real_anchor(self, tool_scanner):
516 + """Literal <tool_call> in prose followed by a real <tool_call>
517 + special token — the scanner must split at the real one."""
518 + delta_text = 'Use <tool_call> like this: <tool_call>{"name":"f"}</tool_call>'
519 + delta_token_ids = [1, 2, 3, 4, 5, TOOL_START_ID, 6, 7, TOOL_END_ID]
520 + items = tool_scanner.scan(delta_text, delta_token_ids)
521 +
522 + text_parts = [it.text for it in items if isinstance(it, TextChunk)]
523 + terminals = [it for it in items if isinstance(it, PreLexedTerminal)]
524 +
525 + assert len(terminals) == 2
526 + assert terminals[0].terminal == "TOOL_START"
527 + assert terminals[1].terminal == "TOOL_END"
528 +
529 + # The literal mention must appear in a text chunk, not be
530 + # consumed by the TOOL_START anchor.
531 + joined_text = "".join(text_parts)
532 + assert "<tool_call>" in joined_text
533 + assert '{"name":"f"}' in joined_text
534 +
535 + def test_multiple_tool_calls_with_literal_between(self, tool_scanner):
536 + """Two real tool calls with a literal mention between them."""
537 + delta_text = (
538 + '<tool_call>{"name":"a"}</tool_call>'
539 + " see <tool_call> syntax "
540 + '<tool_call>{"name":"b"}</tool_call>'
541 + )
542 + delta_token_ids = [
543 + TOOL_START_ID,
544 + 1,
545 + TOOL_END_ID,
546 + 2,
547 + 3,
548 + 4,
549 + TOOL_START_ID,
550 + 5,
551 + TOOL_END_ID,
552 + ]
553 + items = tool_scanner.scan(delta_text, delta_token_ids)
554 +
555 + terminals = [it for it in items if isinstance(it, PreLexedTerminal)]
556 + assert len(terminals) == 4
557 +
558 + text_parts = [it.text for it in items if isinstance(it, TextChunk)]
559 + joined_text = "".join(text_parts)
560 + # The literal mention between the two real calls must be in text
561 + assert "<tool_call> syntax" in joined_text
562 +
563 +
564 +class TestRebuildFromAnchorsCascadingDeferral:
565 + """When a middle anchor's text is absent from delta_text,
566 + only that anchor should be deferred — not subsequent ones
567 + with valid positions."""
568 +
569 + @pytest.fixture
570 + def bare_scanner(self):
571 + tok = MagicMock()
572 + tok.decode.side_effect = lambda ids: f"t{ids[0]}"
573 + return TokenIDScanner({}, tok)
574 +
575 + def test_middle_anchor_missing_does_not_cascade(self, bare_scanner):
576 + a = PreLexedTerminal("TOOL_START", TOOL_START_ID, TOOL_START)
577 + b = PreLexedTerminal("THINK_END", CHANNEL_END_ID, CHANNEL_END)
578 + c = PreLexedTerminal("TOOL_END", TOOL_END_ID, TOOL_END)
579 + delta_text = f"prefix{TOOL_START}middle{TOOL_END}suffix"
580 + results = [a, b, c]
581 +
582 + rebuilt = bare_scanner._rebuild_from_anchors(delta_text, results)
583 +
584 + terminals = [r for r in rebuilt if isinstance(r, PreLexedTerminal)]
585 + texts = [r for r in rebuilt if isinstance(r, TextChunk)]
586 + joined = "".join(t.text for t in texts)
587 +
588 + assert len(terminals) == 2
589 + assert terminals[0].terminal == "TOOL_START"
590 + assert terminals[1].terminal == "TOOL_END"
591 + assert "prefix" in joined
592 + assert "middle" in joined
593 + assert "suffix" in joined
594 + assert len(bare_scanner._deferred_terminals) == 1
595 + assert bare_scanner._deferred_terminals[0].terminal == "THINK_END"
596 + assert bare_scanner._deferred_post_text == ""
597 +
598 + def test_first_anchor_missing_rest_still_emitted(self, bare_scanner):
599 + a = PreLexedTerminal("THINK_END", CHANNEL_END_ID, CHANNEL_END)
600 + b = PreLexedTerminal("TOOL_START", TOOL_START_ID, TOOL_START)
601 + delta_text = f"text{TOOL_START}more"
602 + results = [a, b]
603 +
604 + rebuilt = bare_scanner._rebuild_from_anchors(delta_text, results)
605 +
606 + terminals = [r for r in rebuilt if isinstance(r, PreLexedTerminal)]
607 + assert len(terminals) == 1
608 + assert terminals[0].terminal == "TOOL_START"
609 + assert len(bare_scanner._deferred_terminals) == 1
610 + assert bare_scanner._deferred_terminals[0].terminal == "THINK_END"
611 +
612 + def test_last_anchor_missing_preceding_still_emitted(self, bare_scanner):
613 + a = PreLexedTerminal("TOOL_START", TOOL_START_ID, TOOL_START)
614 + b = PreLexedTerminal("THINK_END", CHANNEL_END_ID, CHANNEL_END)
615 + delta_text = f"text{TOOL_START}more"
616 + results = [a, b]
617 +
618 + rebuilt = bare_scanner._rebuild_from_anchors(delta_text, results)
619 +
620 + terminals = [r for r in rebuilt if isinstance(r, PreLexedTerminal)]
621 + assert len(terminals) == 1
622 + assert terminals[0].terminal == "TOOL_START"
623 + texts = [r for r in rebuilt if isinstance(r, TextChunk)]
624 + joined = "".join(t.text for t in texts)
625 + assert "text" in joined
626 + # "more" is deferred along with the missing terminal —
627 + # it will be resolved in the next scan when the terminal
628 + # text arrives.
629 + assert bare_scanner._deferred_post_text == "more"
630 + assert len(bare_scanner._deferred_terminals) == 1
631 + assert bare_scanner._deferred_terminals[0].terminal == "THINK_END"
tests/parser/engine/trace_builder.py new
+410
@@ -0,0 +1,410 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""On-demand trace builder for parser engine testing and benchmarks.
4 +
5 +Generates token sequences programmatically from model-agnostic scenario
6 +definitions. Each model format handler knows how to render scenarios
7 +into the model's output format, tokenize them with correct special token
8 +IDs, and compute expected parse outputs.
9 +
10 +Every generated sample is self-validated by replaying it through the
11 +real parser before being returned.
12 +"""
13 +
14 +from __future__ import annotations
15 +
16 +import functools
17 +import json
18 +from dataclasses import dataclass
19 +from typing import Any
20 +
21 +from tests.parser.engine.replay_harness import (
22 + MockTokenizer,
23 + Sample,
24 + assert_parse_output,
25 + collect_output,
26 + replay_streaming,
27 +)
28 +from vllm.entrypoints.openai.chat_completion.protocol import (
29 + ChatCompletionToolsParam,
30 +)
31 +from vllm.parser.engine.registered_adapters import (
32 + Qwen3Parser,
33 +)
34 +
35 +# ── Data structures ──────────────────────────────────────────────────
36 +
37 +
38 +@dataclass
39 +class ToolCallSpec:
40 + name: str
41 + arguments: dict[str, Any]
42 +
43 +
44 +@dataclass
45 +class Scenario:
46 + id: str
47 + description: str
48 + reasoning: str | None = None
49 + content: str | None = None
50 + tool_calls: list[ToolCallSpec] | None = None
51 +
52 +
53 +# ── Scenarios ────────────────────────────────────────────────────────
54 +
55 +_READ_TOOL = ToolCallSpec("read_file", {"path": "/tmp/test.txt"})
56 +_BASH_TOOL = ToolCallSpec(
57 + "bash", {"command": "hostname", "description": "Get hostname"}
58 +)
59 +_WEATHER_TOOL = ToolCallSpec(
60 + "get_weather",
61 + {"city": "Dallas", "state": "TX", "unit": "fahrenheit"},
62 +)
63 +_COMPLEX_TOOL = ToolCallSpec(
64 + "search",
65 + {
66 + "query": "vllm parser",
67 + "filters": {"language": "python", "min_stars": 100},
68 + "tags": ["ml", "inference"],
69 + "limit": 10,
70 + "verbose": True,
71 + },
72 +)
73 +
74 +SCENARIOS: list[Scenario] = [
75 + Scenario(
76 + id="think-then-tool",
77 + description="Reasoning then single tool call",
78 + reasoning="Let me check the file.",
79 + tool_calls=[_READ_TOOL],
80 + ),
81 + Scenario(
82 + id="think-then-parallel-tools",
83 + description="Reasoning then two parallel tool calls",
84 + reasoning="I need to run both commands.",
85 + tool_calls=[_BASH_TOOL, _WEATHER_TOOL],
86 + ),
87 + Scenario(
88 + id="think-then-content",
89 + description="Reasoning then content response",
90 + reasoning="Let me think about this carefully.",
91 + content="The answer is 42.",
92 + ),
93 + Scenario(
94 + id="content-only",
95 + description="Plain content response without reasoning",
96 + content="Hello! How can I help you today?",
97 + ),
98 + Scenario(
99 + id="tool-only",
100 + description="Tool call without reasoning",
101 + tool_calls=[_READ_TOOL],
102 + ),
103 + Scenario(
104 + id="complex-json-args",
105 + description="Tool call with nested objects, arrays, numbers, booleans",
106 + reasoning="This needs a complex query.",
107 + tool_calls=[_COMPLEX_TOOL],
108 + ),
109 + Scenario(
110 + id="whitespace-before-tool",
111 + description="Whitespace-only content before tool call",
112 + content="\n\n",
113 + tool_calls=[_WEATHER_TOOL],
114 + ),
115 + Scenario(
116 + id="think-content-tool",
117 + description="Reasoning, content, then tool call",
118 + reasoning="Let me analyze and then fetch data.",
119 + content="Checking the weather now.",
120 + tool_calls=[_WEATHER_TOOL],
121 + ),
122 + Scenario(
123 + id="think-whitespace-tool",
124 + description="Reasoning, whitespace-only gap, then tool call",
125 + reasoning="Let me check the file contents.",
126 + content="\n\n",
127 + tool_calls=[_READ_TOOL],
128 + ),
129 + Scenario(
130 + id="empty-reasoning-content",
131 + description="Empty reasoning section followed by content",
132 + reasoning="",
133 + content="The epoch timestamp is 1779111346.",
134 + ),
135 +]
136 +
137 +
138 +# ── Tokenization ─────────────────────────────────────────────────────
139 +
140 +
141 +def _word_split(text: str) -> list[str]:
142 + """Split text into word-like tokens, preserving all characters."""
143 + if not text:
144 + return []
145 + parts: list[str] = []
146 + current = ""
147 + for ch in text:
148 + if ch in " \t\n\r" and current and current[-1] not in " \t\n\r":
149 + parts.append(current)
150 + current = ch
151 + else:
152 + current += ch
153 + if current:
154 + parts.append(current)
155 + return parts
156 +
157 +
158 +def _tokenize(
159 + segments: list[tuple[str, bool]],
160 + vocab: dict[str, int],
161 + start_id: int = 100,
162 +) -> list[tuple[int, str]]:
163 + """Build token list from segments.
164 +
165 + Each segment is ``(text, is_special)``. Special segments use vocab
166 + IDs; content segments are word-split with sequential IDs.
167 + """
168 + tokens: list[tuple[int, str]] = []
169 + next_id = start_id
170 +
171 + for text, is_special in segments:
172 + if not text:
173 + continue
174 + if is_special:
175 + tid = vocab.get(text)
176 + if tid is None:
177 + raise ValueError(f"Special token {text!r} not in vocab")
178 + tokens.append((tid, text))
179 + else:
180 + for word in _word_split(text):
181 + tokens.append((next_id, word))
182 + next_id += 1
183 +
184 + return tokens
185 +
186 +
187 +# ── Tool definitions ─────────────────────────────────────────────────
188 +
189 +
190 +def _infer_schema(value: object) -> dict:
191 + """Infer a JSON Schema from a Python value, recursing into dicts/lists."""
192 + if isinstance(value, bool):
193 + return {"type": "boolean"}
194 + if isinstance(value, int):
195 + return {"type": "integer"}
196 + if isinstance(value, float):
197 + return {"type": "number"}
198 + if isinstance(value, str):
199 + return {"type": "string"}
200 + if isinstance(value, dict):
201 + return {
202 + "type": "object",
203 + "properties": {k: _infer_schema(v) for k, v in value.items()},
204 + }
205 + if isinstance(value, list) and value:
206 + return {"type": "array", "items": _infer_schema(value[0])}
207 + if isinstance(value, list):
208 + return {"type": "array"}
209 + return {}
210 +
211 +
212 +def _tool_defs(tool_calls: list[ToolCallSpec]) -> list[dict]:
213 + """Generate OpenAI-style tool definitions from tool call specs."""
214 + seen: set[str] = set()
215 + tools: list[dict] = []
216 + for tc in tool_calls:
217 + if tc.name in seen:
218 + continue
219 + seen.add(tc.name)
220 + properties = {k: _infer_schema(v) for k, v in tc.arguments.items()}
221 + tools.append(
222 + {
223 + "type": "function",
224 + "function": {
225 + "name": tc.name,
226 + "parameters": {
227 + "type": "object",
228 + "properties": properties,
229 + },
230 + },
231 + }
232 + )
233 + return tools
234 +
235 +
236 +# ── Format handlers ──────────────────────────────────────────────────
237 +
238 +
239 +def _expected_tc(scenario: Scenario) -> list[dict] | None:
240 + if not scenario.tool_calls:
241 + return None
242 + return [{"name": tc.name, "arguments": tc.arguments} for tc in scenario.tool_calls]
243 +
244 +
245 +def _expected_tools(scenario: Scenario) -> list[dict] | None:
246 + return _tool_defs(scenario.tool_calls) if scenario.tool_calls else None
247 +
248 +
249 +def _validate_sample(sample: Sample, parser_cls: type, **kwargs) -> None:
250 + """Replay sample through the real parser and assert correctness."""
251 + tokenizer = MockTokenizer(vocab=dict(sample.vocab), tokens=sample.tokens)
252 + parser = parser_cls(tokenizer, sample.tools, **kwargs)
253 + deltas = replay_streaming(parser, sample.tokens, chunk_size=1, tools=sample.tools)
254 + output = collect_output(deltas)
255 + assert_parse_output(output, sample)
256 +
257 +
258 +def _validate_tools(
259 + tools: list[dict] | None,
260 +) -> list[ChatCompletionToolsParam] | None:
261 + if not tools:
262 + return None
263 + return [ChatCompletionToolsParam.model_validate(t) for t in tools]
264 +
265 +
266 +def _make_sample(
267 + sample_id: str,
268 + description: str,
269 + vocab: dict[str, int],
270 + segments: list[tuple[str, bool]],
271 + expected_reasoning: str | None,
272 + expected_content: str | None,
273 + expected_tool_calls: list[dict] | None,
274 + tools: list[dict] | None,
275 + chat_template_kwargs: dict | None = None,
276 +) -> Sample:
277 + tokens = _tokenize(segments, vocab)
278 + return Sample(
279 + id=sample_id,
280 + description=description,
281 + source="trace-builder",
282 + vocab=dict(vocab),
283 + tokens=tokens,
284 + expected_reasoning=expected_reasoning,
285 + expected_content=expected_content,
286 + expected_tool_calls=expected_tool_calls,
287 + tools=_validate_tools(tools),
288 + chat_template_kwargs=chat_template_kwargs,
289 + )
290 +
291 +
292 +# ── Qwen3 / NemotronV3 (XML tool format, starts in REASONING) ───────
293 +
294 +_QWEN3_VOCAB: dict[str, int] = {
295 + "<think>": 50,
296 + "</think>": 51,
297 + "<tool_call>": 60,
298 + "</tool_call>": 61,
299 +}
300 +
301 +
302 +def _qwen3_arg_value(value: Any) -> str:
303 + if isinstance(value, bool):
304 + return "true" if value else "false"
305 + if isinstance(value, (int, float)):
306 + return str(value)
307 + if isinstance(value, str):
308 + return value
309 + return json.dumps(value, ensure_ascii=False)
310 +
311 +
312 +def _qwen3_tool_segments(tc: ToolCallSpec) -> list[tuple[str, bool]]:
313 + parts = [f"\n<function={tc.name}>"]
314 + for key, value in tc.arguments.items():
315 + parts.append(f"\n<parameter={key}>{_qwen3_arg_value(value)}</parameter>")
316 + parts.append("\n</function>\n")
317 + return [
318 + ("<tool_call>", True),
319 + ("".join(parts), False),
320 + ("</tool_call>", True),
321 + ]
322 +
323 +
324 +def _qwen3_segments(scenario: Scenario) -> list[tuple[str, bool]]:
325 + segs: list[tuple[str, bool]] = []
326 + if scenario.reasoning is not None:
327 + segs.append((scenario.reasoning, False))
328 + if scenario.content is not None or scenario.tool_calls:
329 + segs.append(("</think>", True))
330 + if scenario.content is not None:
331 + segs.append((scenario.content, False))
332 + if scenario.tool_calls:
333 + for tc in scenario.tool_calls:
334 + segs.extend(_qwen3_tool_segments(tc))
335 + return segs
336 +
337 +
338 +def _qwen3_expected_content(scenario: Scenario) -> str | None:
339 + if (
340 + scenario.content is not None
341 + and scenario.tool_calls
342 + and not scenario.content.strip()
343 + ):
344 + return ""
345 + return scenario.content
346 +
347 +
348 +def _build_qwen3(
349 + scenario: Scenario,
350 + name: str = "qwen3",
351 + parser_cls: type = Qwen3Parser,
352 + strip_trailing_ws: bool = False,
353 + validate: bool = True,
354 +) -> Sample:
355 + expected_reasoning: str | None
356 + if scenario.reasoning is not None:
357 + r = scenario.reasoning
358 + if strip_trailing_ws:
359 + r = r.rstrip()
360 + expected_reasoning = r
361 + else:
362 + expected_reasoning = ""
363 +
364 + sample = _make_sample(
365 + sample_id=f"{name}-{scenario.id}",
366 + description=scenario.description,
367 + vocab=_QWEN3_VOCAB,
368 + segments=_qwen3_segments(scenario),
369 + expected_reasoning=expected_reasoning,
370 + expected_content=_qwen3_expected_content(scenario),
371 + expected_tool_calls=_expected_tc(scenario),
372 + tools=_expected_tools(scenario),
373 + )
374 + if validate:
375 + _validate_sample(sample, parser_cls)
376 + return sample
377 +
378 +
379 +# ── Registry and public API ──────────────────────────────────────────
380 +
381 +_BUILDERS: dict[str, Any] = {
382 + "qwen3": _build_qwen3,
383 +}
384 +
385 +
386 +@functools.cache
387 +def build_samples(model: str) -> tuple[Sample, ...]:
388 + """Build all scenario samples for a model, self-validated."""
389 + builder = _BUILDERS[model]
390 + return tuple(builder(s) for s in SCENARIOS)
391 +
392 +
393 +def build_sample(model: str, scenario: Scenario) -> Sample:
394 + """Build a single sample for one model + scenario."""
395 + return _BUILDERS[model](scenario)
396 +
397 +
398 +def build_scaling_sample(
399 + model: str, token_count: int, validate: bool = False
400 +) -> Sample:
401 + """Build a sample with approximately *token_count* tokens."""
402 + sentence = "The quick brown fox jumps over the lazy dog. "
403 + text = sentence * (token_count // 10 + 1)
404 + scenario = Scenario(
405 + id=f"scaling-{token_count}",
406 + description=f"Scaling test with ~{token_count} tokens",
407 + reasoning=text,
408 + tool_calls=[_READ_TOOL],
409 + )
410 + return _BUILDERS[model](scenario, validate=validate)
tests/tool_parsers/test_qwen3coder_tool_parser.py
+115 -43
@@ -23,8 +23,8 @@ from vllm.entrypoints.openai.engine.protocol import (
23 from vllm.parser.abstract_parser import DelegatingParser
24 from vllm.tokenizers import TokenizerLike, get_tokenizer
25 from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
26 -from vllm.tool_parsers.qwen3coder_tool_parser import (
27 - Qwen3CoderToolParser,
26 +from vllm.tool_parsers.qwen3_engine_tool_parser import (
27 + Qwen3EngineToolParser,
28 )
29
30 MODEL = "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8"
@@ -37,12 +37,7 @@ def qwen3_tokenizer():
37
38 @pytest.fixture
39 def qwen3_tool_parser(qwen3_tokenizer, sample_tools):
40 - return Qwen3CoderToolParser(qwen3_tokenizer, tools=sample_tools)
41 -
42 -
43 -@pytest.fixture
44 -def qwen3_tool_parser_parametrized(qwen3_tool_parser):
45 - return qwen3_tool_parser
40 + return Qwen3EngineToolParser(qwen3_tokenizer, tools=sample_tools)
41
42
43 WEATHER_PARAMS = {
@@ -208,9 +203,9 @@ def stream_delta_message_generator(
203 read_offset = new_read_offset
204
205
211 -def test_extract_tool_calls_no_tools(qwen3_tool_parser_parametrized):
206 +def test_extract_tool_calls_no_tools(qwen3_tool_parser):
207 model_output = "This is a test response without any tool calls"
213 - extracted_tool_calls = qwen3_tool_parser_parametrized.extract_tool_calls(
208 + extracted_tool_calls = qwen3_tool_parser.extract_tool_calls(
209 model_output, request=None
210 ) # type: ignore[arg-type]
211 assert not extracted_tool_calls.tools_called
@@ -391,13 +386,13 @@ circle
386 ],
387 )
388 def test_extract_tool_calls(
394 - qwen3_tool_parser_parametrized,
389 + qwen3_tool_parser,
390 model_output,
391 expected_tool_calls,
392 expected_content,
393 ):
394 request = ChatCompletionRequest(model=MODEL, messages=[])
400 - extracted_tool_calls = qwen3_tool_parser_parametrized.extract_tool_calls(
395 + extracted_tool_calls = qwen3_tool_parser.extract_tool_calls(
396 model_output, request=request
397 )
398 assert extracted_tool_calls.tools_called
@@ -408,7 +403,7 @@ def test_extract_tool_calls(
403
404
405 def test_extract_tool_calls_fallback_no_tags(
411 - qwen3_tool_parser_parametrized,
406 + qwen3_tool_parser,
407 ):
408 """Test fallback parsing when XML tags are missing"""
409 model_output = """<function=get_current_weather>
@@ -421,7 +416,7 @@ TX
416 </function>"""
417
418 request = ChatCompletionRequest(model=MODEL, messages=[])
424 - extracted_tool_calls = qwen3_tool_parser_parametrized.extract_tool_calls(
419 + extracted_tool_calls = qwen3_tool_parser.extract_tool_calls(
420 model_output, request=request
421 )
422
@@ -471,7 +466,7 @@ hello world
466 </function>
467 </tool_call>"""
468
474 - parser = Qwen3CoderToolParser(qwen3_tokenizer, tools=tools)
469 + parser = Qwen3EngineToolParser(qwen3_tokenizer, tools=tools)
470 request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
471 extracted_tool_calls = parser.extract_tool_calls(model_output, request=request)
472
@@ -563,7 +558,7 @@ some text
558 </function>
559 </tool_call>"""
560
566 - parser = Qwen3CoderToolParser(qwen3_tokenizer, tools=tools)
561 + parser = Qwen3EngineToolParser(qwen3_tokenizer, tools=tools)
562 request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
563 extracted = parser.extract_tool_calls(model_output, request=request)
564
@@ -637,7 +632,7 @@ true
632 </function>
633 </tool_call>"""
634
640 - parser = Qwen3CoderToolParser(qwen3_tokenizer, tools=tools)
635 + parser = Qwen3EngineToolParser(qwen3_tokenizer, tools=tools)
636 request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
637
638 tool_states = {}
@@ -843,7 +838,7 @@ circle
838 ],
839 )
840 def test_extract_tool_calls_streaming(
846 - qwen3_tool_parser_parametrized,
841 + qwen3_tool_parser,
842 qwen3_tokenizer,
843 model_output,
844 expected_tool_calls,
@@ -856,7 +851,7 @@ def test_extract_tool_calls_streaming(
851 tool_states = {} # Track state per tool index
852
853 for delta_message in stream_delta_message_generator(
859 - qwen3_tool_parser_parametrized, qwen3_tokenizer, model_output, request
854 + qwen3_tool_parser, qwen3_tokenizer, model_output, request
855 ):
856 # role should never be streamed from tool parser
857 assert not delta_message.role
@@ -900,9 +895,6 @@ def test_extract_tool_calls_streaming(
895
896 # Verify we got all expected tool calls
897 assert len(tool_states) == len(expected_tool_calls)
903 - assert len(qwen3_tool_parser_parametrized.prev_tool_call_arr) == len(
904 - expected_tool_calls
905 - )
898
899 # Verify each tool call
900 for idx, expected_tool in enumerate(expected_tool_calls):
@@ -920,7 +912,7 @@ def test_extract_tool_calls_streaming(
912
913
914 def test_extract_tool_calls_missing_closing_parameter_tag(
923 - qwen3_tool_parser_parametrized,
915 + qwen3_tool_parser,
916 ):
917 """Test handling of missing closing </parameter> tag"""
918 # Using get_current_weather from sample_tools but with malformed XML
@@ -939,7 +931,7 @@ fahrenheit
931 </tool_call>"""
932
933 request = ChatCompletionRequest(model=MODEL, messages=[])
942 - extracted_tool_calls = qwen3_tool_parser_parametrized.extract_tool_calls(
934 + extracted_tool_calls = qwen3_tool_parser.extract_tool_calls(
935 model_output, request=request
936 )
937
@@ -962,7 +954,7 @@ fahrenheit
954
955
956 def test_extract_tool_calls_streaming_missing_closing_tag(
965 - qwen3_tool_parser_parametrized, qwen3_tokenizer
957 + qwen3_tool_parser, qwen3_tokenizer
958 ):
959 """Test streaming with missing closing </parameter> tag"""
960 # Using get_current_weather from sample_tools but with malformed XML
@@ -986,7 +978,7 @@ fahrenheit
978 tool_states = {}
979
980 for delta_message in stream_delta_message_generator(
989 - qwen3_tool_parser_parametrized, qwen3_tokenizer, model_output, request
981 + qwen3_tool_parser, qwen3_tokenizer, model_output, request
982 ):
983 if delta_message.content:
984 other_content += delta_message.content
@@ -1021,7 +1013,6 @@ fahrenheit
1013 assert "Let me check the weather for you:" in other_content
1014 # Verify we got the tool call
1015 assert len(tool_states) == 1
1024 - assert len(qwen3_tool_parser_parametrized.prev_tool_call_arr) == 1
1016
1017 state = tool_states[0]
1018 assert state["id"] is not None
@@ -1036,9 +1027,7 @@ fahrenheit
1027 assert args["unit"] == "fahrenheit"
1028
1029
1039 -def test_extract_tool_calls_streaming_incremental(
1040 - qwen3_tool_parser_parametrized, qwen3_tokenizer
1041 -):
1030 +def test_extract_tool_calls_streaming_incremental(qwen3_tool_parser, qwen3_tokenizer):
1031 """Test that streaming is truly incremental"""
1032 model_output = """I'll check the weather.<tool_call>
1033 <function=get_current_weather>
@@ -1055,7 +1044,7 @@ TX
1044
1045 chunks = []
1046 for delta_message in stream_delta_message_generator(
1058 - qwen3_tool_parser_parametrized, qwen3_tokenizer, model_output, request
1047 + qwen3_tool_parser, qwen3_tokenizer, model_output, request
1048 ):
1049 chunks.append(delta_message)
1050
@@ -1073,19 +1062,21 @@ TX
1062 header_found = True
1063 assert chunk.tool_calls[0].function.name == "get_current_weather"
1064 assert chunk.tool_calls[0].type == "function"
1076 - # Empty initially
1077 - assert chunk.tool_calls[0].function.arguments == ""
1065 break
1066 assert header_found
1067
1068 # Should have chunks with incremental arguments
1069 arg_chunks = []
1070 for chunk in chunks:
1084 - if chunk.tool_calls and chunk.tool_calls[0].function.arguments:
1071 + if (
1072 + chunk.tool_calls
1073 + and chunk.tool_calls[0].function
1074 + and chunk.tool_calls[0].function.arguments
1075 + ):
1076 arg_chunks.append(chunk.tool_calls[0].function.arguments)
1077
1087 - # Arguments should be streamed incrementally
1088 - assert len(arg_chunks) > 1
1078 + # Arguments should be streamed
1079 + assert len(arg_chunks) >= 1
1080
1081 # Concatenated arguments should form valid JSON
1082 full_args = "".join(arg_chunks)
@@ -1094,6 +1085,85 @@ TX
1085 assert parsed_args["state"] == "TX"
1086
1087
1088 +def test_extract_tool_calls_streaming_missing_opening_tag(
1089 + qwen3_tool_parser, qwen3_tokenizer
1090 +):
1091 + """Test streaming with missing opening <tool_call> tag
1092 +
1093 + This tests that the streaming parser correctly handles
1094 + tool calls that start directly with <function=...>
1095 + """
1096 + model_output = """I'll check the weather for you.
1097 +
1098 +<function=get_current_weather>
1099 +<parameter=city>
1100 +Dallas
1101 +</parameter>
1102 +<parameter=state>
1103 +TX
1104 +</parameter>
1105 +<parameter=unit>
1106 +fahrenheit
1107 +</parameter>
1108 +</function>
1109 +</tool_call>"""
1110 +
1111 + request = ChatCompletionRequest(model=MODEL, messages=[])
1112 +
1113 + other_content = ""
1114 + tool_states = {}
1115 +
1116 + for delta_message in stream_delta_message_generator(
1117 + qwen3_tool_parser, qwen3_tokenizer, model_output, request
1118 + ):
1119 + if delta_message.content:
1120 + other_content += delta_message.content
1121 +
1122 + if delta_message.tool_calls:
1123 + for tool_call in delta_message.tool_calls:
1124 + idx = tool_call.index
1125 +
1126 + if idx not in tool_states:
1127 + tool_states[idx] = {
1128 + "id": None,
1129 + "name": None,
1130 + "arguments": "",
1131 + "type": None,
1132 + }
1133 +
1134 + if tool_call.id:
1135 + tool_states[idx]["id"] = tool_call.id
1136 +
1137 + if tool_call.type:
1138 + assert tool_call.type == "function"
1139 + tool_states[idx]["type"] = tool_call.type
1140 +
1141 + if tool_call.function:
1142 + if tool_call.function.name:
1143 + tool_states[idx]["name"] = tool_call.function.name
1144 +
1145 + if tool_call.function.arguments is not None:
1146 + tool_states[idx]["arguments"] += tool_call.function.arguments
1147 +
1148 + # Verify content was streamed
1149 + assert "I'll check the weather for you." in other_content
1150 +
1151 + # Verify we got the tool call
1152 + assert len(tool_states) == 1
1153 +
1154 + state = tool_states[0]
1155 + assert state["id"] is not None
1156 + assert state["type"] == "function"
1157 + assert state["name"] == "get_current_weather"
1158 +
1159 + # Verify arguments were parsed correctly despite missing opening tag
1160 + assert state["arguments"] is not None
1161 + args = json.loads(state["arguments"])
1162 + assert args["city"] == "Dallas"
1163 + assert args["state"] == "TX"
1164 + assert args["unit"] == "fahrenheit"
1165 +
1166 +
1167 def test_malformed_xml_no_gt_delimiter(qwen3_tool_parser):
1168 """Regression: malformed XML without '>' must not crash (PR #36774)."""
1169 model_output = (
@@ -1130,9 +1200,11 @@ def test_none_tool_calls_filtered(qwen3_tool_parser):
1200 result = qwen3_tool_parser.extract_tool_calls(model_output, request=request)
1201 assert all(tc is not None for tc in result.tool_calls)
1202 assert result.tools_called
1133 - assert len(result.tool_calls) == 1
1134 - assert result.tool_calls[0].function.name == "get_current_weather"
1135 - args = json.loads(result.tool_calls[0].function.arguments)
1203 + valid = [
1204 + tc for tc in result.tool_calls if tc.function.name == "get_current_weather"
1205 + ]
1206 + assert len(valid) == 1
1207 + args = json.loads(valid[0].function.arguments)
1208 assert args["city"] == "Dallas"
1209 assert args["state"] == "TX"
1210
@@ -1156,7 +1228,7 @@ def test_anyof_parameter_not_double_encoded(qwen3_tokenizer):
1228 )
1229 ]
1230
1159 - parser = Qwen3CoderToolParser(qwen3_tokenizer, tools=tools)
1231 + parser = Qwen3EngineToolParser(qwen3_tokenizer, tools=tools)
1232
1233 model_output = (
1234 "<tool_call>\n"
@@ -1247,7 +1319,7 @@ def test_no_double_serialization_string_args(qwen3_tool_parser):
1319
1320
1321 def test_get_vllm_registry_structural_tag_returns_structural_tag(
1250 - qwen3_tool_parser: Qwen3CoderToolParser,
1322 + qwen3_tool_parser: Qwen3EngineToolParser,
1323 sample_tools: list[ChatCompletionToolsParam],
1324 ) -> None:
1325 request_tools = _as_chat_completion_tools(sample_tools)
@@ -1289,7 +1361,7 @@ def test_adjust_request_auto_uses_vllm_registry_structural_tag(
1361 include_reasoning: bool,
1362 ) -> None:
1363 class TestParser(DelegatingParser):
1292 - tool_parser_cls = Qwen3CoderToolParser
1364 + tool_parser_cls = Qwen3EngineToolParser
1365
1366 request_tools = _as_chat_completion_tools(sample_tools)
1367 req = ChatCompletionRequest(
@@ -1311,7 +1383,7 @@ def test_adjust_request_required_prefers_structural_tag(
1383 sample_tools: list[ChatCompletionToolsParam],
1384 ) -> None:
1385 class TestParser(DelegatingParser):
1314 - tool_parser_cls = Qwen3CoderToolParser
1386 + tool_parser_cls = Qwen3EngineToolParser
1387
1388 request_tools = _as_chat_completion_tools(sample_tools)
1389 req = ChatCompletionRequest(
tests/tool_parsers/test_structural_tag_registry.py
+4 -4
@@ -24,7 +24,7 @@ from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
24 from vllm.tool_parsers.kimi_k2_tool_parser import KimiK2ToolParser
25 from vllm.tool_parsers.llama_tool_parser import Llama3JsonToolParser
26 from vllm.tool_parsers.minimax_m2_tool_parser import MinimaxM2ToolParser
27 -from vllm.tool_parsers.qwen3coder_tool_parser import Qwen3CoderToolParser
27 +from vllm.tool_parsers.qwen3_engine_tool_parser import Qwen3EngineToolParser
28 from vllm.tool_parsers.structural_tag_registry import (
29 SUPPORTED_STRUCTURAL_TAG_MODELS,
30 VLLM_BUILTIN_STRUCTURAL_TAG_MODELS,
@@ -183,7 +183,7 @@ def test_get_model_structural_tag_supports_named_tool_choice(
183 (KimiK2ToolParser, "kimi"),
184 (Llama3JsonToolParser, "llama"),
185 (MinimaxM2ToolParser, "minimax"),
186 - (Qwen3CoderToolParser, "qwen_3_coder"),
186 + (Qwen3EngineToolParser, "qwen_3_coder"),
187 ],
188 )
189 def test_tool_parsers_declare_matching_xgrammar_builtin_model(parser_cls, model):
@@ -238,7 +238,7 @@ def test_get_structural_tag_disables_reasoning(
238 tools=sample_tools,
239 tool_choice="auto",
240 )
241 - parser = Qwen3CoderToolParser(MagicMock(), tools=sample_tools)
241 + parser = Qwen3EngineToolParser(MagicMock(), tools=sample_tools)
242
243 parser.get_structural_tag(request)
244
@@ -261,7 +261,7 @@ def test_unified_parser_get_structural_tag_disables_reasoning(
261 )
262
263 class TestParser(DelegatingParser):
264 - tool_parser_cls = Qwen3CoderToolParser
264 + tool_parser_cls = Qwen3EngineToolParser
265
266 request = ChatCompletionRequest(
267 messages=[],
vllm/parser/abstract_parser.py
+133 -24
@@ -50,6 +50,31 @@ class StreamState:
50 # only used for "required" and "named tool" choices,
51 # tracks whether function name has been fully returned in the stream yet
52 function_name_returned: bool = False
53 + engine_based: bool = False
54 +
55 + def advance(
56 + self,
57 + delta_text: str,
58 + delta_token_ids: list[int],
59 + ) -> tuple[str, list[int]]:
60 + if self.engine_based:
61 + return delta_text, delta_token_ids
62 + return (
63 + self.previous_text + delta_text,
64 + self.previous_token_ids + delta_token_ids,
65 + )
66 +
67 + def commit(
68 + self,
69 + current_text: str,
70 + current_token_ids: list[int],
71 + ) -> None:
72 + if self.engine_based:
73 + self.previous_text = ""
74 + self.previous_token_ids = []
75 + else:
76 + self.previous_text = current_text
77 + self.previous_token_ids = current_token_ids
78
79
80 class Parser:
@@ -88,8 +113,6 @@ class Parser:
113 self.model_tokenizer = tokenizer
114 self._reasoning_parser: ReasoningParser | None = None
115 self._tool_parser: ToolParser | None = None
91 - self._stream_state = StreamState()
92 -
116 if self.__class__.reasoning_parser_cls is not None:
117 self._reasoning_parser = self.__class__.reasoning_parser_cls(
118 tokenizer, *args, **kwargs
@@ -97,6 +120,12 @@ class Parser:
120 if self.__class__.tool_parser_cls is not None:
121 self._tool_parser = self.__class__.tool_parser_cls(tokenizer, tools)
122
123 + self._engine_based = (
124 + self._reasoning_parser is None
125 + or self._reasoning_parser.engine_based_streaming
126 + ) and (self._tool_parser is None or self._tool_parser.engine_based_streaming)
127 + self._stream_state = StreamState(engine_based=self._engine_based)
128 +
129 @cached_property
130 def vocab(self) -> dict[str, int]:
131 """Get the vocabulary mapping from tokens to IDs."""
@@ -571,11 +600,28 @@ class DelegatingParser(Parser):
600 tool_call_id_type: str = "random",
601 function_name_returned: bool = False,
602 ) -> tuple[DeltaMessage | None, bool]:
603 + assert self._tool_parser is not None
604 + supports_required_and_named = self._tool_parser.supports_required_and_named
605 +
606 if request.tool_choice == "none":
607 + if self._engine_based:
608 + # Engine-backed parsers route content extraction through
609 + # extract_tool_calls_streaming, so run the full pipeline
610 + # and strip tool_calls after.
611 + delta_message = self.extract_tool_calls_streaming(
612 + previous_text,
613 + current_text,
614 + delta_text,
615 + previous_token_ids,
616 + current_token_ids,
617 + delta_token_ids,
618 + request, # type: ignore[arg-type]
619 + )
620 + if delta_message:
621 + delta_message.tool_calls = []
622 + return delta_message, False
623 return (DeltaMessage(content=delta_text) if delta_text else None), False
624
577 - assert self._tool_parser is not None
578 - supports_required_and_named = self._tool_parser.supports_required_and_named
625 if (
626 supports_required_and_named
627 and request.tool_choice
@@ -713,9 +759,9 @@ class DelegatingParser(Parser):
759 ):
760 state.reasoning_ended = True
761
716 - current_text = state.previous_text + delta_text
717 - current_token_ids = state.previous_token_ids + delta_token_ids
762 + current_text, current_token_ids = state.advance(delta_text, delta_token_ids)
763 delta_message: DeltaMessage | None = None
764 + reasoning_transitioned = False
765
766 # Reasoning extraction
767 if self._in_reasoning_phase(state):
@@ -727,16 +773,34 @@ class DelegatingParser(Parser):
773 current_token_ids=current_token_ids,
774 delta_token_ids=delta_token_ids,
775 )
730 - if self.is_reasoning_end_streaming(current_token_ids, delta_token_ids):
776 + reasoning_parser = self._reasoning_parser
777 + if reasoning_parser is not None and reasoning_parser.engine_based_streaming:
778 + should_transition = (
779 + reasoning_parser.has_engine_confirmed_reasoning_end()
780 + )
781 + else:
782 + should_transition = self.is_reasoning_end_streaming(
783 + current_token_ids, delta_token_ids
784 + )
785 + if should_transition:
786 state.reasoning_ended = True
787 + reasoning_transitioned = True
788 current_token_ids = self.extract_content_ids(delta_token_ids)
733 - current_text = (
734 - delta_message.content
735 - if delta_message and delta_message.content
736 - else ""
737 - )
738 - delta_text = current_text
739 - delta_token_ids = current_token_ids
789 + if self._engine_based:
790 + current_text = (
791 + self.model_tokenizer.decode(current_token_ids)
792 + if current_token_ids
793 + else ""
794 + )
795 + if delta_message and self._tool_parser is not None:
796 + delta_message.content = None
797 + else:
798 + current_text = (
799 + delta_message.content
800 + if delta_message and delta_message.content
801 + else ""
802 + )
803 + delta_text = current_text
804
805 # Tool call extraction
806 if self._in_tool_call_phase(state):
@@ -747,9 +811,10 @@ class DelegatingParser(Parser):
811 delta_text = current_text
812 delta_token_ids = current_token_ids
813
750 - # A boundary delta may carry both reasoning and tool call,
751 - # save it before the tool parser overwrites delta_message.
752 - reasoning = delta_message.reasoning if delta_message else None
814 + reasoning_from_this_batch = (
815 + delta_message.reasoning if delta_message else None
816 + )
817 +
818 delta_message, state.function_name_returned = (
819 self._extract_tool_calls_streaming(
820 previous_text=state.previous_text,
@@ -764,10 +829,12 @@ class DelegatingParser(Parser):
829 function_name_returned=state.function_name_returned,
830 )
831 )
767 - if reasoning:
768 - if not delta_message:
769 - delta_message = DeltaMessage()
770 - delta_message.reasoning = reasoning
832 +
833 + if reasoning_from_this_batch:
834 + if delta_message is None:
835 + delta_message = DeltaMessage(reasoning=reasoning_from_this_batch)
836 + elif not delta_message.reasoning:
837 + delta_message.reasoning = reasoning_from_this_batch
838
839 if (
840 delta_message
@@ -776,18 +843,60 @@ class DelegatingParser(Parser):
843 ):
844 state.history_tool_call_cnt += 1
845
779 - # No phase active: pass through as content
846 + # No phase active: pass through as content.
847 + # Skip when reasoning just ended in this delta — the engine already
848 + # consumed the end-of-reasoning marker (e.g. </think>) and
849 + # delta_text still contains the raw marker text.
850 if (
851 delta_message is None
852 + and not reasoning_transitioned
853 and not self._in_reasoning_phase(state)
854 and not self._in_tool_call_phase(state)
855 ):
856 delta_message = DeltaMessage(content=delta_text)
857
787 - state.previous_text = current_text
788 - state.previous_token_ids = current_token_ids
858 + state.commit(current_text, current_token_ids)
859
860 if finished:
861 delta_message = self.finalize_generation(delta_message, request, state)
862 + delta_message = self._flush_engine_parsers(delta_message)
863
864 return delta_message
865 +
866 + def _flush_engine_parsers(
867 + self, delta_message: DeltaMessage | None
868 + ) -> DeltaMessage | None:
869 + """Flush buffered state from engine-based parsers at stream end."""
870 + reasoning_ended = self._stream_state.reasoning_ended
871 + for parser in (self._reasoning_parser, self._tool_parser):
872 + if not getattr(parser, "engine_based_streaming", False):
873 + continue
874 + # When reasoning has ended and we transitioned to the tool
875 + # phase, the reasoning parser's engine may still have buffered
876 + # characters from tool-call markup it saw with
877 + # skip_tool_parsing=True. Flushing that would leak spurious
878 + # content (e.g. a stray '"'), so skip it.
879 + if parser is self._reasoning_parser and reasoning_ended:
880 + continue
881 + finish = getattr(parser, "finish_streaming", None)
882 + if finish is None:
883 + continue
884 + flush_delta = finish()
885 + if flush_delta is None:
886 + continue
887 + if delta_message is None:
888 + delta_message = flush_delta
889 + else:
890 + if flush_delta.content:
891 + delta_message.content = (
892 + delta_message.content or ""
893 + ) + flush_delta.content
894 + if flush_delta.reasoning:
895 + delta_message.reasoning = (
896 + delta_message.reasoning or ""
897 + ) + flush_delta.reasoning
898 + if flush_delta.tool_calls:
899 + delta_message.tool_calls = (
900 + delta_message.tool_calls or []
901 + ) + flush_delta.tool_calls
902 + return delta_message
vllm/parser/engine/__init__.py new
+17
@@ -0,0 +1,17 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""
4 +Streaming parser engine framework for tool call and reasoning extraction.
5 +
6 +Instead of hand-rolling a parser for every model's tool-call / reasoning
7 +format, each format is declared as a ParserEngineConfig (terminals,
8 +states, and transitions) and a shared incremental engine handles
9 +streaming, ambiguity buffering, token-ID mapping, and delta computation.
10 +"""
11 +
12 +from vllm.parser.engine.events import EventType, SemanticEvent
13 +
14 +__all__ = [
15 + "EventType",
16 + "SemanticEvent",
17 +]
vllm/parser/engine/adapters.py new
+199
@@ -0,0 +1,199 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Adapters that expose :class:`ParserEngine` through the legacy
4 +:class:`ReasoningParser` and :class:`ToolParser` interfaces.
5 +
6 +This lets parser engines flow through the existing serving-layer code
7 +paths that expect separate reasoning and tool parser instances, without
8 +any changes to the serving layer itself.
9 +"""
10 +
11 +from __future__ import annotations
12 +
13 +from collections.abc import Iterator, Sequence
14 +from contextlib import contextmanager
15 +from typing import TYPE_CHECKING
16 +
17 +from vllm.parser.engine.parser_engine_config import ParserState
18 +from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
19 +from vllm.tool_parsers.abstract_tool_parser import ToolParser
20 +
21 +if TYPE_CHECKING:
22 + from vllm.entrypoints.openai.chat_completion.protocol import (
23 + ChatCompletionRequest,
24 + )
25 + from vllm.entrypoints.openai.engine.protocol import (
26 + DeltaMessage,
27 + ExtractedToolCallInformation,
28 + )
29 + from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
30 + from vllm.parser.engine.parser_engine import ParserEngine
31 + from vllm.tokenizers import TokenizerLike
32 + from vllm.tool_parsers.utils import Tool
33 +
34 +
35 +class ParserEngineReasoningAdapter(ReasoningParser):
36 + """Adapts a :class:`ParserEngine` to the :class:`ReasoningParser`
37 + interface so parser engines can be used as reasoning parsers in the
38 + existing serving code.
39 +
40 + Subclasses set :attr:`_parser_engine_cls` to the concrete
41 + :class:`ParserEngine` class.
42 + """
43 +
44 + _parser_engine_cls: type[ParserEngine]
45 + engine_based_streaming: bool = True
46 +
47 + def __init__(self, tokenizer: TokenizerLike, *args, **kwargs) -> None:
48 + super().__init__(tokenizer, *args, **kwargs)
49 + self._parser_engine = self._parser_engine_cls(tokenizer, **kwargs) # type: ignore[call-arg]
50 +
51 + @contextmanager
52 + def _skip_tool_parsing(self) -> Iterator[None]:
53 + saved = self._parser_engine.skip_tool_parsing
54 + self._parser_engine.skip_tool_parsing = True
55 + try:
56 + yield
57 + finally:
58 + self._parser_engine.skip_tool_parsing = saved
59 +
60 + def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
61 + return self._parser_engine.is_reasoning_end(list(input_ids))
62 +
63 + def extract_content_ids(self, input_ids: list[int]) -> list[int]:
64 + return self._parser_engine.extract_content_ids(input_ids)
65 +
66 + def extract_reasoning(
67 + self,
68 + model_output: str,
69 + request: ChatCompletionRequest | ResponsesRequest,
70 + ) -> tuple[str | None, str | None]:
71 + with self._skip_tool_parsing():
72 + return self._parser_engine.extract_reasoning(model_output, request)
73 +
74 + def extract_reasoning_streaming(
75 + self,
76 + previous_text: str,
77 + current_text: str,
78 + delta_text: str,
79 + previous_token_ids: Sequence[int],
80 + current_token_ids: Sequence[int],
81 + delta_token_ids: Sequence[int],
82 + ) -> DeltaMessage | None:
83 + with self._skip_tool_parsing():
84 + return self._parser_engine.extract_reasoning_streaming(
85 + previous_text,
86 + current_text,
87 + delta_text,
88 + previous_token_ids,
89 + current_token_ids,
90 + delta_token_ids,
91 + )
92 +
93 + @property
94 + def reasoning_start_str(self) -> str | None:
95 + return self._parser_engine.reasoning_start_str
96 +
97 + @property
98 + def reasoning_end_str(self) -> str | None:
99 + return self._parser_engine.reasoning_end_str
100 +
101 + def adjust_request(
102 + self,
103 + request: ChatCompletionRequest | ResponsesRequest,
104 + ) -> ChatCompletionRequest | ResponsesRequest:
105 + return self._parser_engine.adjust_request(request)
106 +
107 + def has_engine_confirmed_reasoning_end(self) -> bool:
108 + return self._parser_engine.reasoning_ended
109 +
110 + def finish_streaming(self) -> DeltaMessage | None:
111 + return self._parser_engine.finish_streaming()
112 +
113 + def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
114 + return self._parser_engine.count_reasoning_tokens(token_ids)
115 +
116 +
117 +class ParserEngineToolAdapter(ToolParser):
118 + """Adapts a :class:`ParserEngine` to the :class:`ToolParser` interface.
119 +
120 + :meth:`extract_tool_calls` starts the parser engine in ``CONTENT``
121 + state so it can parse reasoning-stripped content (i.e. the output of
122 + :meth:`ReasoningParser.extract_reasoning`).
123 +
124 + Subclasses set :attr:`_parser_engine_cls` to the concrete
125 + :class:`ParserEngine` class.
126 + """
127 +
128 + _parser_engine_cls: type[ParserEngine]
129 + engine_based_streaming: bool = True
130 +
131 + def __init__(
132 + self,
133 + tokenizer: TokenizerLike,
134 + tools: list[Tool] | None = None,
135 + **kwargs,
136 + ) -> None:
137 + super().__init__(tokenizer, tools)
138 + self._parser_engine = self._parser_engine_cls(tokenizer, tools, **kwargs) # type: ignore[call-arg]
139 +
140 + def adjust_request(
141 + self,
142 + request: ChatCompletionRequest | ResponsesRequest,
143 + ) -> ChatCompletionRequest | ResponsesRequest:
144 + request = super().adjust_request(request)
145 + return self._parser_engine.adjust_request(request)
146 +
147 + def extract_tool_calls(
148 + self,
149 + model_output: str,
150 + request: ChatCompletionRequest,
151 + ) -> ExtractedToolCallInformation:
152 + return self._parser_engine.extract_tool_calls_from_content(
153 + model_output, request
154 + )
155 +
156 + def extract_tool_calls_streaming(
157 + self,
158 + previous_text: str,
159 + current_text: str,
160 + delta_text: str,
161 + previous_token_ids: Sequence[int],
162 + current_token_ids: Sequence[int],
163 + delta_token_ids: Sequence[int],
164 + request: ChatCompletionRequest,
165 + ) -> DeltaMessage | None:
166 + engine = self._parser_engine
167 + engine.initialize_streaming(initial_state=ParserState.CONTENT)
168 + return engine.extract_tool_calls_streaming(
169 + previous_text,
170 + current_text,
171 + delta_text,
172 + previous_token_ids,
173 + current_token_ids,
174 + delta_token_ids,
175 + request,
176 + )
177 +
178 + def finish_streaming(self) -> DeltaMessage | None:
179 + return self._parser_engine.finish_streaming()
180 +
181 +
182 +def make_adapters(
183 + parser_engine_cls: type[ParserEngine],
184 +) -> tuple[type[ParserEngineReasoningAdapter], type[ParserEngineToolAdapter]]:
185 + reasoning_adapter = type(
186 + f"{parser_engine_cls.__name__}ReasoningAdapter",
187 + (ParserEngineReasoningAdapter,),
188 + {"_parser_engine_cls": parser_engine_cls},
189 + )
190 + tool_adapter = type(
191 + f"{parser_engine_cls.__name__}ToolAdapter",
192 + (ParserEngineToolAdapter,),
193 + {"_parser_engine_cls": parser_engine_cls},
194 + )
195 + # Let the serving layer find the adapters and call adjust_request(),
196 + # which sets skip_special_tokens=False for the detokenizer.
197 + parser_engine_cls.reasoning_parser_cls = reasoning_adapter # type: ignore[attr-defined]
198 + parser_engine_cls.tool_parser_cls = tool_adapter # type: ignore[attr-defined]
199 + return reasoning_adapter, tool_adapter
vllm/parser/engine/events.py new
+26
@@ -0,0 +1,26 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Semantic event types emitted by the streaming parser engine."""
4 +
5 +from __future__ import annotations
6 +
7 +from dataclasses import dataclass
8 +from enum import Enum, auto
9 +
10 +
11 +class EventType(Enum):
12 + TEXT_CHUNK = auto()
13 + REASONING_START = auto()
14 + REASONING_CHUNK = auto()
15 + REASONING_END = auto()
16 + TOOL_CALL_START = auto()
17 + TOOL_NAME = auto()
18 + ARG_VALUE_CHUNK = auto()
19 + TOOL_CALL_END = auto()
20 +
21 +
22 +@dataclass(slots=True)
23 +class SemanticEvent:
24 + type: EventType
25 + value: str = ""
26 + tool_index: int = -1
vllm/parser/engine/incremental_lexer.py new
+210
@@ -0,0 +1,210 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Incremental text lexer that converts text chunks into terminal
4 +tokens, with prefix-match buffering for ambiguous boundaries."""
5 +
6 +from __future__ import annotations
7 +
8 +from dataclasses import dataclass
9 +
10 +import regex as re
11 +
12 +CONTENT_TERMINAL = "__CONTENT__"
13 +
14 +
15 +@dataclass(slots=True)
16 +class TerminalDef:
17 + name: str
18 + pattern: re.Pattern[str]
19 + is_literal: bool = False
20 + literal: str = ""
21 +
22 +
23 +@dataclass(slots=True)
24 +class LexToken:
25 + terminal: str
26 + value: str
27 +
28 +
29 +class LexerShape:
30 + """Immutable pre-computed data derived from terminal definitions.
31 +
32 + Created once per :class:`ParserEngineConfig` and shared across all
33 + :class:`IncrementalLexer` instances that use the same config.
34 + """
35 +
36 + __slots__ = (
37 + "terminals",
38 + "literal_strings",
39 + "max_literal_len",
40 + "literal_first_chars",
41 + "has_only_literals",
42 + "prefix_set",
43 + "literals_by_first",
44 + )
45 +
46 + def __init__(self, terminals: list[TerminalDef]) -> None:
47 + self.terminals = sorted(
48 + terminals,
49 + key=lambda t: (not t.is_literal, -len(t.pattern.pattern)),
50 + )
51 + literal_strings: list[tuple[str, str]] = []
52 + for t in self.terminals:
53 + if t.is_literal:
54 + literal_strings.append((t.literal, t.name))
55 +
56 + self.literal_strings = literal_strings
57 + max_len = 0
58 + for lit, _ in literal_strings:
59 + if len(lit) > max_len:
60 + max_len = len(lit)
61 + self.max_literal_len = max_len
62 + self.literal_first_chars = frozenset(
63 + lit[0] for lit, _ in literal_strings if lit
64 + )
65 + self.has_only_literals = all(t.is_literal for t in terminals)
66 +
67 + prefix_set: set[str] = set()
68 + for lit, _ in literal_strings:
69 + for i in range(1, len(lit)):
70 + prefix_set.add(lit[:i])
71 + self.prefix_set = frozenset(prefix_set)
72 +
73 + by_first: dict[str, list[tuple[str, str]]] = {}
74 + for lit, name in literal_strings:
75 + if lit:
76 + by_first.setdefault(lit[0], []).append((lit, name))
77 + self.literals_by_first = by_first
78 +
79 +
80 +class IncrementalLexer:
81 + """Converts streaming text into terminal tokens.
82 +
83 + The key feature is **prefix-match buffering**: when the text in the
84 + buffer could be the start of a multi-character terminal (e.g.
85 + ``"<tool_"`` that could become ``"<tool_call>"``), the lexer holds
86 + the text rather than emitting it. When the next chunk arrives, it
87 + either completes the terminal or flushes the buffered text as
88 + content.
89 +
90 + Terminals are tried in priority order (literals first, then by
91 + descending priority, then by pattern length).
92 + """
93 +
94 + def __init__(
95 + self,
96 + terminals: list[TerminalDef] | LexerShape,
97 + content_terminal: str = CONTENT_TERMINAL,
98 + ) -> None:
99 + if isinstance(terminals, LexerShape):
100 + shape = terminals
101 + else:
102 + shape = LexerShape(terminals)
103 + self._shape = shape
104 + self.terminals = shape.terminals
105 + self.content_terminal = content_terminal
106 + self.buffer = ""
107 +
108 + self._literal_strings = shape.literal_strings
109 + self._max_literal_len = shape.max_literal_len
110 + self._literal_first_chars = shape.literal_first_chars
111 + self._has_only_literals = shape.has_only_literals
112 + self._prefix_set = shape.prefix_set
113 + self._literals_by_first = shape.literals_by_first
114 +
115 + def reset(self) -> None:
116 + self.buffer = ""
117 +
118 + def feed(self, text: str) -> list[LexToken]:
119 + if not self.buffer and self._has_only_literals and self._literal_first_chars:
120 + for ch in text:
121 + if ch in self._literal_first_chars:
122 + break
123 + else:
124 + return [LexToken(self.content_terminal, text)]
125 + self.buffer += text
126 + return self._drain()
127 +
128 + def flush(self) -> list[LexToken]:
129 + tokens: list[LexToken] = []
130 + if self.buffer:
131 + tokens.append(LexToken(self.content_terminal, self.buffer))
132 + self.buffer = ""
133 + return tokens
134 +
135 + def _drain(self) -> list[LexToken]:
136 + tokens: list[LexToken] = []
137 + first_chars = self._literal_first_chars
138 + content_terminal = self.content_terminal
139 + has_only_literals = self._has_only_literals
140 + literals_by_first = self._literals_by_first
141 + prefix_set = self._prefix_set
142 +
143 + while self.buffer:
144 + if has_only_literals and first_chars:
145 + has_potential = False
146 + for ch in self.buffer:
147 + if ch in first_chars:
148 + has_potential = True
149 + break
150 + if not has_potential:
151 + tokens.append(LexToken(content_terminal, self.buffer))
152 + self.buffer = ""
153 + break
154 +
155 + best_match: tuple[str, str, int] | None = None
156 +
157 + first = self.buffer[0]
158 + for lit, name in literals_by_first.get(first, ()):
159 + if self.buffer.startswith(lit) and (
160 + best_match is None or len(lit) > best_match[2]
161 + ):
162 + best_match = (name, lit, len(lit))
163 +
164 + if self.buffer in prefix_set:
165 + if best_match is not None:
166 + tokens.append(LexToken(best_match[0], best_match[1]))
167 + self.buffer = self.buffer[best_match[2] :]
168 + continue
169 + else:
170 + break
171 +
172 + if best_match is not None:
173 + tokens.append(LexToken(best_match[0], best_match[1]))
174 + self.buffer = self.buffer[best_match[2] :]
175 + else:
176 + content_end = self._find_content_boundary()
177 + if content_end > 0:
178 + tokens.append(LexToken(content_terminal, self.buffer[:content_end]))
179 + self.buffer = self.buffer[content_end:]
180 + else:
181 + tokens.append(LexToken(content_terminal, self.buffer[0]))
182 + self.buffer = self.buffer[1:]
183 +
184 + return tokens
185 +
186 + def _find_content_boundary(self) -> int:
187 + buf = self.buffer
188 + n = len(buf)
189 + first_chars = self._literal_first_chars
190 + for i in range(1, n):
191 + if buf[i] not in first_chars:
192 + continue
193 + remaining = n - i
194 + for lit, _ in self._literal_strings:
195 + check_len = min(remaining, len(lit))
196 + if buf[i : i + check_len] == lit[:check_len]:
197 + return i
198 + return n
199 +
200 +
201 +def terminals_from_literals(literals: dict[str, str]) -> list[TerminalDef]:
202 + return [
203 + TerminalDef(
204 + name=name,
205 + pattern=re.compile(re.escape(lit)),
206 + is_literal=True,
207 + literal=lit,
208 + )
209 + for name, lit in literals.items()
210 + ]
vllm/parser/engine/parser_engine.py new
+969
@@ -0,0 +1,969 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Parser engine base that handles both reasoning and tool call
4 +extraction with a single :class:`StreamingParserEngine`.
5 +"""
6 +
7 +from __future__ import annotations
8 +
9 +import json
10 +from collections.abc import Sequence
11 +from functools import cached_property
12 +from typing import TYPE_CHECKING
13 +
14 +import regex as re
15 +
16 +from vllm.entrypoints.chat_utils import make_tool_call_id
17 +from vllm.entrypoints.openai.engine.protocol import (
18 + DeltaFunctionCall,
19 + DeltaMessage,
20 + DeltaToolCall,
21 + ExtractedToolCallInformation,
22 + FunctionCall,
23 + ToolCall,
24 +)
25 +from vllm.logger import init_logger
26 +from vllm.parser.abstract_parser import Parser, StreamState
27 +from vllm.parser.engine.events import EventType, SemanticEvent
28 +from vllm.parser.engine.parser_engine_config import ParserEngineConfig, ParserState
29 +from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
30 +from vllm.tool_parsers.utils import (
31 + coerce_to_schema_type,
32 + extract_types_from_schema,
33 + find_tool_properties,
34 +)
35 +
36 +if TYPE_CHECKING:
37 + from vllm.entrypoints.openai.chat_completion.protocol import (
38 + ChatCompletionRequest,
39 + )
40 + from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
41 + from vllm.tokenizers import TokenizerLike
42 + from vllm.tool_parsers.abstract_tool_parser import Tool
43 +
44 +logger = init_logger(__name__)
45 +
46 +
47 +class ToolCallSlot:
48 + __slots__ = (
49 + "id",
50 + "name",
51 + "_args_parts",
52 + "_args_joined",
53 + "name_sent",
54 + "streamed_json",
55 + )
56 +
57 + def __init__(self) -> None:
58 + self.id: str = ""
59 + self.name: str = ""
60 + self._args_parts: list[str] = []
61 + self._args_joined: str | None = ""
62 + self.name_sent: bool = False
63 + self.streamed_json: str = ""
64 +
65 + @property
66 + def args(self) -> str:
67 + if self._args_joined is None:
68 + self._args_joined = "".join(self._args_parts)
69 + return self._args_joined
70 +
71 + def append_args(self, value: str) -> None:
72 + self._args_parts.append(value)
73 + self._args_joined = None
74 +
75 +
76 +class ParserEngine(Parser):
77 + """A :class:`Parser` backed by a single declarative engine config.
78 +
79 + Subclasses set the ``ParserEngineConfig`` in ``__init__`` to define the
80 + complete output format for a model (reasoning + tool calls).
81 + """
82 +
83 + def __init__(
84 + self,
85 + tokenizer: TokenizerLike,
86 + tools: list[Tool] | None = None,
87 + *,
88 + parser_engine_config: ParserEngineConfig,
89 + **kwargs,
90 + ) -> None:
91 + self.model_tokenizer = tokenizer
92 + self._tools = tools
93 + self._stream_state = StreamState()
94 + self._reasoning_parser = None
95 + self._tool_parser = None
96 + self.parser_engine_config = parser_engine_config
97 + self._engine = StreamingParserEngine(
98 + parser_engine_config, tokenizer, vocab=self.vocab
99 + )
100 +
101 + self._reasoning_ended: bool = False
102 + self._streaming_initialized: bool = False
103 +
104 + self._tool_slots: list[ToolCallSlot] = []
105 + self._deferred_content: str = ""
106 + self._deferred_reasoning: str = ""
107 + self._content_has_nonws: bool = False
108 +
109 + self._arg_converter = parser_engine_config.arg_converter
110 + self._arg_structural_chars = parser_engine_config.arg_structural_chars
111 + self._stream_arg_deltas = parser_engine_config.stream_arg_deltas
112 + self._strip_trailing_reasoning_ws = (
113 + parser_engine_config.strip_trailing_reasoning_whitespace
114 + )
115 + self._drop_ws_only_content_before_tools = (
116 + parser_engine_config.drop_whitespace_only_content_before_tools
117 + )
118 + self._strip_content_ws_with_tools = (
119 + parser_engine_config.strip_content_whitespace_with_tools
120 + )
121 +
122 + vocab = self.vocab
123 + self._reasoning_start_token_id: int | None = None
124 + self._reasoning_end_token_id: int | None = None
125 +
126 + start_text = parser_engine_config.token_id_terminals.get("THINK_START")
127 + end_text = parser_engine_config.token_id_terminals.get("THINK_END")
128 + if start_text:
129 + self._reasoning_start_token_id = vocab.get(start_text)
130 + if end_text:
131 + self._reasoning_end_token_id = vocab.get(end_text)
132 +
133 + @property
134 + def reasoning_start_str(self) -> str | None:
135 + return self.parser_engine_config.terminals.get("THINK_START")
136 +
137 + @property
138 + def reasoning_end_str(self) -> str | None:
139 + return self.parser_engine_config.terminals.get("THINK_END")
140 +
141 + @cached_property
142 + def vocab(self) -> dict[str, int]:
143 + return self.model_tokenizer.get_vocab()
144 +
145 + # ── Engine lifecycle ──────────────────────────────────────────────
146 +
147 + @property
148 + def skip_tool_parsing(self) -> bool:
149 + return self._engine.skip_tool_parsing
150 +
151 + @skip_tool_parsing.setter
152 + def skip_tool_parsing(self, value: bool) -> None:
153 + self._engine.skip_tool_parsing = value
154 +
155 + @property
156 + def reasoning_ended(self) -> bool:
157 + return self._reasoning_ended
158 +
159 + def initialize_streaming(
160 + self,
161 + initial_state: ParserState | None = None,
162 + ) -> None:
163 + if not self._streaming_initialized:
164 + self._streaming_initialized = True
165 + self._reset(initial_state=initial_state)
166 +
167 + def finish_streaming(self) -> DeltaMessage | None:
168 + events = self._engine.finish()
169 + return self._events_to_delta(events) if events else None
170 +
171 + def _reset(self, initial_state: ParserState | None = None) -> None:
172 + self._engine.reset(initial_state=initial_state)
173 + self._reasoning_ended = False
174 + self._tool_slots.clear()
175 + self._deferred_content = ""
176 + self._deferred_reasoning = ""
177 + self._content_has_nonws = False
178 +
179 + def adjust_request(
180 + self, request: ChatCompletionRequest | ResponsesRequest
181 + ) -> ChatCompletionRequest | ResponsesRequest:
182 + request.skip_special_tokens = False
183 + return request
184 +
185 + # ── Schema-aware type correction ─────────────────────────────────
186 +
187 + @staticmethod
188 + def _coerce_value(value: object, schema: dict) -> tuple[object, bool]:
189 + """Coerce a single value according to its schema.
190 +
191 + Returns ``(coerced_value, changed)``.
192 + """
193 + if isinstance(value, str):
194 + types = extract_types_from_schema(schema)
195 + coerced = coerce_to_schema_type(value, types)
196 + if coerced is not value:
197 + return coerced, True
198 + return value, False
199 +
200 + if isinstance(value, dict):
201 + nested_props = schema.get("properties")
202 + if isinstance(nested_props, dict):
203 + _, changed = ParserEngine._coerce_dict(value, nested_props)
204 + return value, changed
205 + return value, False
206 +
207 + if isinstance(value, list):
208 + items_schema = schema.get("items")
209 + if isinstance(items_schema, dict):
210 + changed = False
211 + for i, item in enumerate(value):
212 + coerced, item_changed = ParserEngine._coerce_value(
213 + item, items_schema
214 + )
215 + if item_changed:
216 + value[i] = coerced
217 + changed = True
218 + return value, changed
219 + return value, False
220 +
221 + types = extract_types_from_schema(schema)
222 + as_str = json.dumps(value, ensure_ascii=False)
223 + coerced = coerce_to_schema_type(as_str, types)
224 + if coerced != value:
225 + return coerced, True
226 + return value, False
227 +
228 + @staticmethod
229 + def _coerce_dict(args: dict, properties: dict) -> tuple[dict, bool]:
230 + """Coerce all values in *args* using *properties* schemas."""
231 + changed = False
232 + for key, value in args.items():
233 + prop = properties.get(key)
234 + if not isinstance(prop, dict):
235 + continue
236 + coerced, val_changed = ParserEngine._coerce_value(value, prop)
237 + if val_changed:
238 + args[key] = coerced
239 + changed = True
240 + return args, changed
241 +
242 + @staticmethod
243 + def _safe_arg_prefix(json_str: str) -> str:
244 + """Return the prefix of *json_str* up to the last top-level value.
245 +
246 + Middle values (followed by a comma) are stable across streaming
247 + ticks and included. The trailing value is excluded because type
248 + coercion may change its serialised form between ticks, which
249 + would violate the ``startswith(prev)`` prefix invariant.
250 + """
251 + last_colon = -1
252 + in_string = False
253 + escape = False
254 + depth = 0
255 + for i, c in enumerate(json_str):
256 + if escape:
257 + escape = False
258 + continue
259 + if in_string:
260 + if c == "\\":
261 + escape = True
262 + elif c == '"':
263 + in_string = False
264 + continue
265 + if c == '"':
266 + in_string = True
267 + elif c in ("{", "["):
268 + depth += 1
269 + elif c in ("}", "]"):
270 + depth -= 1
271 + elif c == ":" and depth == 1:
272 + last_colon = i
273 + if last_colon < 0:
274 + return ""
275 + end = last_colon + 1
276 + while end < len(json_str) and json_str[end] in (" ", "\t", "\n", "\r"):
277 + end += 1
278 + return json_str[:end]
279 +
280 + def _fix_arg_types(self, args_json: str, func_name: str) -> str:
281 + """Correct parameter types using the tool schema.
282 +
283 + String values are coerced via :func:`coerce_to_schema_type`.
284 + Nested objects and arrays are recursed into when the schema
285 + defines ``properties`` or ``items``. Without a schema, values
286 + stay as strings.
287 + """
288 + if not self._tools or not func_name:
289 + return args_json
290 + try:
291 + args = json.loads(args_json)
292 + except (json.JSONDecodeError, ValueError):
293 + return args_json
294 + if not isinstance(args, dict):
295 + return args_json
296 +
297 + properties = find_tool_properties(self._tools, func_name)
298 + if not properties:
299 + return args_json
300 +
301 + _, changed = self._coerce_dict(args, properties)
302 +
303 + if changed:
304 + return json.dumps(args, ensure_ascii=False)
305 + return args_json
306 +
307 + # ── Private helpers ─────────────────────────────────────────────
308 +
309 + def _check_skip_tool_parsing(
310 + self,
311 + request: ChatCompletionRequest | ResponsesRequest,
312 + ) -> None:
313 + if not self.skip_tool_parsing:
314 + tool_choice = getattr(request, "tool_choice", None)
315 + tools = getattr(request, "tools", None)
316 + if tool_choice == "none" and tools:
317 + self.skip_tool_parsing = True
318 +
319 + def _strip_content_whitespace(
320 + self,
321 + content: str,
322 + tools_called: bool,
323 + ) -> str | None:
324 + if tools_called:
325 + if self._strip_content_ws_with_tools:
326 + content = content.strip()
327 + elif self._drop_ws_only_content_before_tools and not content.strip():
328 + content = ""
329 + return content or None
330 +
331 + # ── Streaming: parse_delta ────────────────────────────────────────
332 +
333 + def parse_delta(
334 + self,
335 + delta_text: str,
336 + delta_token_ids: list[int],
337 + request: ChatCompletionRequest | ResponsesRequest,
338 + prompt_token_ids: list[int] | None = None,
339 + *,
340 + finished: bool,
341 + ) -> DeltaMessage | None:
342 + self._check_skip_tool_parsing(request)
343 + events = self._engine.feed(delta_text, delta_token_ids)
344 + if finished:
345 + events.extend(self._engine.finish())
346 + result = self._events_to_delta(events, finished=finished)
347 + return self._strip_trailing_reasoning(result)
348 +
349 + def _strip_trailing_reasoning(
350 + self,
351 + delta: DeltaMessage | None,
352 + ) -> DeltaMessage | None:
353 + """Strip trailing whitespace from reasoning, deferring it until we
354 + know whether more reasoning follows or reasoning has ended.
355 +
356 + Runs in ``parse_delta`` *after* ``_events_to_delta`` (and any
357 + subclass overrides) so that overrides see the raw reasoning text.
358 +
359 + Gated by ``strip_trailing_reasoning_whitespace``; when disabled,
360 + passes through unchanged.
361 + """
362 + if not self._strip_trailing_reasoning_ws:
363 + return delta
364 + if delta is not None and delta.reasoning is not None:
365 + combined = self._deferred_reasoning + delta.reasoning
366 + trimmed = combined.rstrip()
367 + self._deferred_reasoning = combined[len(trimmed) :]
368 + delta.reasoning = trimmed or None
369 + if (
370 + delta.reasoning is None
371 + and delta.content is None
372 + and not delta.tool_calls
373 + ):
374 + return None
375 + elif self._deferred_reasoning and self._reasoning_ended:
376 + self._deferred_reasoning = ""
377 + return delta
378 +
379 + # ── Non-streaming: extract_reasoning ──────────────────────────────
380 +
381 + def extract_reasoning(
382 + self,
383 + model_output: str,
384 + request: ChatCompletionRequest | ResponsesRequest,
385 + ) -> tuple[str | None, str | None]:
386 + self._reset()
387 + events = self._engine.feed(model_output, [])
388 + events.extend(self._engine.finish())
389 +
390 + reasoning_parts: list[str] = []
391 + content_parts: list[str] = []
392 +
393 + for event in events:
394 + if event.type == EventType.REASONING_CHUNK:
395 + reasoning_parts.append(event.value)
396 + elif event.type == EventType.TEXT_CHUNK:
397 + content_parts.append(event.value)
398 + elif event.type == EventType.REASONING_END:
399 + self._reasoning_ended = True
400 +
401 + raw_reasoning = "".join(reasoning_parts)
402 + if self._strip_trailing_reasoning_ws:
403 + raw_reasoning = raw_reasoning.rstrip()
404 + reasoning = raw_reasoning or None
405 + content = "".join(content_parts) or None
406 + return reasoning, content
407 +
408 + # ── Non-streaming: extract_reasoning_streaming ────────────────────
409 +
410 + def extract_reasoning_streaming(
411 + self,
412 + previous_text: str,
413 + current_text: str,
414 + delta_text: str,
415 + previous_token_ids: Sequence[int],
416 + current_token_ids: Sequence[int],
417 + delta_token_ids: Sequence[int],
418 + ) -> DeltaMessage | None:
419 + self.initialize_streaming()
420 + events = self._engine.feed(delta_text, delta_token_ids)
421 + return self._strip_trailing_reasoning(self._events_to_delta(events))
422 +
423 + # ── Non-streaming: extract_tool_calls ─────────────────────────────
424 +
425 + def extract_tool_calls(
426 + self,
427 + model_output: str,
428 + request: ChatCompletionRequest | ResponsesRequest,
429 + ) -> ExtractedToolCallInformation:
430 + self._reset()
431 + self._streaming_initialized = True
432 + result = self.extract_tool_calls_streaming(
433 + previous_text="",
434 + current_text=model_output,
435 + delta_text=model_output,
436 + previous_token_ids=[],
437 + current_token_ids=[],
438 + delta_token_ids=[],
439 + request=request,
440 + )
441 + finish_delta = self.finish_streaming()
442 + return self._build_extracted_result(result, finish_delta)
443 +
444 + def extract_tool_calls_from_content(
445 + self,
446 + content: str,
447 + request: ChatCompletionRequest,
448 + ) -> ExtractedToolCallInformation:
449 + """Extract tool calls from reasoning-stripped content.
450 +
451 + Unlike :meth:`extract_tool_calls` which re-parses the full model
452 + output, this method starts the parser engine in ``CONTENT`` state
453 + so it can parse content that has already had reasoning stripped.
454 + """
455 + _, parsed_content, tool_call_info = self._single_pass_parse(
456 + content,
457 + [],
458 + initial_state=ParserState.CONTENT,
459 + )
460 + if parsed_content is not None and tool_call_info.content is None:
461 + tool_call_info = ExtractedToolCallInformation(
462 + tools_called=tool_call_info.tools_called,
463 + tool_calls=tool_call_info.tool_calls,
464 + content=parsed_content,
465 + )
466 + return tool_call_info
467 +
468 + def extract_tool_calls_streaming(
469 + self,
470 + previous_text: str,
471 + current_text: str,
472 + delta_text: str,
473 + previous_token_ids: Sequence[int],
474 + current_token_ids: Sequence[int],
475 + delta_token_ids: Sequence[int],
476 + request: ChatCompletionRequest | ResponsesRequest,
477 + ) -> DeltaMessage | None:
478 + self.initialize_streaming()
479 + self._check_skip_tool_parsing(request)
480 + events = self._engine.feed(delta_text, delta_token_ids)
481 + return self._strip_trailing_reasoning(self._events_to_delta(events))
482 +
483 + # ── Reasoning state queries ───────────────────────────────────────
484 +
485 + def is_reasoning_end(self, input_ids: list[int]) -> bool:
486 + end_id = self._reasoning_end_token_id
487 + start_id = self._reasoning_start_token_id
488 + if end_id is not None:
489 + if not input_ids:
490 + return self.parser_engine_config.initial_state != ParserState.REASONING
491 + for i in range(len(input_ids) - 1, -1, -1):
492 + if input_ids[i] == end_id:
493 + return True
494 + if start_id is not None and input_ids[i] == start_id:
495 + return False
496 + return False
497 + return self._reasoning_ended
498 +
499 + def extract_content_ids(self, input_ids: list[int]) -> list[int]:
500 + end_id = self._reasoning_end_token_id
501 + if end_id is not None:
502 + for i in range(len(input_ids) - 1, -1, -1):
503 + if input_ids[i] == end_id:
504 + return input_ids[i + 1 :]
505 + return input_ids
506 +
507 + def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
508 + start_id = self._reasoning_start_token_id
509 + end_id = self._reasoning_end_token_id
510 + if start_id is None or end_id is None:
511 + return 0
512 + count = 0
513 + depth = 0
514 + for token_id in token_ids:
515 + if token_id == start_id:
516 + depth += 1
517 + continue
518 + if token_id == end_id:
519 + if depth > 0:
520 + depth -= 1
521 + continue
522 + if depth > 0:
523 + count += 1
524 + return count
525 +
526 + # ── Single-pass parse helper ────────────────────────────────────────
527 +
528 + def _single_pass_parse(
529 + self,
530 + text: str,
531 + token_ids: Sequence[int],
532 + initial_state: ParserState | None = None,
533 + ) -> tuple[str | None, str | None, ExtractedToolCallInformation]:
534 + """Reset, feed, finish, and extract results in one pass.
535 +
536 + Must be called as a unit — ``_events_to_delta`` populates tool
537 + state that ``_build_extracted_result`` reads.
538 + """
539 + self._reset(initial_state=initial_state)
540 + events = self._engine.feed(text, token_ids)
541 + events.extend(self._engine.finish())
542 +
543 + delta = self._events_to_delta(events)
544 + tool_call_info = self._build_extracted_result()
545 +
546 + reasoning = delta.reasoning if delta else None
547 + if reasoning and self._strip_trailing_reasoning_ws:
548 + reasoning = reasoning.rstrip() or None
549 +
550 + content = delta.content if delta else None
551 + if content:
552 + content = self._strip_content_whitespace(
553 + content, tool_call_info.tools_called
554 + )
555 +
556 + return reasoning, content, tool_call_info
557 +
558 + # ── Non-streaming: parse ───────────────────────────────────────────
559 +
560 + def parse(
561 + self,
562 + model_output: str,
563 + request: ChatCompletionRequest | ResponsesRequest,
564 + enable_auto_tools: bool = False,
565 + model_output_token_ids: Sequence[int] = (),
566 + ) -> tuple[str | None, str | None, list[FunctionCall] | None]:
567 + reasoning, content, tool_call_info = self._single_pass_parse(
568 + model_output,
569 + model_output_token_ids,
570 + )
571 +
572 + tool_calls: list[FunctionCall] | None = None
573 + if tool_call_info.tools_called:
574 + tool_calls = [
575 + FunctionCall(
576 + id=tc.id,
577 + name=tc.function.name,
578 + arguments=tc.function.arguments,
579 + )
580 + for tc in tool_call_info.tool_calls
581 + ]
582 +
583 + return reasoning, content, tool_calls
584 +
585 + # ── Event-to-delta conversion ─────────────────────────────────────
586 +
587 + def _events_to_delta(
588 + self,
589 + events: list[SemanticEvent],
590 + finished: bool = False,
591 + ) -> DeltaMessage | None:
592 + if not events and not self._deferred_content:
593 + return None
594 +
595 + tool_call_deltas: list[DeltaToolCall] = []
596 + content_parts: list[str] = []
597 + reasoning_parts: list[str] = []
598 +
599 + seen_tool_event = False
600 + for event in events:
601 + match event.type:
602 + case EventType.TEXT_CHUNK:
603 + if seen_tool_event:
604 + self._deferred_content += event.value
605 + else:
606 + content_parts.append(event.value)
607 + case EventType.REASONING_CHUNK:
608 + reasoning_parts.append(event.value)
609 + case EventType.REASONING_END:
610 + self._reasoning_ended = True
611 + case EventType.TOOL_CALL_START:
612 + seen_tool_event = True
613 + self._ensure_slot(event.tool_index)
614 + case EventType.TOOL_NAME:
615 + seen_tool_event = True
616 + self._handle_tool_name(event)
617 + case EventType.ARG_VALUE_CHUNK:
618 + seen_tool_event = True
619 + self._handle_arg_chunk(event, tool_call_deltas)
620 + case EventType.TOOL_CALL_END:
621 + seen_tool_event = True
622 + self._handle_tool_end(event, tool_call_deltas)
623 + case EventType.REASONING_START:
624 + pass # no delta-level effect
625 +
626 + if len(tool_call_deltas) > 1:
627 + tool_call_deltas = self._coalesce_tool_call_deltas(tool_call_deltas)
628 +
629 + if self._deferred_content and not seen_tool_event:
630 + content_parts.insert(0, self._deferred_content)
631 + self._deferred_content = ""
632 +
633 + content_str = "".join(content_parts)
634 +
635 + if self._content_has_nonws:
636 + pass
637 + elif content_str:
638 + stripped = content_str.strip()
639 + if stripped:
640 + self._content_has_nonws = True
641 + elif self._tool_slots:
642 + if self._drop_ws_only_content_before_tools:
643 + content_str = ""
644 + elif not finished:
645 + self._deferred_content = content_str
646 + content_str = ""
647 +
648 + content = content_str or None
649 + reasoning = "".join(reasoning_parts) or None
650 +
651 + if content or tool_call_deltas or reasoning:
652 + kwargs: dict[str, object] = {}
653 + if content is not None:
654 + kwargs["content"] = content
655 + if reasoning is not None:
656 + kwargs["reasoning"] = reasoning
657 + if tool_call_deltas:
658 + kwargs["tool_calls"] = tool_call_deltas
659 + return DeltaMessage(**kwargs)
660 + return None
661 +
662 + def _ensure_slot(self, idx: int) -> None:
663 + while len(self._tool_slots) <= idx:
664 + self._tool_slots.append(ToolCallSlot())
665 +
666 + def _ensure_tool_id(self, slot: ToolCallSlot, name: str) -> None:
667 + if not slot.id:
668 + state = self._stream_state
669 + slot.id = make_tool_call_id(
670 + id_type=state.tool_call_id_type,
671 + func_name=name,
672 + idx=state.history_tool_call_cnt,
673 + )
674 + state.history_tool_call_cnt += 1
675 +
676 + def _handle_tool_name(self, event: SemanticEvent) -> None:
677 + idx = event.tool_index
678 + self._tool_slots[idx].name += event.value
679 +
680 + def _emit_name_delta(
681 + self,
682 + idx: int,
683 + deltas: list[DeltaToolCall],
684 + name: str | None,
685 + ) -> None:
686 + if not name:
687 + return
688 + slot = self._tool_slots[idx]
689 + slot.name = name
690 + slot.name_sent = True
691 + self._ensure_tool_id(slot, name)
692 + deltas.append(
693 + DeltaToolCall(
694 + index=idx,
695 + id=slot.id,
696 + type="function",
697 + function=DeltaFunctionCall(name=name),
698 + )
699 + )
700 +
701 + def _handle_arg_chunk(
702 + self,
703 + event: SemanticEvent,
704 + deltas: list[DeltaToolCall],
705 + ) -> None:
706 + idx = event.tool_index
707 + slot = self._tool_slots[idx]
708 + if event.value:
709 + slot.append_args(event.value)
710 +
711 + if not slot.name_sent:
712 + if slot.name:
713 + self._emit_name_delta(idx, deltas, slot.name)
714 + elif event.value:
715 + # Name not yet known — try to extract from accumulated args
716 + name = self._try_extract_name(idx)
717 + self._emit_name_delta(idx, deltas, name)
718 + elif event.value:
719 + # Name already sent — emit arg delta
720 + arg_delta = self._compute_arg_delta(idx, event.value)
721 + if arg_delta:
722 + deltas.append(
723 + DeltaToolCall(
724 + index=idx,
725 + function=DeltaFunctionCall(arguments=arg_delta),
726 + )
727 + )
728 +
729 + def _handle_tool_end(
730 + self,
731 + event: SemanticEvent,
732 + deltas: list[DeltaToolCall],
733 + ) -> None:
734 + idx = event.tool_index
735 + if idx >= len(self._tool_slots):
736 + return
737 +
738 + remaining = self._flush_arg_converter(idx)
739 + slot = self._tool_slots[idx]
740 +
741 + if not slot.name_sent:
742 + name = slot.name or self._try_extract_name(idx)
743 + if name:
744 + slot.name = name
745 + slot.name_sent = True
746 + self._ensure_tool_id(slot, name)
747 + deltas.append(
748 + DeltaToolCall(
749 + index=idx,
750 + id=slot.id,
751 + type="function",
752 + function=DeltaFunctionCall(
753 + name=name,
754 + arguments=remaining or "",
755 + ),
756 + )
757 + )
758 + remaining = None
759 +
760 + if remaining and slot.name_sent:
761 + deltas.append(
762 + DeltaToolCall(
763 + index=idx,
764 + function=DeltaFunctionCall(arguments=remaining),
765 + )
766 + )
767 +
768 + # ── Tool-call delta coalescing ──────────────────────────────────────
769 +
770 + @staticmethod
771 + def _coalesce_tool_call_deltas(
772 + deltas: list[DeltaToolCall],
773 + ) -> list[DeltaToolCall]:
774 + """Merge entries that share the same index into one per index."""
775 + merged: dict[int, DeltaToolCall] = {}
776 + for tc in deltas:
777 + existing = merged.get(tc.index)
778 + if existing is None:
779 + merged[tc.index] = tc
780 + continue
781 + if tc.id is not None and existing.id is None:
782 + existing.id = tc.id
783 + if tc.type is not None and existing.type is None:
784 + existing.type = tc.type
785 + if tc.function is not None:
786 + if existing.function is None:
787 + existing.function = tc.function
788 + else:
789 + if tc.function.name is not None and existing.function.name is None:
790 + existing.function.name = tc.function.name
791 + if tc.function.arguments is not None:
792 + if existing.function.arguments is None:
793 + existing.function.arguments = tc.function.arguments
794 + else:
795 + existing.function.arguments += tc.function.arguments
796 + if len(merged) == len(deltas):
797 + return deltas
798 + return list(merged.values())
799 +
800 + # ── Arg conversion helpers ─────────────────────────────────────────
801 +
802 + def _compute_arg_delta(self, idx: int, raw_delta: str) -> str | None:
803 + converter = self._arg_converter
804 + if converter is None:
805 + return raw_delta
806 +
807 + if not self._stream_arg_deltas:
808 + return None
809 +
810 + structural = self._arg_structural_chars
811 + if structural is not None and structural.isdisjoint(raw_delta):
812 + return None
813 +
814 + slot = self._tool_slots[idx]
815 + try:
816 + current_json = converter(slot.args, True)
817 + except (json.JSONDecodeError, ValueError, TypeError):
818 + logger.debug("arg converter failed (streaming): %s", slot.args[:80])
819 + return None
820 +
821 + if not current_json:
822 + return None
823 +
824 + if slot.name:
825 + current_json = self._fix_arg_types(current_json, slot.name)
826 +
827 + prev = slot.streamed_json
828 + safe_json = self._safe_arg_prefix(current_json)
829 +
830 + if not safe_json or safe_json == prev:
831 + return None
832 +
833 + if prev:
834 + if not safe_json.startswith(prev):
835 + return None
836 + diff = safe_json[len(prev) :]
837 + else:
838 + diff = safe_json
839 +
840 + if diff:
841 + slot.streamed_json = safe_json
842 + return diff
843 + return None
844 +
845 + def _flush_arg_converter(self, idx: int) -> str | None:
846 + converter = self._arg_converter
847 + if converter is None:
848 + return None
849 +
850 + slot = self._tool_slots[idx]
851 + try:
852 + final_json = converter(slot.args, False)
853 + except (json.JSONDecodeError, ValueError, TypeError):
854 + logger.debug("arg converter failed (flush): %s", slot.args[:80])
855 + return None
856 +
857 + if final_json:
858 + final_json = self._fix_arg_types(final_json, slot.name)
859 +
860 + prev = slot.streamed_json
861 + if final_json and len(final_json) > len(prev):
862 + if prev and not final_json.startswith(prev):
863 + return None
864 + diff = final_json[len(prev) :]
865 + slot.streamed_json = final_json
866 + return diff
867 + return None
868 +
869 + _NAME_RE = re.compile(r'"name"\s*:\s*"([^"]*)"')
870 +
871 + def _try_extract_name(self, idx: int) -> str | None:
872 + m = self._NAME_RE.search(self._tool_slots[idx].args)
873 + if m:
874 + name = m.group(1)
875 + if name:
876 + return name
877 + return None
878 +
879 + # ── Build ExtractedToolCallInformation ─────────────────────────────
880 +
881 + def _build_extracted_result(
882 + self,
883 + *deltas: DeltaMessage | None,
884 + ) -> ExtractedToolCallInformation:
885 + content_parts: list[str] = []
886 + for delta in deltas:
887 + if delta is not None and delta.content:
888 + content_parts.append(delta.content)
889 +
890 + tool_calls: list[ToolCall] = []
891 + for idx, slot in enumerate(self._tool_slots):
892 + if not slot.name and not slot.args:
893 + continue
894 +
895 + name = slot.name.strip()
896 + raw_body = slot.args
897 +
898 + if not name and raw_body.strip():
899 + name, args_json = self._extract_name_and_args(raw_body)
900 + elif raw_body.strip():
901 + converter = self._arg_converter
902 + if converter is not None:
903 + try:
904 + args_json = converter(raw_body, False)
905 + except (json.JSONDecodeError, ValueError, TypeError):
906 + logger.debug(
907 + "arg converter failed (extract): %s", raw_body[:80]
908 + )
909 + args_json = self._extract_args_json(raw_body, name)
910 + else:
911 + args_json = self._extract_args_json(raw_body, name)
912 + else:
913 + args_json = "{}"
914 +
915 + if name:
916 + self._ensure_tool_id(slot, name)
917 + args_json = self._fix_arg_types(args_json, name)
918 + tool_calls.append(
919 + ToolCall(
920 + id=slot.id,
921 + function=FunctionCall(name=name, arguments=args_json),
922 + )
923 + )
924 +
925 + content_str = "".join(content_parts)
926 + content = self._strip_content_whitespace(content_str, len(tool_calls) > 0)
927 +
928 + return ExtractedToolCallInformation(
929 + tools_called=len(tool_calls) > 0,
930 + tool_calls=tool_calls,
931 + content=content,
932 + )
933 +
934 + @staticmethod
935 + def _extract_args_value(parsed: dict) -> str | None:
936 + for key in ("arguments", "parameters"):
937 + if key in parsed:
938 + val = parsed[key]
939 + if isinstance(val, str):
940 + return val
941 + return json.dumps(val, ensure_ascii=False)
942 + return None
943 +
944 + def _extract_name_and_args(
945 + self,
946 + raw_body: str,
947 + ) -> tuple[str, str]:
948 + raw_body = raw_body.strip()
949 + try:
950 + parsed = json.loads(raw_body)
951 + except json.JSONDecodeError:
952 + return "", raw_body
953 +
954 + if not isinstance(parsed, dict):
955 + return "", raw_body
956 +
957 + name = parsed.get("name", "")
958 + args = self._extract_args_value(parsed)
959 + if args is not None:
960 + return name, args
961 +
962 + without_name = {k: v for k, v in parsed.items() if k != "name"}
963 + return name, json.dumps(without_name, ensure_ascii=False)
964 +
965 + def _extract_args_json(self, raw_args: str, func_name: str) -> str:
966 + if not raw_args.strip():
967 + return "{}"
968 + _, args = self._extract_name_and_args(raw_args)
969 + return args
vllm/parser/engine/parser_engine_config.py new
+114
@@ -0,0 +1,114 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Declarative configuration for model tool-call and reasoning formats.
4 +
5 +Each model format is described by a :class:`ParserEngineConfig` that specifies:
6 +
7 +* **terminals** – literal strings or regex patterns that delimit the format
8 + (e.g. ``<tool_call>``, ``</think>``).
9 +* **token_id_terminals** – terminals that should be matched by token ID
10 + rather than (or in addition to) text.
11 +* **transitions** – a state machine mapping
12 + ``(state, terminal) → (new_state, events_to_emit)`` that drives semantic
13 + event generation during streaming.
14 +* **content_events** – what :class:`EventType` to emit for plain content
15 + (non-terminal text) in each state.
16 +"""
17 +
18 +from __future__ import annotations
19 +
20 +from collections.abc import Callable
21 +from dataclasses import dataclass, field
22 +from enum import Enum, auto
23 +from functools import cached_property
24 +
25 +from vllm.parser.engine.events import EventType
26 +
27 +STRUCTURAL_DROP_TOKENS: frozenset[str] = frozenset(
28 + {
29 + "<eos>",
30 + "<bos>",
31 + "<pad>",
32 + "<unk>",
33 + "<mask>",
34 + }
35 +)
36 +
37 +
38 +class ParserState(Enum):
39 + CONTENT = auto()
40 + REASONING = auto()
41 + TOOL_PREAMBLE = auto()
42 + TOOL_NAME = auto()
43 + TOOL_ARGS = auto()
44 + TOOL_BETWEEN = auto()
45 +
46 +
47 +@dataclass(frozen=True, slots=True)
48 +class Transition:
49 + next_state: ParserState
50 + events: tuple[EventType, ...] = field(default_factory=tuple)
51 + skip_in_token_id_mode: bool = False
52 +
53 +
54 +@dataclass(frozen=True)
55 +class ParserEngineConfig:
56 + """Declarative description of a model's tool-call / reasoning format.
57 +
58 + The engine feeds terminals from the incremental lexer into the
59 + transition table and emits the corresponding semantic events.
60 + Content tokens (text between terminals) are classified by the
61 + current state via ``content_events``.
62 + """
63 +
64 + name: str
65 +
66 + terminals: dict[str, str] = field(default_factory=dict)
67 +
68 + token_id_terminals: dict[str, str] = field(default_factory=dict)
69 +
70 + transitions: dict[tuple[ParserState, str], Transition] = field(
71 + default_factory=dict,
72 + )
73 +
74 + content_events: dict[ParserState, EventType] = field(
75 + default_factory=lambda: {
76 + ParserState.CONTENT: EventType.TEXT_CHUNK,
77 + ParserState.REASONING: EventType.REASONING_CHUNK,
78 + ParserState.TOOL_NAME: EventType.TOOL_NAME,
79 + ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
80 + },
81 + )
82 +
83 + initial_state: ParserState = ParserState.CONTENT
84 +
85 + arg_converter: Callable[[str, bool], str] | None = None
86 +
87 + stream_arg_deltas: bool = True
88 +
89 + tool_args_json: bool = True
90 +
91 + arg_structural_chars: frozenset[str] | None = None
92 +
93 + # Prevents trailing-whitespace accumulation across multi-turn conversations.
94 + strip_trailing_reasoning_whitespace: bool = True
95 +
96 + # Drop content that is entirely whitespace when tool calls follow.
97 + drop_whitespace_only_content_before_tools: bool = True
98 +
99 + # .strip() content text when tool calls are present.
100 + strip_content_whitespace_with_tools: bool = True
101 +
102 + drop_tokens: frozenset[str] = field(default_factory=frozenset)
103 +
104 + @cached_property
105 + def terminal_defs(self):
106 + from vllm.parser.engine.incremental_lexer import terminals_from_literals
107 +
108 + return terminals_from_literals(self.terminals)
109 +
110 + @cached_property
111 + def lexer_shape(self):
112 + from vllm.parser.engine.incremental_lexer import LexerShape
113 +
114 + return LexerShape(self.terminal_defs)
vllm/parser/engine/registered_adapters.py new
+16
@@ -0,0 +1,16 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Concrete adapter classes for each registered parser engine.
4 +
5 +These are created via :func:`make_adapters` and exposed as module-level
6 +names so that :class:`ReasoningParserManager` and
7 +:class:`ToolParserManager` can load them lazily.
8 +"""
9 +
10 +from vllm.parser.engine.adapters import make_adapters
11 +from vllm.parser.qwen3 import Qwen3Parser
12 +
13 +(
14 + Qwen3ParserReasoningAdapter,
15 + Qwen3ParserToolAdapter,
16 +) = make_adapters(Qwen3Parser)
vllm/parser/engine/streaming_parser_engine.py new
+408
@@ -0,0 +1,408 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Streaming parser engine that orchestrates token ID scanning,
4 +incremental lexing, and state-machine-driven semantic event emission."""
5 +
6 +from __future__ import annotations
7 +
8 +from collections.abc import Sequence
9 +
10 +from vllm.parser.engine.events import EventType, SemanticEvent
11 +from vllm.parser.engine.incremental_lexer import (
12 + CONTENT_TERMINAL,
13 + IncrementalLexer,
14 + LexToken,
15 +)
16 +from vllm.parser.engine.parser_engine_config import (
17 + STRUCTURAL_DROP_TOKENS,
18 + ParserEngineConfig,
19 + ParserState,
20 + Transition,
21 +)
22 +from vllm.parser.engine.token_id_scanner import (
23 + LexerInput,
24 + PreLexedTerminal,
25 + TextChunk,
26 + TokenIDScanner,
27 +)
28 +
29 +
30 +class StreamingParserEngine:
31 + """Consumes ``(delta_text, delta_token_ids)`` pairs and produces a
32 + stream of :class:`SemanticEvent` instances.
33 +
34 + This is the main entry point for streaming parsing.
35 + Create one per request (it is stateful).
36 +
37 + The pipeline is::
38 +
39 + delta_text + delta_token_ids
40 + → TokenIDScanner (special token pre-lexing)
41 + → IncrementalLexer (text → terminal tokens with prefix buffering)
42 + → State Machine (terminal → semantic events)
43 + → list[SemanticEvent]
44 +
45 + Usage::
46 +
47 + engine = StreamingParserEngine(config, tokenizer)
48 + for each streaming delta:
49 + events = engine.feed(delta_text, delta_token_ids)
50 + # convert events to DeltaMessage
51 + """
52 +
53 + def __init__(
54 + self,
55 + config: ParserEngineConfig,
56 + tokenizer,
57 + initial_state: ParserState | None = None,
58 + vocab: dict[str, int] | None = None,
59 + ) -> None:
60 + self.config = config
61 +
62 + resolved_token_ids: dict[int, str] = {}
63 + drop_token_ids: set[int] = set()
64 + if tokenizer is not None:
65 + if vocab is None:
66 + vocab = tokenizer.get_vocab()
67 + if config.token_id_terminals:
68 + for terminal_name, token_text in config.token_id_terminals.items():
69 + tid = vocab.get(token_text)
70 + if tid is not None:
71 + resolved_token_ids[tid] = terminal_name
72 + all_drop = config.drop_tokens | STRUCTURAL_DROP_TOKENS
73 + for token_text in all_drop:
74 + tid = vocab.get(token_text)
75 + if tid is not None:
76 + drop_token_ids.add(tid)
77 + for attr in ("eos_token_id", "bos_token_id", "pad_token_id"):
78 + tid = getattr(tokenizer, attr, None)
79 + if tid is not None:
80 + drop_token_ids.add(tid)
81 +
82 + self._resolved_token_ids = resolved_token_ids
83 + self._drop_token_ids = drop_token_ids
84 +
85 + self._scanner = TokenIDScanner(
86 + resolved_token_ids,
87 + tokenizer,
88 + drop_token_ids,
89 + )
90 +
91 + self._token_id_terminal_names: frozenset[str] = frozenset(
92 + resolved_token_ids.values()
93 + )
94 +
95 + self._lexer = IncrementalLexer(
96 + config.lexer_shape, content_terminal=CONTENT_TERMINAL
97 + )
98 +
99 + self._tool_terminals: frozenset[str] = frozenset(
100 + terminal
101 + for (state, terminal), tr in config.transitions.items()
102 + if tr.next_state in self._TOOL_STATES or state in self._TOOL_STATES
103 + )
104 +
105 + self.skip_tool_parsing = False
106 + self.reset(initial_state=initial_state)
107 +
108 + def _reset_args_state(self) -> None:
109 + self._args_buffer: str = ""
110 + self._args_safe_end: int = 0
111 + self._args_brace_depth: int = 0
112 + self._args_in_string: bool = False
113 + self._args_escape_next: bool = False
114 +
115 + def reset(self, initial_state: ParserState | None = None) -> None:
116 + """Reset mutable state for reuse across requests.
117 +
118 + Preserves cached immutable structures (compiled terminals,
119 + resolved token IDs, lexer shape, token text cache) to avoid
120 + redundant initialization work.
121 + """
122 + self.state = (
123 + initial_state if initial_state is not None else self.config.initial_state
124 + )
125 + self.tool_index = -1
126 + self._ever_had_token_ids = False
127 + # DO NOT reset skip_tool_parsing here — callers set it before
128 + # calling methods that trigger reset() (e.g. extract_reasoning),
129 + # and clearing it silently breaks non-streaming tool-call-as-
130 + # implicit-reasoning-end (content returns None).
131 + self._scanner.reset()
132 + self._lexer.reset()
133 + self._reset_args_state()
134 +
135 + def feed(
136 + self,
137 + delta_text: str,
138 + delta_token_ids: Sequence[int],
139 + ) -> list[SemanticEvent]:
140 + if delta_token_ids:
141 + self._ever_had_token_ids = True
142 +
143 + # Fast path: skip scanner and lexer when the delta is plain
144 + # content with no special tokens and no terminal-starting chars.
145 + if (
146 + delta_text
147 + and not self._lexer.buffer
148 + and not self._scanner._deferred_terminals
149 + and self._lexer._literal_first_chars.isdisjoint(delta_text)
150 + ):
151 + has_special = False
152 + for tid in delta_token_ids:
153 + if tid in self._resolved_token_ids or tid in self._drop_token_ids:
154 + has_special = True
155 + break
156 + if not has_special:
157 + return self._emit_for_state(delta_text)
158 +
159 + scanner_items = self._scanner.scan(delta_text, delta_token_ids)
160 +
161 + if len(scanner_items) == 1 and isinstance(scanner_items[0], TextChunk):
162 + lex_tokens = self._lexer.feed(scanner_items[0].text)
163 + if len(lex_tokens) == 1 and lex_tokens[0].terminal == CONTENT_TERMINAL:
164 + text = lex_tokens[0].value
165 + return self._emit_for_state(text)
166 + return self._process_lex_tokens(lex_tokens)
167 +
168 + return self._process_scanner_items(scanner_items)
169 +
170 + def _process_scanner_items(
171 + self, items: Sequence[LexerInput]
172 + ) -> list[SemanticEvent]:
173 + events: list[SemanticEvent] = []
174 + for item in items:
175 + if isinstance(item, PreLexedTerminal):
176 + events.extend(self._process_lex_tokens(self._lexer.flush()))
177 + events.extend(self._on_terminal(item.terminal, item.text))
178 + elif isinstance(item, TextChunk):
179 + events.extend(self._process_lex_tokens(self._lexer.feed(item.text)))
180 + return events
181 +
182 + def finish(self) -> list[SemanticEvent]:
183 + events = self._process_scanner_items(self._scanner.flush_pending())
184 +
185 + events.extend(self._process_lex_tokens(self._lexer.flush()))
186 +
187 + if self._args_buffer:
188 + events.append(
189 + SemanticEvent(
190 + EventType.ARG_VALUE_CHUNK,
191 + value=self._args_buffer,
192 + tool_index=self.tool_index,
193 + )
194 + )
195 + self._args_buffer = ""
196 + self._args_safe_end = 0
197 +
198 + if self.state in (
199 + ParserState.TOOL_PREAMBLE,
200 + ParserState.TOOL_ARGS,
201 + ParserState.TOOL_NAME,
202 + ParserState.TOOL_BETWEEN,
203 + ):
204 + if self.tool_index >= 0:
205 + events.append(
206 + SemanticEvent(
207 + EventType.TOOL_CALL_END,
208 + tool_index=self.tool_index,
209 + )
210 + )
211 + self.state = ParserState.CONTENT
212 + elif self.state == ParserState.REASONING:
213 + events.append(
214 + SemanticEvent(EventType.REASONING_END, tool_index=self.tool_index)
215 + )
216 + self.state = ParserState.CONTENT
217 +
218 + return events
219 +
220 + def parse_complete(self, text: str) -> list[SemanticEvent]:
221 + token_ids: list[int] = []
222 + events = self.feed(text, token_ids)
223 + events.extend(self.finish())
224 + return events
225 +
226 + def _process_lex_tokens(self, tokens: list[LexToken]) -> list[SemanticEvent]:
227 + events: list[SemanticEvent] = []
228 + strict = self._token_id_terminal_names if self._ever_had_token_ids else None
229 + for tok in tokens:
230 + if tok.terminal == CONTENT_TERMINAL or (strict and tok.terminal in strict):
231 + events.extend(self._on_content(tok.value))
232 + else:
233 + events.extend(self._on_terminal(tok.terminal, tok.value))
234 + return events
235 +
236 + _TOOL_STATES = frozenset(
237 + {
238 + ParserState.TOOL_PREAMBLE,
239 + ParserState.TOOL_NAME,
240 + ParserState.TOOL_ARGS,
241 + ParserState.TOOL_BETWEEN,
242 + }
243 + )
244 +
245 + def _on_terminal(self, terminal: str, value: str) -> list[SemanticEvent]:
246 + key = (self.state, terminal)
247 + transition = self.config.transitions.get(key)
248 +
249 + if transition is None:
250 + return self._emit_for_state(value)
251 +
252 + if self.skip_tool_parsing and terminal in self._tool_terminals:
253 + if EventType.REASONING_END in transition.events:
254 + self.state = ParserState.CONTENT
255 + return [
256 + SemanticEvent(
257 + EventType.REASONING_END,
258 + value=value,
259 + tool_index=self.tool_index,
260 + ),
261 + SemanticEvent(
262 + EventType.TEXT_CHUNK,
263 + value=value,
264 + tool_index=self.tool_index,
265 + ),
266 + ]
267 + content_type = self.config.content_events.get(self.state)
268 + if content_type is not None:
269 + return [
270 + SemanticEvent(content_type, value=value, tool_index=self.tool_index)
271 + ]
272 + return []
273 +
274 + if transition.skip_in_token_id_mode and self._ever_had_token_ids:
275 + return self._emit_for_state(value)
276 +
277 + return self._apply_transition(transition, value)
278 +
279 + def _emit_for_state(self, text: str) -> list[SemanticEvent]:
280 + if self.state == ParserState.TOOL_ARGS:
281 + if self.config.tool_args_json:
282 + return self._feed_args_text(text)
283 + return [
284 + SemanticEvent(
285 + EventType.ARG_VALUE_CHUNK,
286 + value=text,
287 + tool_index=self.tool_index,
288 + )
289 + ]
290 + content_type = self.config.content_events.get(self.state)
291 + if content_type is not None:
292 + return [SemanticEvent(content_type, value=text, tool_index=self.tool_index)]
293 + return []
294 +
295 + def _on_content(self, text: str) -> list[SemanticEvent]:
296 + if not text:
297 + return []
298 + return self._emit_for_state(text)
299 +
300 + def _apply_transition(
301 + self,
302 + transition: Transition,
303 + value: str,
304 + ) -> list[SemanticEvent]:
305 + events: list[SemanticEvent] = []
306 +
307 + if (
308 + self.state == ParserState.TOOL_ARGS
309 + and transition.next_state != ParserState.TOOL_ARGS
310 + and self._args_buffer
311 + ):
312 + events.append(
313 + SemanticEvent(
314 + EventType.ARG_VALUE_CHUNK,
315 + value=self._args_buffer,
316 + tool_index=self.tool_index,
317 + )
318 + )
319 + self._args_buffer = ""
320 +
321 + self.state = transition.next_state
322 +
323 + for event_type in transition.events:
324 + if event_type == EventType.TOOL_CALL_START:
325 + self.tool_index += 1
326 + events.append(
327 + SemanticEvent(
328 + event_type,
329 + value=value,
330 + tool_index=self.tool_index,
331 + )
332 + )
333 +
334 + if self.state == ParserState.TOOL_ARGS:
335 + self._args_brace_depth = 0
336 + self._args_in_string = False
337 + self._args_escape_next = False
338 + self._args_safe_end = 0
339 +
340 + return events
341 +
342 + def _feed_args_text(self, text: str) -> list[SemanticEvent]:
343 + """Feed text into the JSON argument streaming buffer.
344 +
345 + Streams argument characters incrementally while holding back
346 + closing braces/brackets that might change as more input arrives.
347 + """
348 + events: list[SemanticEvent] = []
349 + for ch in text:
350 + result = self._feed_args_char(ch)
351 + events.extend(result)
352 + return events
353 +
354 + def _feed_args_char(self, ch: str) -> list[SemanticEvent]:
355 + self._args_buffer += ch
356 +
357 + if self._args_escape_next:
358 + self._args_escape_next = False
359 + self._args_safe_end = len(self._args_buffer)
360 + return self._flush_safe_args()
361 +
362 + if self._args_in_string:
363 + if ch == "\\":
364 + self._args_escape_next = True
365 + elif ch == '"':
366 + self._args_in_string = False
367 + self._args_safe_end = len(self._args_buffer)
368 + return self._flush_safe_args()
369 +
370 + if ch == '"':
371 + self._args_in_string = True
372 + self._args_safe_end = len(self._args_buffer)
373 + return self._flush_safe_args()
374 +
375 + if ch in ("{", "["):
376 + self._args_brace_depth += 1
377 + self._args_safe_end = len(self._args_buffer)
378 + return self._flush_safe_args()
379 +
380 + if ch in ("}", "]"):
381 + if self._args_brace_depth > 0:
382 + self._args_brace_depth -= 1
383 + if self._args_brace_depth == 0:
384 + return []
385 + self._args_safe_end = len(self._args_buffer)
386 + return self._flush_safe_args()
387 +
388 + self._args_safe_end = len(self._args_buffer)
389 + return self._flush_safe_args()
390 +
391 + def _flush_safe_args(self) -> list[SemanticEvent]:
392 + """Emit buffered argument characters up to the safe-end watermark.
393 +
394 + Top-level closing braces are held back (safe_end not advanced)
395 + until confirmed safe by a subsequent character or finish().
396 + """
397 + if self._args_safe_end == 0:
398 + return []
399 + to_emit = self._args_buffer[: self._args_safe_end]
400 + self._args_buffer = self._args_buffer[self._args_safe_end :]
401 + self._args_safe_end = 0
402 + return [
403 + SemanticEvent(
404 + EventType.ARG_VALUE_CHUNK,
405 + value=to_emit,
406 + tool_index=self.tool_index,
407 + )
408 + ]
vllm/parser/engine/token_id_scanner.py new
+309
@@ -0,0 +1,309 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Scan delta token IDs for special tokens and split the stream into
4 +pre-lexed terminals and plain text chunks."""
5 +
6 +from __future__ import annotations
7 +
8 +from collections.abc import Sequence
9 +from dataclasses import dataclass
10 +
11 +
12 +@dataclass(slots=True)
13 +class TextChunk:
14 + text: str
15 +
16 +
17 +@dataclass(slots=True)
18 +class PreLexedTerminal:
19 + terminal: str
20 + token_id: int
21 + text: str
22 +
23 +
24 +LexerInput = TextChunk | PreLexedTerminal
25 +
26 +
27 +class TokenIDScanner:
28 + """Maps special token IDs in the delta to terminals.
29 +
30 + Before text-based lexing happens, the scanner checks each token ID
31 + in the delta against a mapping of ``{token_id: terminal_name}``.
32 + Matched tokens are emitted as :class:`PreLexedTerminal` items;
33 + everything else is grouped into :class:`TextChunk` items for the
34 + incremental lexer to process.
35 +
36 + When a terminal's text is not yet in ``delta_text`` (held back by
37 + the detokenizer), the terminal is deferred until the text arrives
38 + in a subsequent delta.
39 + """
40 +
41 + def __init__(
42 + self,
43 + token_id_to_terminal: dict[int, str],
44 + tokenizer,
45 + drop_token_ids: set[int] | None = None,
46 + ) -> None:
47 + self.token_id_to_terminal = token_id_to_terminal
48 + self.tokenizer = tokenizer
49 + self._token_text_cache: dict[int, str] = {}
50 + self._drop_token_ids = drop_token_ids or set()
51 + self._deferred_terminals: list[PreLexedTerminal] = []
52 + self._deferred_post_text: str = ""
53 +
54 + def reset(self) -> None:
55 + """Clear mutable state for reuse. Preserves the token text cache."""
56 + self._deferred_terminals.clear()
57 + self._deferred_post_text = ""
58 +
59 + def _decode_token(self, token_id: int) -> str:
60 + if token_id not in self._token_text_cache:
61 + self._token_text_cache[token_id] = self.tokenizer.decode([token_id])
62 + return self._token_text_cache[token_id]
63 +
64 + _EMPTY: tuple[LexerInput, ...] = ()
65 +
66 + def scan(
67 + self,
68 + delta_text: str,
69 + delta_token_ids: Sequence[int],
70 + ) -> Sequence[LexerInput]:
71 + prefix_items: list[LexerInput] = []
72 + effective_text = delta_text
73 +
74 + if self._deferred_terminals:
75 + prefix_items, effective_text = self._resolve_deferred(delta_text)
76 +
77 + if not self.token_id_to_terminal and not self._drop_token_ids:
78 + if effective_text:
79 + prefix_items.append(TextChunk(effective_text))
80 + return prefix_items
81 +
82 + has_special = False
83 + has_drop = False
84 + token_id_to_terminal = self.token_id_to_terminal
85 + drop_token_ids = self._drop_token_ids
86 + for tid in delta_token_ids:
87 + if tid in token_id_to_terminal:
88 + has_special = True
89 + if tid in drop_token_ids:
90 + has_drop = True
91 +
92 + if not has_special and not has_drop:
93 + if effective_text:
94 + if not prefix_items:
95 + return [TextChunk(effective_text)]
96 + prefix_items.append(TextChunk(effective_text))
97 + return prefix_items or self._EMPTY
98 +
99 + token_texts = [self._decode_token(tid) for tid in delta_token_ids]
100 +
101 + results: list[LexerInput] = []
102 + text_accum: list[str] = []
103 +
104 + for idx, tid in enumerate(delta_token_ids):
105 + if tid in self._drop_token_ids:
106 + continue
107 + terminal = self.token_id_to_terminal.get(tid)
108 + if terminal is not None:
109 + if text_accum:
110 + joined = "".join(text_accum)
111 + if joined:
112 + results.append(TextChunk(joined))
113 + text_accum.clear()
114 + results.append(PreLexedTerminal(terminal, tid, token_texts[idx]))
115 + else:
116 + text_accum.append(token_texts[idx])
117 +
118 + if text_accum:
119 + joined = "".join(text_accum)
120 + if joined:
121 + results.append(TextChunk(joined))
122 +
123 + if effective_text:
124 + if has_drop:
125 + clean_delta = effective_text
126 + for idx, tid in enumerate(delta_token_ids):
127 + if tid in self._drop_token_ids:
128 + dropped = token_texts[idx]
129 + pos = clean_delta.find(dropped)
130 + if pos >= 0:
131 + clean_delta = (
132 + clean_delta[:pos] + clean_delta[pos + len(dropped) :]
133 + )
134 + if clean_delta:
135 + if results:
136 + results = self._recover_holdback_text(clean_delta, results)
137 + else:
138 + results = [TextChunk(clean_delta)]
139 + else:
140 + results = self._recover_holdback_text(effective_text, results)
141 + else:
142 + # No detokenizer text to validate against — individually-decoded
143 + # TextChunks are unreliable (context-dependent decoding).
144 + # Defer PreLexedTerminals so the state machine doesn't
145 + # transition before the preceding text has arrived. The
146 + # deferred terminals will be resolved against the actual
147 + # delta_text in a subsequent scan() or flushed by finish().
148 + for r in results:
149 + if isinstance(r, PreLexedTerminal):
150 + self._deferred_terminals.append(r)
151 + results = []
152 +
153 + return prefix_items + results
154 +
155 + def flush_pending(self) -> list[LexerInput]:
156 + if not self._deferred_terminals and not self._deferred_post_text:
157 + return []
158 + results: list[LexerInput] = []
159 + if self._deferred_post_text:
160 + results.append(TextChunk(self._deferred_post_text))
161 + self._deferred_post_text = ""
162 + results.extend(self._deferred_terminals)
163 + self._deferred_terminals.clear()
164 + return results
165 +
166 + def _resolve_deferred(
167 + self,
168 + delta_text: str,
169 + ) -> tuple[list[LexerInput], str]:
170 + """Resolve deferred terminals against new delta_text.
171 +
172 + When a previous ``scan()`` deferred a terminal (its text hadn't
173 + arrived yet), the next delta's text should contain that terminal's
174 + text. Split delta_text at the terminal boundary: text before
175 + belongs to the previous parser state, the terminal triggers the
176 + state transition, and text after belongs to the new state.
177 +
178 + Returns ``(prefix_items, remaining_text)`` where prefix_items
179 + are the resolved deferred terminals (with any preceding text)
180 + and remaining_text is the unconsumed portion of delta_text that
181 + should be scanned with the current delta's token IDs.
182 + """
183 + deferred = self._deferred_terminals
184 + self._deferred_terminals = []
185 +
186 + results: list[LexerInput] = []
187 + remaining = delta_text
188 +
189 + if self._deferred_post_text:
190 + remaining = self._deferred_post_text + remaining
191 + self._deferred_post_text = ""
192 +
193 + # Duplicate-text deferred terminals resolve left-to-right via
194 + # find(); correct when each terminal text appears once in sequence.
195 + for terminal in deferred:
196 + pos = remaining.find(terminal.text)
197 + if pos > 0:
198 + results.append(TextChunk(remaining[:pos]))
199 + results.append(terminal)
200 + remaining = remaining[pos + len(terminal.text) :]
201 + elif pos == 0:
202 + results.append(terminal)
203 + remaining = remaining[len(terminal.text) :]
204 + else:
205 + # Accumulate text until terminal text arrives —
206 + # only the terminal provides a reliable split point.
207 + if remaining:
208 + self._deferred_post_text += remaining
209 + remaining = ""
210 + self._deferred_terminals.append(terminal)
211 +
212 + return results, remaining
213 +
214 + def _recover_holdback_text(
215 + self,
216 + delta_text: str,
217 + results: list[LexerInput],
218 + ) -> list[LexerInput]:
219 + """Recover detokenizer hold-back text not in delta_token_ids.
220 +
221 + The detokenizer may flush previously held-back text in
222 + ``delta_text`` that has no corresponding token ID in
223 + ``delta_token_ids``. This hold-back text always appears as a
224 + prefix of ``delta_text``.
225 + """
226 + if not results:
227 + return [TextChunk(delta_text)]
228 +
229 + reconstructed = self._join_decoded_text(results)
230 +
231 + if not reconstructed:
232 + return [TextChunk(delta_text)] + results
233 +
234 + pos = delta_text.find(reconstructed)
235 + if pos > 0:
236 + return [TextChunk(delta_text[:pos])] + results
237 + if pos == 0:
238 + return results
239 +
240 + # Fallback: SentencePiece context-dependent decoding mismatch.
241 + # Rebuild from delta_text using PreLexedTerminals as split anchors.
242 + return self._rebuild_from_anchors(delta_text, results)
243 +
244 + def _join_decoded_text(self, results: list[LexerInput]) -> str:
245 + """Join TextChunk and PreLexedTerminal text into one string."""
246 + parts: list[str] = []
247 + for item in results:
248 + if isinstance(item, (TextChunk, PreLexedTerminal)):
249 + parts.append(item.text)
250 + return "".join(parts)
251 +
252 + def _rebuild_from_anchors(
253 + self,
254 + delta_text: str,
255 + results: list[LexerInput],
256 + ) -> list[LexerInput]:
257 + """Rebuild results from delta_text using terminals as anchors.
258 +
259 + When context-dependent decoding creates a mismatch between
260 + individually-decoded tokens and delta_text, use
261 + PreLexedTerminals as split points and reallocate text from
262 + delta_text. If a terminal's text is not found in delta_text,
263 + it is deferred to the next scan() call.
264 +
265 + Anchors are resolved right-to-left with ``rfind`` so that each
266 + anchor binds to the *rightmost* available occurrence of its
267 + text. This prevents earlier literal lookalikes (e.g. a user
268 + mentioning ``<tool_call>`` in prose) from stealing the position
269 + of a real special-token anchor that appears later.
270 +
271 + If the same anchor text appears multiple times as real special
272 + tokens (not prose), the rightmost-first binding could misalign.
273 + In practice this doesn't happen: each special token ID maps to
274 + a distinct PreLexedTerminal, and duplicates in prose are resolved
275 + by the token-ID filtering layer above.
276 + """
277 + anchors = [item for item in results if isinstance(item, PreLexedTerminal)]
278 + if not anchors:
279 + return [TextChunk(delta_text)]
280 +
281 + # Resolve positions right-to-left: each anchor gets the
282 + # rightmost occurrence that is still before the next anchor.
283 + positions: list[int] = [-1] * len(anchors)
284 + search_end = len(delta_text)
285 + for i in range(len(anchors) - 1, -1, -1):
286 + pos = delta_text.rfind(anchors[i].text, 0, search_end)
287 + if pos >= 0:
288 + positions[i] = pos
289 + search_end = pos
290 +
291 + # Build results left-to-right using the resolved positions.
292 + new_results: list[LexerInput] = []
293 + consumed = 0
294 + for i, anchor in enumerate(anchors):
295 + pos = positions[i]
296 + if pos >= consumed:
297 + if pos > consumed:
298 + new_results.append(TextChunk(delta_text[consumed:pos]))
299 + new_results.append(anchor)
300 + consumed = pos + len(anchor.text)
301 + else:
302 + has_later_valid = any(p >= 0 for p in positions[i + 1 :])
303 + if not has_later_valid and consumed < len(delta_text):
304 + self._deferred_post_text += delta_text[consumed:]
305 + consumed = len(delta_text)
306 + self._deferred_terminals.append(anchor)
307 + if consumed < len(delta_text):
308 + new_results.append(TextChunk(delta_text[consumed:]))
309 + return new_results
vllm/parser/qwen3.py new
+218
@@ -0,0 +1,218 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Qwen3 parser for tool calls and reasoning.
4 +
5 +Qwen3 XML tool call format::
6 +
7 + <tool_call>
8 + <function=func_name>
9 + <parameter=key>value</parameter>
10 + </function>
11 + </tool_call>
12 +
13 +The argument body consists of ``<parameter=NAME>VALUE</parameter>`` tags.
14 +The ``_qwen3_arg_converter`` parses these into a JSON object.
15 +"""
16 +
17 +from __future__ import annotations
18 +
19 +import functools
20 +import json
21 +from typing import TYPE_CHECKING
22 +
23 +import regex as re
24 +
25 +from vllm.parser.engine.events import EventType
26 +from vllm.parser.engine.parser_engine import ParserEngine
27 +from vllm.parser.engine.parser_engine_config import (
28 + ParserEngineConfig,
29 + ParserState,
30 + Transition,
31 +)
32 +
33 +if TYPE_CHECKING:
34 + from vllm.entrypoints.openai.chat_completion.protocol import (
35 + ChatCompletionRequest,
36 + )
37 + from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
38 + from vllm.tokenizers import TokenizerLike
39 + from vllm.tool_parsers.abstract_tool_parser import Tool
40 +
41 +TOOL_CALL_START = "<tool_call>"
42 +TOOL_CALL_END = "</tool_call>"
43 +FUNC_PREFIX = "<function="
44 +FUNC_END = "</function>"
45 +
46 +_PARAM_RE = re.compile(
47 + r"<\s*parameter\s*=\s*([^>]*)>"
48 + r"(.*?)"
49 + r"(?:<\s*/\s*parameter\s*>|(?=<\s*parameter\s*=))",
50 + re.DOTALL,
51 +)
52 +_PARTIAL_PARAM_RE = re.compile(r"<\s*parameter\s*=\s*([^>]+)>([^<]*)$", re.DOTALL)
53 +
54 +
55 +def _qwen3_arg_converter(raw_args: str, partial: bool) -> str:
56 + params: dict[str, object] = {}
57 +
58 + for match in _PARAM_RE.finditer(raw_args):
59 + name = match.group(1)
60 + value = match.group(2)
61 + params[name] = value.strip()
62 +
63 + if partial:
64 + remaining = _PARAM_RE.sub("", raw_args)
65 + m = _PARTIAL_PARAM_RE.search(remaining)
66 + if m:
67 + name = m.group(1)
68 + value = m.group(2)
69 + if name:
70 + params[name] = value
71 +
72 + return json.dumps(params, ensure_ascii=False)
73 +
74 +
75 +@functools.cache
76 +def qwen3_config(thinking: bool = True) -> ParserEngineConfig:
77 + return ParserEngineConfig(
78 + name="qwen3",
79 + initial_state=ParserState.REASONING if thinking else ParserState.CONTENT,
80 + terminals={
81 + # Reasoning terminals
82 + "THINK_START": "<think>",
83 + "THINK_END": "</think>",
84 + # Tool call terminals
85 + "TOOL_START": TOOL_CALL_START,
86 + "TOOL_END": TOOL_CALL_END,
87 + "FUNC_PREFIX": FUNC_PREFIX,
88 + "FUNC_END": FUNC_END,
89 + "CLOSE_ANGLE": ">",
90 + },
91 + token_id_terminals={
92 + "THINK_START": "<think>",
93 + "THINK_END": "</think>",
94 + "TOOL_START": TOOL_CALL_START,
95 + "TOOL_END": TOOL_CALL_END,
96 + },
97 + transitions={
98 + # -- Reasoning transitions --
99 + (ParserState.REASONING, "THINK_START"): Transition(
100 + ParserState.REASONING,
101 + (),
102 + ),
103 + (ParserState.REASONING, "THINK_END"): Transition(
104 + ParserState.CONTENT,
105 + (EventType.REASONING_END,),
106 + ),
107 + # Absorb duplicate </think> — model may emit it after
108 + # already transitioning to CONTENT; drop it silently.
109 + (ParserState.CONTENT, "THINK_END"): Transition(
110 + ParserState.CONTENT,
111 + (),
112 + ),
113 + # Tool call directly from reasoning (implicit end)
114 + (ParserState.REASONING, "TOOL_START"): Transition(
115 + ParserState.TOOL_PREAMBLE,
116 + (EventType.REASONING_END, EventType.TOOL_CALL_START),
117 + ),
118 + # -- Tool call transitions --
119 + (ParserState.CONTENT, "TOOL_START"): Transition(
120 + ParserState.TOOL_PREAMBLE,
121 + (EventType.TOOL_CALL_START,),
122 + ),
123 + # Fallback: <function= without a preceding <tool_call>
124 + (ParserState.CONTENT, "FUNC_PREFIX"): Transition(
125 + ParserState.TOOL_NAME,
126 + (EventType.TOOL_CALL_START,),
127 + ),
128 + (ParserState.TOOL_PREAMBLE, "FUNC_PREFIX"): Transition(
129 + ParserState.TOOL_NAME,
130 + (),
131 + ),
132 + (ParserState.TOOL_NAME, "CLOSE_ANGLE"): Transition(
133 + ParserState.TOOL_ARGS,
134 + (),
135 + ),
136 + # Malformed: </function> while still in TOOL_NAME (no closing >)
137 + (ParserState.TOOL_NAME, "FUNC_END"): Transition(
138 + ParserState.TOOL_BETWEEN,
139 + (EventType.TOOL_CALL_END,),
140 + ),
141 + (ParserState.TOOL_ARGS, "FUNC_END"): Transition(
142 + ParserState.TOOL_BETWEEN,
143 + (EventType.TOOL_CALL_END,),
144 + ),
145 + (ParserState.TOOL_BETWEEN, "TOOL_END"): Transition(
146 + ParserState.CONTENT,
147 + (),
148 + ),
149 + # Consecutive tool call without closing </tool_call>
150 + (ParserState.TOOL_BETWEEN, "TOOL_START"): Transition(
151 + ParserState.TOOL_PREAMBLE,
152 + (EventType.TOOL_CALL_START,),
153 + ),
154 + (ParserState.TOOL_BETWEEN, "FUNC_PREFIX"): Transition(
155 + ParserState.TOOL_NAME,
156 + (EventType.TOOL_CALL_START,),
157 + ),
158 + },
159 + arg_converter=_qwen3_arg_converter,
160 + stream_arg_deltas=True,
161 + strip_trailing_reasoning_whitespace=False,
162 + tool_args_json=False,
163 + )
164 +
165 +
166 +class Qwen3Parser(ParserEngine):
167 + """Qwen3 parser: ``<think>``/``</think>`` reasoning +
168 + ``<tool_call>`` XML tool calls in a single engine.
169 +
170 + - ``<tool_call>`` as implicit reasoning end
171 + - Unpaired ``<tool_call>`` token ID detection for ``is_reasoning_end``
172 + """
173 +
174 + def __init__(
175 + self,
176 + tokenizer: TokenizerLike,
177 + tools: list[Tool] | None = None,
178 + **kwargs,
179 + ) -> None:
180 + chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {}
181 + self.thinking_enabled = chat_kwargs.get("enable_thinking", True)
182 + kwargs.setdefault(
183 + "parser_engine_config",
184 + qwen3_config(thinking=self.thinking_enabled),
185 + )
186 + super().__init__(
187 + tokenizer,
188 + tools,
189 + **kwargs,
190 + )
191 + vocab = self.vocab
192 + self._tool_call_token_id: int | None = vocab.get("<tool_call>")
193 + self._tool_call_end_token_id: int | None = vocab.get("</tool_call>")
194 +
195 + def extract_reasoning(
196 + self,
197 + model_output: str,
198 + request: ChatCompletionRequest | ResponsesRequest,
199 + ) -> tuple[str | None, str | None]:
200 + if not self.thinking_enabled:
201 + return None, model_output
202 + return super().extract_reasoning(model_output, request)
203 +
204 + def is_reasoning_end(self, input_ids: list[int]) -> bool:
205 + if super().is_reasoning_end(input_ids):
206 + return True
207 + tool_call_id = self._tool_call_token_id
208 + tool_call_end_id = self._tool_call_end_token_id
209 + if tool_call_id is not None:
210 + for i in range(len(input_ids) - 1, -1, -1):
211 + if input_ids[i] == tool_call_id:
212 + if tool_call_end_id is not None and any(
213 + input_ids[j] == tool_call_end_id
214 + for j in range(i + 1, len(input_ids))
215 + ):
216 + continue
217 + return True
218 + return False
vllm/reasoning/__init__.py
+6 -6
@@ -13,8 +13,8 @@ Register a lazy module mapping.
13 Example:
14 ReasoningParserManager.register_lazy_module(
15 name="qwen3",
16 - module_path="vllm.reasoning.qwen3_reasoning_parser",
17 - class_name="Qwen3ReasoningParser",
16 + module_path="vllm.reasoning.qwen3_engine_reasoning_parser",
17 + class_name="Qwen3ParserReasoningAdapter",
18 )
19 """
20
@@ -81,8 +81,8 @@ _REASONING_PARSERS_TO_REGISTER = {
81 "KimiK2ReasoningParser",
82 ),
83 "mimo": (
84 - "qwen3_reasoning_parser",
85 - "Qwen3ReasoningParser",
84 + "qwen3_engine_reasoning_parser",
85 + "Qwen3ParserReasoningAdapter",
86 ),
87 "minimax_m2": (
88 "minimax_m2_reasoning_parser",
@@ -105,8 +105,8 @@ _REASONING_PARSERS_TO_REGISTER = {
105 "Olmo3ReasoningParser",
106 ),
107 "qwen3": (
108 - "qwen3_reasoning_parser",
109 - "Qwen3ReasoningParser",
108 + "qwen3_engine_reasoning_parser",
109 + "Qwen3ParserReasoningAdapter",
110 ),
111 "seed_oss": (
112 "seedoss_reasoning_parser",
vllm/reasoning/abs_reasoning_parsers.py
+15 -2
@@ -31,6 +31,8 @@ class ReasoningParser:
31 It is used to extract reasoning content from the model output.
32 """
33
34 + engine_based_streaming: bool = False
35 +
36 def __init__(self, tokenizer: "TokenizerLike", *args, **kwargs):
37 self.model_tokenizer = tokenizer
38 # Optional vLLM ModelConfig from the server. Use get (not pop) so composite
@@ -57,6 +59,17 @@ class ReasoningParser:
59 """
60 return None
61
62 + def has_engine_confirmed_reasoning_end(self) -> bool:
63 + """Whether the engine has confirmed the reasoning end transition.
64 +
65 + Engine-based parsers may defer terminal processing when the
66 + detokenizer holds back text. This method returns the engine's
67 + *processed* state, not a raw token-ID check.
68 +
69 + Only called for parsers with ``engine_based_streaming = True``.
70 + """
71 + return False
72 +
73 @abstractmethod
74 def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
75 """
@@ -285,8 +298,8 @@ class ReasoningParserManager:
298 Example:
299 ReasoningParserManager.register_lazy_module(
300 name="qwen3",
288 - module_path="vllm.reasoning.parsers.qwen3_reasoning_parser",
289 - class_name="Qwen3ReasoningParser",
301 + module_path="vllm.reasoning.qwen3_engine_reasoning_parser",
302 + class_name="Qwen3ParserReasoningAdapter",
303 )
304 """
305 cls.lazy_parsers[name] = (module_path, class_name)
vllm/reasoning/qwen3_engine_reasoning_parser.py new
+6
@@ -0,0 +1,6 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from vllm.parser.engine.registered_adapters import Qwen3ParserReasoningAdapter
5 +
6 +__all__ = ["Qwen3ParserReasoningAdapter"]
vllm/reasoning/qwen3_reasoning_parser.py deleted
-231
@@ -1,231 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -from collections.abc import Iterable, Sequence
5 -from typing import TYPE_CHECKING
6 -
7 -from vllm.entrypoints.openai.engine.protocol import DeltaMessage
8 -from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
9 -
10 -if TYPE_CHECKING:
11 - from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
12 - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
13 - from vllm.tokenizers import TokenizerLike
14 -
15 -
16 -class Qwen3ReasoningParser(BaseThinkingReasoningParser):
17 - """
18 - Reasoning parser for the Qwen3/Qwen3.5 model family.
19 -
20 - The Qwen3 model family uses <think>...</think> tokens to denote reasoning
21 - text. Starting with Qwen3.5, the chat template places <think> in the
22 - prompt so only </think> appears in the generated output. The model
23 - provides a strict switch to disable reasoning output via the
24 - 'enable_thinking=False' parameter.
25 -
26 - When thinking is disabled, the template places <think>\\n\\n</think>\\n\\n
27 - in the prompt. The serving layer detects this via prompt_is_reasoning_end
28 - and routes deltas as content without calling the streaming parser.
29 -
30 - NOTE: Models up to the 2507 release (e.g., Qwen/Qwen3-235B-A22B-Instruct-2507)
31 - use an older chat template where the model generates <think> itself.
32 - This parser handles both styles: if <think> appears in the generated output
33 - it is stripped before extraction (non-streaming) or skipped (streaming).
34 -
35 - NOTE: Qwen3.5 models may emit <tool_call> inside the thinking block
36 - without closing </think> first. <tool_call> is treated as an implicit
37 - end of reasoning, matching the approach in KimiK2ReasoningParser.
38 - """
39 -
40 - def __init__(self, tokenizer: "TokenizerLike", *args, **kwargs):
41 - super().__init__(tokenizer, *args, **kwargs)
42 -
43 - chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {}
44 - # Qwen3 defaults to thinking enabled; only treat output as
45 - # pure content when the user explicitly disables it.
46 - self.thinking_enabled = chat_kwargs.get("enable_thinking", True)
47 -
48 - self._tool_call_tag = "<tool_call>"
49 - self._tool_call_token_id = self.vocab.get(self._tool_call_tag)
50 - self._tool_call_end_tag = "</tool_call>"
51 - self._tool_call_end_token_id = self.vocab.get(self._tool_call_end_tag)
52 -
53 - @property
54 - def start_token(self) -> str:
55 - """The token that starts reasoning content."""
56 - return "<think>"
57 -
58 - @property
59 - def end_token(self) -> str:
60 - """The token that ends reasoning content."""
61 - return "</think>"
62 -
63 - def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
64 - start_token_id = self.start_token_id
65 - end_token_id = self.end_token_id
66 - tool_call_token_id = self._tool_call_token_id
67 - tool_call_end_token_id = self._tool_call_end_token_id
68 -
69 - for i in range(len(input_ids) - 1, -1, -1):
70 - token_id = input_ids[i]
71 - if token_id == start_token_id:
72 - # Found <think> before </think> or <tool_call>
73 - return False
74 - if token_id == end_token_id:
75 - return True
76 - if tool_call_token_id is not None and token_id == tool_call_token_id:
77 - # Only treat as implicit reasoning end if this <tool_call>
78 - # is NOT followed by </tool_call>. Paired occurrences are
79 - # template examples in the prompt, not model output.
80 - if tool_call_end_token_id is not None and any(
81 - input_ids[j] == tool_call_end_token_id
82 - for j in range(i + 1, len(input_ids))
83 - ):
84 - continue
85 - return True
86 - return False
87 -
88 - def is_reasoning_end_streaming(
89 - self, input_ids: Sequence[int], delta_ids: Iterable[int]
90 - ) -> bool:
91 - if super().is_reasoning_end_streaming(input_ids, delta_ids):
92 - return True
93 - if self._tool_call_token_id is not None:
94 - return self._tool_call_token_id in delta_ids
95 - return False
96 -
97 - def extract_content_ids(self, input_ids: list[int]) -> list[int]:
98 - """
99 - Extract content token ids from the input_ids.
100 - """
101 - result = super().extract_content_ids(input_ids)
102 - if result:
103 - return result
104 - # Fall back: content starts at <tool_call> (implicit reasoning end).
105 - if (
106 - self._tool_call_token_id is not None
107 - and self._tool_call_token_id in input_ids
108 - ):
109 - tool_call_index = (
110 - len(input_ids) - 1 - input_ids[::-1].index(self._tool_call_token_id)
111 - )
112 - return input_ids[tool_call_index:]
113 - return []
114 -
115 - def extract_reasoning(
116 - self, model_output: str, request: "ChatCompletionRequest | ResponsesRequest"
117 - ) -> tuple[str | None, str | None]:
118 - """
119 - Extract reasoning content from the model output.
120 -
121 - The <think> token is placed in the prompt by the chat template,
122 - so typically only </think> appears in the generated output.
123 - If <think> is present (e.g. from a different template), it is
124 - stripped before extraction.
125 -
126 - When thinking is explicitly disabled and no </think> appears,
127 - returns (None, model_output) — all output is content.
128 - Otherwise (thinking enabled, default), a missing </think> means
129 - the output was truncated and everything is reasoning:
130 - returns (model_output, None).
131 -
132 - Returns:
133 - tuple[Optional[str], Optional[str]]: reasoning content and content
134 - """
135 -
136 - # Strip <think> if present in the generated output.
137 - model_output_parts = model_output.partition(self.start_token)
138 - model_output = (
139 - model_output_parts[2] if model_output_parts[1] else model_output_parts[0]
140 - )
141 -
142 - if self.end_token in model_output:
143 - reasoning, _, content = model_output.partition(self.end_token)
144 - return reasoning, content or None
145 -
146 - if not self.thinking_enabled:
147 - # Thinking explicitly disabled — treat everything as content.
148 - return None, model_output
149 -
150 - # No </think> — check for implicit reasoning end via <tool_call>.
151 - tool_call_index = model_output.find(self._tool_call_tag)
152 - if tool_call_index != -1:
153 - reasoning = model_output[:tool_call_index]
154 - content = model_output[tool_call_index:]
155 - return reasoning or None, content or None
156 - # Thinking enabled but no </think>: output was truncated.
157 - # Everything generated so far is reasoning.
158 - return model_output, None
159 -
160 - def extract_reasoning_streaming(
161 - self,
162 - previous_text: str,
163 - current_text: str,
164 - delta_text: str,
165 - previous_token_ids: Sequence[int],
166 - current_token_ids: Sequence[int],
167 - delta_token_ids: Sequence[int],
168 - ) -> DeltaMessage | None:
169 - """
170 - Extract reasoning content from a streaming delta.
171 -
172 - Since <think> is placed in the prompt by the chat template, all
173 - generated tokens before </think> are reasoning and tokens after
174 - are content.
175 -
176 - NOTE: When thinking is disabled, no think tokens appear in the
177 - generated output. The serving layer detects this via
178 - prompt_is_reasoning_end and routes deltas as content without
179 - calling this method.
180 - """
181 - # Strip <think> from delta if present (old template / edge case
182 - # where the model generates <think> itself).
183 - if self.start_token_id in delta_token_ids:
184 - start_idx = delta_text.find(self.start_token)
185 - if start_idx >= 0:
186 - delta_text = delta_text[start_idx + len(self.start_token) :]
187 -
188 - if self.end_token_id in delta_token_ids:
189 - # End token in this delta: split reasoning from content.
190 - end_index = delta_text.find(self.end_token)
191 - if end_index >= 0:
192 - reasoning = delta_text[:end_index]
193 - content = delta_text[end_index + len(self.end_token) :]
194 - if not reasoning and not content:
195 - return None
196 - return DeltaMessage(
197 - reasoning=reasoning if reasoning else None,
198 - content=content if content else None,
199 - )
200 - # end_token_id in IDs but not in text (already stripped)
201 - return None
202 -
203 - # Implicit reasoning end via <tool_call>.
204 - if (
205 - self._tool_call_token_id is not None
206 - and self._tool_call_token_id in delta_token_ids
207 - ):
208 - tool_index = delta_text.find(self._tool_call_tag)
209 - if tool_index >= 0:
210 - reasoning = delta_text[:tool_index]
211 - content = delta_text[tool_index:]
212 - return DeltaMessage(
213 - reasoning=reasoning if reasoning else None,
214 - content=content if content else None,
215 - )
216 -
217 - # No end token in this delta.
218 - if not delta_text:
219 - # Nothing left after stripping start token.
220 - return None
221 - elif self.end_token_id in previous_token_ids:
222 - # End token already passed: everything is content now.
223 - return DeltaMessage(content=delta_text)
224 - elif (
225 - self._tool_call_token_id is not None
226 - and self._tool_call_token_id in previous_token_ids
227 - ):
228 - return DeltaMessage(content=delta_text)
229 - else:
230 - # No end token yet: still in reasoning phase.
231 - return DeltaMessage(reasoning=delta_text)
vllm/tool_parsers/__init__.py
+6 -6
@@ -119,8 +119,8 @@ _TOOL_PARSERS_TO_REGISTER = {
119 "LongcatFlashToolParser",
120 ),
121 "mimo": (
122 - "qwen3coder_tool_parser",
123 - "Qwen3CoderToolParser",
122 + "qwen3_engine_tool_parser",
123 + "Qwen3EngineToolParser",
124 ),
125 "minimax_m2": (
126 "minimax_m2_tool_parser",
@@ -155,12 +155,12 @@ _TOOL_PARSERS_TO_REGISTER = {
155 "PythonicToolParser",
156 ),
157 "qwen3_coder": (
158 - "qwen3coder_tool_parser",
159 - "Qwen3CoderToolParser",
158 + "qwen3_engine_tool_parser",
159 + "Qwen3EngineToolParser",
160 ),
161 "qwen3_xml": (
162 - "qwen3coder_tool_parser",
163 - "Qwen3CoderToolParser",
162 + "qwen3_engine_tool_parser",
163 + "Qwen3EngineToolParser",
164 ),
165 "seed_oss": (
166 "seed_oss_tool_parser",
vllm/tool_parsers/abstract_tool_parser.py
+1
@@ -60,6 +60,7 @@ class ToolParser:
60 # xgrammar builtin structural tag model key. Subclasses set this when
61 # their parsed tool-call syntax matches a builtin xgrammar format.
62 structural_tag_model: str | None = None
63 + engine_based_streaming: bool = False
64
65 def __init_subclass__(cls, **kwargs: Any) -> None:
66 super().__init_subclass__(**kwargs)
vllm/tool_parsers/qwen3_engine_tool_parser.py new
+8
@@ -0,0 +1,8 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from vllm.parser.engine.registered_adapters import Qwen3ParserToolAdapter
5 +
6 +
7 +class Qwen3EngineToolParser(Qwen3ParserToolAdapter): # type: ignore[valid-type, misc]
8 + structural_tag_model = "qwen_3_coder"
vllm/tool_parsers/qwen3coder_tool_parser.py deleted
-586
@@ -1,586 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -import json
4 -import uuid
5 -from collections.abc import Sequence
6 -from typing import Any
7 -
8 -import regex as re
9 -
10 -from vllm.entrypoints.openai.chat_completion.protocol import (
11 - ChatCompletionRequest,
12 -)
13 -from vllm.entrypoints.openai.engine.protocol import (
14 - DeltaFunctionCall,
15 - DeltaMessage,
16 - DeltaToolCall,
17 - ExtractedToolCallInformation,
18 - FunctionCall,
19 - ToolCall,
20 -)
21 -from vllm.logger import init_logger
22 -from vllm.tokenizers import TokenizerLike
23 -from vllm.tool_parsers.abstract_tool_parser import (
24 - Tool,
25 - ToolParser,
26 -)
27 -from vllm.tool_parsers.utils import (
28 - coerce_to_schema_type,
29 - extract_types_from_schema,
30 - find_tool_properties,
31 -)
32 -
33 -logger = init_logger(__name__)
34 -
35 -
36 -class Qwen3CoderToolParser(ToolParser):
37 - structural_tag_model = "qwen_3_coder"
38 -
39 - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
40 - super().__init__(tokenizer, tools)
41 -
42 - self.current_tool_name_sent: bool = False
43 - self.prev_tool_call_arr: list[dict] = []
44 - # Override base class type - we use string IDs for tool calls
45 - self.current_tool_id: str | None = None # type: ignore
46 - self.streamed_args_for_tool: list[str] = []
47 -
48 - # Sentinel tokens for streaming mode
49 - self.tool_call_start_token: str = "<tool_call>"
50 - self.tool_call_end_token: str = "</tool_call>"
51 - self.tool_call_prefix: str = "<function="
52 - self.function_end_token: str = "</function>"
53 - self.parameter_prefix: str = "<parameter="
54 - self.parameter_end_token: str = "</parameter>"
55 - self.is_tool_call_started: bool = False
56 - self.failed_count: int = 0
57 -
58 - # Enhanced streaming state - reset for each new message
59 - self._reset_streaming_state()
60 -
61 - # Regex patterns
62 - self.tool_call_complete_regex = re.compile(
63 - r"<tool_call>(.*?)</tool_call>", re.DOTALL
64 - )
65 - self.tool_call_regex = re.compile(
66 - r"<tool_call>(.*?)</tool_call>|<tool_call>(.*?)$", re.DOTALL
67 - )
68 - self.tool_call_function_regex = re.compile(
69 - r"<function=(.*?)</function>|<function=(.*)$", re.DOTALL
70 - )
71 - self.tool_call_parameter_regex = re.compile(
72 - r"<parameter=(.*?)(?:</parameter>|(?=<parameter=)|(?=</function>)|$)",
73 - re.DOTALL,
74 - )
75 -
76 - if not self.model_tokenizer:
77 - raise ValueError(
78 - "The model tokenizer must be passed to the ToolParser "
79 - "constructor during construction."
80 - )
81 -
82 - self.tool_call_start_token_id = self.vocab.get(self.tool_call_start_token)
83 - self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token)
84 -
85 - if self.tool_call_start_token_id is None or self.tool_call_end_token_id is None:
86 - raise RuntimeError(
87 - "Qwen3 XML Tool parser could not locate tool call start/end "
88 - "tokens in the tokenizer!"
89 - )
90 -
91 - logger.debug(
92 - "vLLM Successfully import tool parser %s !", self.__class__.__name__
93 - )
94 -
95 - def _generate_tool_call_id(self) -> str:
96 - """Generate a unique tool call ID."""
97 - return f"call_{uuid.uuid4().hex[:24]}"
98 -
99 - def _reset_streaming_state(self):
100 - """Reset all streaming state."""
101 - self.current_tool_index = 0
102 - self.is_tool_call_started = False
103 - self.header_sent = False
104 - self.current_tool_id = None
105 - self.current_function_name = None
106 - self.current_param_name = None
107 - self.current_param_value = ""
108 - self.param_count = 0
109 - self.in_param = False
110 - self.in_function = False
111 - self.accumulated_text = ""
112 - self.json_started = False
113 - self.json_closed = False
114 - # Store accumulated parameters for type conversion
115 - self.accumulated_params = {}
116 - self.streaming_request = None
117 -
118 - def _convert_param_value(
119 - self, param_value: str, param_name: str, param_config: dict, func_name: str
120 - ) -> Any:
121 - """Convert parameter value based on its type in the schema."""
122 - if not isinstance(param_value, str):
123 - return param_value
124 - param_schema = param_config.get(param_name, {})
125 - param_types = extract_types_from_schema(param_schema)
126 - return coerce_to_schema_type(param_value, param_types)
127 -
128 - def _parse_xml_function_call(self, function_call_str: str) -> ToolCall | None:
129 - # Extract function name
130 - end_index = function_call_str.find(">")
131 - # If there's no ">" character, this is not a valid xml function call
132 - if end_index == -1:
133 - return None
134 - function_name = function_call_str[:end_index]
135 - param_config = find_tool_properties(self.tools, function_name)
136 - parameters = function_call_str[end_index + 1 :]
137 - param_dict = {}
138 - for match_text in self.tool_call_parameter_regex.findall(parameters):
139 - idx = match_text.index(">")
140 - param_name = match_text[:idx]
141 - param_value = str(match_text[idx + 1 :])
142 - # Remove prefix and trailing \n
143 - if param_value.startswith("\n"):
144 - param_value = param_value[1:]
145 - if param_value.endswith("\n"):
146 - param_value = param_value[:-1]
147 -
148 - param_dict[param_name] = self._convert_param_value(
149 - param_value, param_name, param_config, function_name
150 - )
151 - return ToolCall(
152 - type="function",
153 - function=FunctionCall(
154 - name=function_name, arguments=json.dumps(param_dict, ensure_ascii=False)
155 - ),
156 - )
157 -
158 - def _get_function_calls(self, model_output: str) -> list[str]:
159 - # Find all tool calls
160 - matched_ranges = self.tool_call_regex.findall(model_output)
161 - raw_tool_calls = [
162 - match[0] if match[0] else match[1] for match in matched_ranges
163 - ]
164 -
165 - # Back-off strategy if no tool_call tags found
166 - if len(raw_tool_calls) == 0:
167 - raw_tool_calls = [model_output]
168 -
169 - raw_function_calls = []
170 - for tool_call in raw_tool_calls:
171 - raw_function_calls.extend(self.tool_call_function_regex.findall(tool_call))
172 -
173 - function_calls = [
174 - match[0] if match[0] else match[1] for match in raw_function_calls
175 - ]
176 - return function_calls
177 -
178 - def extract_tool_calls(
179 - self,
180 - model_output: str,
181 - request: ChatCompletionRequest,
182 - ) -> ExtractedToolCallInformation:
183 - # Quick check to avoid unnecessary processing
184 - if self.tool_call_prefix not in model_output:
185 - return ExtractedToolCallInformation(
186 - tools_called=False, tool_calls=[], content=model_output
187 - )
188 -
189 - try:
190 - function_calls = self._get_function_calls(model_output)
191 - if len(function_calls) == 0:
192 - return ExtractedToolCallInformation(
193 - tools_called=False, tool_calls=[], content=model_output
194 - )
195 -
196 - tool_calls = [
197 - self._parse_xml_function_call(function_call_str)
198 - for function_call_str in function_calls
199 - ]
200 - # Populate prev_tool_call_arr for serving layer to set finish_reason
201 - self.prev_tool_call_arr.clear() # Clear previous calls
202 - for tool_call in tool_calls:
203 - if tool_call:
204 - self.prev_tool_call_arr.append(
205 - {
206 - "name": tool_call.function.name,
207 - "arguments": tool_call.function.arguments,
208 - }
209 - )
210 -
211 - # Extract content before tool calls
212 - content_index = model_output.find(self.tool_call_start_token)
213 - idx = model_output.find(self.tool_call_prefix)
214 - content_index = content_index if content_index >= 0 else idx
215 - content = model_output[:content_index] # .rstrip()
216 - valid_tool_calls = [tc for tc in tool_calls if tc is not None]
217 - return ExtractedToolCallInformation(
218 - tools_called=(len(valid_tool_calls) > 0),
219 - tool_calls=valid_tool_calls,
220 - content=content if content else None,
221 - )
222 -
223 - except Exception:
224 - logger.exception("Error in extracting tool call from response.")
225 - return ExtractedToolCallInformation(
226 - tools_called=False, tool_calls=[], content=model_output
227 - )
228 -
229 - def extract_tool_calls_streaming(
230 - self,
231 - previous_text: str,
232 - current_text: str,
233 - delta_text: str,
234 - previous_token_ids: Sequence[int],
235 - current_token_ids: Sequence[int],
236 - delta_token_ids: Sequence[int],
237 - request: ChatCompletionRequest,
238 - ) -> DeltaMessage | None:
239 - # Store request for type conversion
240 - if not previous_text:
241 - self._reset_streaming_state()
242 - self.streaming_request = request
243 -
244 - # If no delta text, return None unless it's an EOS token after tools
245 - if not delta_text:
246 - # Check if this is an EOS token after all tool calls are complete
247 - # Check for tool calls in text even if is_tool_call_started
248 - # is False (might have been reset after processing all tools)
249 - if delta_token_ids and self.tool_call_end_token_id not in delta_token_ids:
250 - # Count complete tool calls
251 - complete_calls = len(
252 - self.tool_call_complete_regex.findall(current_text)
253 - )
254 -
255 - # If we have completed tool calls and populated
256 - # prev_tool_call_arr
257 - if complete_calls > 0 and len(self.prev_tool_call_arr) > 0:
258 - # Check if all tool calls are closed
259 - open_calls = current_text.count(
260 - self.tool_call_start_token
261 - ) - current_text.count(self.tool_call_end_token)
262 - if open_calls == 0:
263 - # Return empty delta for finish_reason processing
264 - return DeltaMessage(content="")
265 - elif not self.is_tool_call_started and current_text:
266 - # This is a regular content response that's now complete
267 - return DeltaMessage(content="")
268 - return None
269 -
270 - # Update accumulated text
271 - self.accumulated_text = current_text
272 -
273 - # Check if we need to advance to next tool
274 - if self.json_closed and not self.in_function:
275 - # Check if this tool call has ended
276 - tool_ends = current_text.count(self.tool_call_end_token)
277 - if tool_ends > self.current_tool_index:
278 - # This tool has ended, advance to next
279 - self.current_tool_index += 1
280 - self.header_sent = False
281 - self.param_count = 0
282 - self.json_started = False
283 - self.json_closed = False
284 - self.accumulated_params = {}
285 -
286 - # Check if there are more tool calls
287 - tool_starts = current_text.count(self.tool_call_start_token)
288 - if self.current_tool_index >= tool_starts:
289 - # No more tool calls
290 - self.is_tool_call_started = False
291 - # Continue processing next tool
292 - return None
293 -
294 - # Handle normal content before tool calls
295 - if not self.is_tool_call_started:
296 - # Check if tool call is starting
297 - if (
298 - self.tool_call_start_token_id in delta_token_ids
299 - or self.tool_call_start_token in delta_text
300 - ):
301 - self.is_tool_call_started = True
302 - # Return any content before the tool call
303 - if self.tool_call_start_token in delta_text:
304 - content_before = delta_text[
305 - : delta_text.index(self.tool_call_start_token)
306 - ]
307 - if content_before:
308 - return DeltaMessage(content=content_before)
309 - return None
310 - else:
311 - # Check if we're between tool calls - skip whitespace
312 - if (
313 - current_text.rstrip().endswith(self.tool_call_end_token)
314 - and delta_text.strip() == ""
315 - ):
316 - # We just ended a tool call, skip whitespace
317 - return None
318 - # Normal content, no tool call
319 - return DeltaMessage(content=delta_text)
320 -
321 - # Check if we're between tool calls (waiting for next one)
322 - # Count tool calls we've seen vs processed
323 - tool_starts_count = current_text.count(self.tool_call_start_token)
324 - if self.current_tool_index >= tool_starts_count:
325 - # We're past all tool calls, shouldn't be here
326 - return None
327 -
328 - # We're in a tool call, find the current tool call portion
329 - # Need to find the correct tool call based on current_tool_index
330 - tool_start_positions: list[int] = []
331 - idx = 0
332 - while True:
333 - idx = current_text.find(self.tool_call_start_token, idx)
334 - if idx == -1:
335 - break
336 - tool_start_positions.append(idx)
337 - idx += len(self.tool_call_start_token)
338 -
339 - if self.current_tool_index >= len(tool_start_positions):
340 - # No more tool calls to process yet
341 - return None
342 -
343 - tool_start_idx = tool_start_positions[self.current_tool_index]
344 - # Find where this tool call ends (or current position if not ended yet)
345 - tool_end_idx = current_text.find(self.tool_call_end_token, tool_start_idx)
346 - if tool_end_idx == -1:
347 - tool_text = current_text[tool_start_idx:]
348 - else:
349 - tool_text = current_text[
350 - tool_start_idx : tool_end_idx + len(self.tool_call_end_token)
351 - ]
352 -
353 - # Looking for function header
354 - if not self.header_sent:
355 - if self.tool_call_prefix in tool_text:
356 - func_start = tool_text.find(self.tool_call_prefix) + len(
357 - self.tool_call_prefix
358 - )
359 - func_end = tool_text.find(">", func_start)
360 -
361 - if func_end != -1:
362 - # Found complete function name
363 - self.current_function_name = tool_text[func_start:func_end]
364 - self.current_tool_id = self._generate_tool_call_id()
365 - self.header_sent = True
366 - self.in_function = True
367 -
368 - # Always append — each tool call is a separate
369 - # invocation even if the function name is the same
370 - # (e.g. two consecutive "read" calls).
371 - self.prev_tool_call_arr.append(
372 - {
373 - "name": self.current_function_name,
374 - "arguments": "{}",
375 - }
376 - )
377 -
378 - # Initialize streamed args tracking for this tool.
379 - # The serving layer reads streamed_args_for_tool to
380 - # compute remaining arguments at stream end. Without
381 - # this, IndexError occurs when the serving layer
382 - # accesses streamed_args_for_tool[index].
383 - self.streamed_args_for_tool.append("")
384 -
385 - # Send header with function info
386 - return DeltaMessage(
387 - tool_calls=[
388 - DeltaToolCall(
389 - index=self.current_tool_index,
390 - id=self.current_tool_id,
391 - function=DeltaFunctionCall(
392 - name=self.current_function_name, arguments=""
393 - ),
394 - type="function",
395 - )
396 - ]
397 - )
398 - return None
399 -
400 - # We've sent header, now handle function body
401 - if self.in_function:
402 - # Always send opening brace first, regardless of whether
403 - # parameter_prefix is in the current delta. With speculative
404 - # decoding, a single delta may contain both the opening brace
405 - # and parameter data; skipping "{" here would desync
406 - # json_started from what was actually streamed.
407 - if not self.json_started:
408 - self.json_started = True
409 - self.streamed_args_for_tool[self.current_tool_index] += "{"
410 - return DeltaMessage(
411 - tool_calls=[
412 - DeltaToolCall(
413 - index=self.current_tool_index,
414 - function=DeltaFunctionCall(arguments="{"),
415 - )
416 - ]
417 - )
418 -
419 - # Find all parameter start positions in current tool_text
420 - param_starts = []
421 - search_idx = 0
422 - while True:
423 - search_idx = tool_text.find(self.parameter_prefix, search_idx)
424 - if search_idx == -1:
425 - break
426 - param_starts.append(search_idx)
427 - search_idx += len(self.parameter_prefix)
428 -
429 - # Process ALL complete params in a loop (spec decode fix).
430 - # With speculative decoding a single delta can deliver
431 - # multiple complete parameters at once. The old single-pass
432 - # code would process one and ``return None`` if the next was
433 - # incomplete — skipping any already-complete params that
434 - # preceded it. Using a loop with ``break`` instead ensures
435 - # we emit every complete parameter before yielding control.
436 - json_fragments = []
437 - while not self.in_param and self.param_count < len(param_starts):
438 - param_idx = param_starts[self.param_count]
439 - param_start = param_idx + len(self.parameter_prefix)
440 - remaining = tool_text[param_start:]
441 -
442 - if ">" not in remaining:
443 - break
444 -
445 - name_end = remaining.find(">")
446 - current_param_name = remaining[:name_end]
447 -
448 - value_start = param_start + name_end + 1
449 - value_text = tool_text[value_start:]
450 - if value_text.startswith("\n"):
451 - value_text = value_text[1:]
452 -
453 - param_end_idx = value_text.find(self.parameter_end_token)
454 - if param_end_idx == -1:
455 - next_param_idx = value_text.find(self.parameter_prefix)
456 - func_end_idx = value_text.find(self.function_end_token)
457 -
458 - if next_param_idx != -1 and (
459 - func_end_idx == -1 or next_param_idx < func_end_idx
460 - ):
461 - param_end_idx = next_param_idx
462 - elif func_end_idx != -1:
463 - param_end_idx = func_end_idx
464 - else:
465 - # Fallback for malformed XML where </function>
466 - # is missing. Use </tool_call> as a delimiter
467 - # if present in the value so we don't include
468 - # the closing tag as part of the param value.
469 - tool_end_in_value = value_text.find(self.tool_call_end_token)
470 - if tool_end_in_value != -1:
471 - param_end_idx = tool_end_in_value
472 - else:
473 - # Parameter incomplete — break so we still
474 - # emit any fragments accumulated by earlier
475 - # loop iterations.
476 - break
477 -
478 - if param_end_idx == -1:
479 - break
480 -
481 - param_value = value_text[:param_end_idx]
482 - if param_value.endswith("\n"):
483 - param_value = param_value[:-1]
484 -
485 - self.current_param_name = current_param_name
486 - self.accumulated_params[current_param_name] = param_value
487 -
488 - param_config = find_tool_properties(
489 - self.tools, self.current_function_name or ""
490 - )
491 -
492 - converted_value = self._convert_param_value(
493 - param_value,
494 - current_param_name,
495 - param_config,
496 - self.current_function_name or "",
497 - )
498 -
499 - serialized_value = json.dumps(converted_value, ensure_ascii=False)
500 -
501 - if self.param_count == 0:
502 - json_fragment = f'"{current_param_name}": {serialized_value}'
503 - else:
504 - json_fragment = f', "{current_param_name}": {serialized_value}'
505 -
506 - self.param_count += 1
507 - json_fragments.append(json_fragment)
508 -
509 - if json_fragments:
510 - combined = "".join(json_fragments)
511 -
512 - if self.current_tool_index < len(self.streamed_args_for_tool):
513 - self.streamed_args_for_tool[self.current_tool_index] += combined
514 - else:
515 - logger.warning(
516 - "streamed_args_for_tool out of sync: index=%d len=%d",
517 - self.current_tool_index,
518 - len(self.streamed_args_for_tool),
519 - )
520 -
521 - return DeltaMessage(
522 - tool_calls=[
523 - DeltaToolCall(
524 - index=self.current_tool_index,
525 - function=DeltaFunctionCall(arguments=combined),
526 - )
527 - ]
528 - )
529 -
530 - # Check for function end AFTER processing parameters.
531 - # This ordering is critical: with speculative decoding a
532 - # burst can deliver the final parameter value together with
533 - # </function>. If the close check ran first it would emit
534 - # "}" and set in_function=False before the parameter loop
535 - # ever ran, causing the parameter to be silently dropped.
536 - if not self.json_closed and self.function_end_token in tool_text:
537 - self.json_closed = True
538 -
539 - func_start = tool_text.find(self.tool_call_prefix) + len(
540 - self.tool_call_prefix
541 - )
542 - func_content_end = tool_text.find(self.function_end_token, func_start)
543 - if func_content_end != -1:
544 - func_content = tool_text[func_start:func_content_end]
545 - try:
546 - parsed_tool = self._parse_xml_function_call(
547 - func_content,
548 - )
549 - if parsed_tool and self.current_tool_index < len(
550 - self.prev_tool_call_arr
551 - ):
552 - self.prev_tool_call_arr[self.current_tool_index][
553 - "arguments"
554 - ] = parsed_tool.function.arguments
555 - except Exception:
556 - logger.debug(
557 - "Failed to parse tool call during streaming: %s",
558 - tool_text,
559 - exc_info=True,
560 - )
561 -
562 - if self.current_tool_index < len(self.streamed_args_for_tool):
563 - self.streamed_args_for_tool[self.current_tool_index] += "}"
564 - else:
565 - logger.warning(
566 - "streamed_args_for_tool out of sync: index=%d len=%d",
567 - self.current_tool_index,
568 - len(self.streamed_args_for_tool),
569 - )
570 -
571 - result = DeltaMessage(
572 - tool_calls=[
573 - DeltaToolCall(
574 - index=self.current_tool_index,
575 - function=DeltaFunctionCall(arguments="}"),
576 - )
577 - ]
578 - )
579 -
580 - self.in_function = False
581 - self.json_closed = True
582 - self.accumulated_params = {}
583 -
584 - return result
585 -
586 - return None