A high-throughput and memory-efficient inference and serving engine for LLMs
[Frontend] Replace legacy Gemma4 parsers with engine-based implementation (#45588)
Signed-off-by: Ben Browning <bbrownin@redhat.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
Ben Browning committed
Jun 15, 2026 at 17:34 UTC
76a373eff47a35f828636774b63ba0315e8f15d0
20 files changed
+2808
-1332
tests/parser/engine/replay_harness.py
+9
-4
@@ -33,6 +33,7 @@ class Sample:
33
expected_tool_calls: list[dict] | None
34
tools: list[dict] | None = None
35
chat_template_kwargs: dict | None = None
36
+ prompt_token_ids: list[int] | None = None
37
38
39
@dataclass
@@ -120,6 +121,7 @@ def replay_streaming(
121
holdback_chars: int = 0,
122
finished_on_last: bool = False,
123
tools: list[dict] | None = None,
124
+ prompt_token_ids: list[int] | None = None,
125
) -> list[DeltaMessage | None]:
126
"""Feed tokens through ``parser.parse_delta()`` at a given chunk size.
127
@@ -146,6 +148,7 @@ def replay_streaming(
148
all_texts = [text for _, text in tokens]
149
150
request = _test_request(tools=tools)
151
+ first_prompt_ids = prompt_token_ids if prompt_token_ids is not None else []
152
153
if holdback_chars <= 0:
154
chunks = list(range(0, len(tokens), chunk_size))
@@ -159,7 +162,7 @@ def replay_streaming(
162
delta_text,
163
batch_ids,
164
request,
162
- prompt_token_ids=[] if start == 0 else None,
165
+ prompt_token_ids=first_prompt_ids if start == 0 else None,
166
finished=finished_on_last and is_last,
167
)
168
results.append(result)
@@ -192,7 +195,7 @@ def replay_streaming(
195
delta_text,
196
batch_ids,
197
request,
195
- prompt_token_ids=[] if is_first else None,
198
+ prompt_token_ids=first_prompt_ids if is_first else None,
199
finished=finished_on_last and is_last_chunk,
200
)
201
results.append(result)
@@ -205,7 +208,7 @@ def replay_streaming(
208
delta_text,
209
batch_ids,
210
request,
208
- prompt_token_ids=[] if is_first else None,
211
+ prompt_token_ids=first_prompt_ids if is_first else None,
212
finished=finished_on_last,
213
)
214
results.append(result)
@@ -218,6 +221,7 @@ def replay_with_text_holdback(
221
tokens: list[tuple[int, str]],
222
text_delay: int = 1,
223
tools: list[dict] | None = None,
224
+ prompt_token_ids: list[int] | None = None,
225
) -> list[DeltaMessage | None]:
226
"""Replay token-by-token with text arriving *text_delay* steps late.
227
@@ -235,6 +239,7 @@ def replay_with_text_holdback(
239
"""
240
results: list[DeltaMessage | None] = []
241
request = _test_request(tools=tools)
242
+ first_prompt_ids = prompt_token_ids if prompt_token_ids is not None else []
243
244
n = len(tokens)
245
held_texts: list[str] = []
@@ -256,7 +261,7 @@ def replay_with_text_holdback(
261
delta_text,
262
[token_id],
263
request,
259
- prompt_token_ids=[] if i == 0 else None,
264
+ prompt_token_ids=first_prompt_ids if i == 0 else None,
265
finished=is_last,
266
)
267
results.append(result)
tests/parser/engine/test_delegating_replay.py
+18
-9
@@ -29,8 +29,9 @@ from vllm.parser.parser_manager import ParserManager
29
30
_TOOLS_VALIDATOR = TypeAdapter(list[ChatCompletionToolsParam])
31
32
-_PAIRINGS: dict[str, tuple[str, str]] = {
33
- "engine": ("qwen3_coder", "qwen3"),
32
+_PAIRINGS: dict[str, tuple[str, str, str]] = {
33
+ "engine": ("qwen3_coder", "qwen3", "qwen3"),
34
+ "gemma4_engine": ("gemma4", "gemma4", "gemma4"),
35
}
36
37
CHUNK_SIZES = [1, 2, 3, 5, 11, 23, None]
@@ -38,7 +39,7 @@ CHUNK_SIZES = [1, 2, 3, 5, 11, 23, None]
39
40
@lru_cache
41
def _get_delegating_parser_cls(pairings: str) -> type[Parser]:
41
- tool_name, reasoning_name = _PAIRINGS[pairings]
42
+ tool_name, reasoning_name, _ = _PAIRINGS[pairings]
43
parser_cls = ParserManager.get_parser(
44
tool_parser_name=tool_name,
45
reasoning_parser_name=reasoning_name,
@@ -48,16 +49,23 @@ def _get_delegating_parser_cls(pairings: str) -> type[Parser]:
49
return parser_cls
50
51
51
-_all_samples = build_samples("qwen3")
52
+def _pairing_samples() -> list[tuple[str, object]]:
53
+ items: list[tuple[str, object]] = []
54
+ for pairing_name, (_, _, model) in _PAIRINGS.items():
55
+ for sample in build_samples(model):
56
+ items.append((pairing_name, sample))
57
+ return items
58
59
60
+_all_pairing_samples = _pairing_samples()
61
+
62
+
63
+@pytest.mark.parametrize("chunk_size", CHUNK_SIZES, ids=lambda c: f"chunk={c}")
64
@pytest.mark.parametrize(
55
- "pairings",
56
- list(_PAIRINGS),
57
- ids=lambda p: f"mode={p}",
65
+ "pairings,sample",
66
+ _all_pairing_samples,
67
+ ids=lambda v: v.id if hasattr(v, "id") else v,
68
)
59
-@pytest.mark.parametrize("chunk_size", CHUNK_SIZES, ids=lambda c: f"chunk={c}")
60
-@pytest.mark.parametrize("sample", _all_samples, ids=lambda s: s.id)
69
def test_delegating_replay(sample, chunk_size, pairings):
70
parser_cls = _get_delegating_parser_cls(pairings=pairings)
71
@@ -77,6 +85,7 @@ def test_delegating_replay(sample, chunk_size, pairings):
85
chunk_size=chunk_size,
86
finished_on_last=True,
87
tools=sample.tools,
88
+ prompt_token_ids=sample.prompt_token_ids,
89
)
90
output = collect_output(deltas)
91
assert_parse_output(output, sample)
tests/parser/engine/test_gemma4_streaming_reasoning.py
new
+1201
@@ -0,0 +1,1201 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Tests for the unified Gemma4 parser engine."""
4
+
5
+import json
6
+from unittest.mock import MagicMock
7
+
8
+import pytest
9
+
10
+from tests.parser.engine.streaming_helpers import (
11
+ collect_content,
12
+ collect_function_name,
13
+ collect_tool_arguments,
14
+ simulate_tool_streaming,
15
+)
16
+from vllm.entrypoints.openai.chat_completion.protocol import (
17
+ ChatCompletionRequest,
18
+)
19
+from vllm.entrypoints.openai.engine.protocol import DeltaMessage
20
+from vllm.parser.gemma4 import Gemma4Parser
21
+
22
+# ── Special token IDs (arbitrary but consistent) ─────────────────────
23
+CHANNEL_START_ID = 50 # <|channel>
24
+CHANNEL_END_ID = 51 # <channel|>
25
+TOOL_CALL_START_ID = 48 # <|tool_call>
26
+TOOL_CALL_END_ID = 49 # <tool_call|>
27
+QUOTED_ID = 52 # <|"|>
28
+SPECIAL_TOKEN_MAP = {
29
+ CHANNEL_START_ID: "<|channel>",
30
+ CHANNEL_END_ID: "<channel|>",
31
+ TOOL_CALL_START_ID: "<|tool_call>",
32
+ TOOL_CALL_END_ID: "<tool_call|>",
33
+ QUOTED_ID: '<|"|>',
34
+}
35
+
36
+SPECIAL_TEXT_TO_ID = {v: k for k, v in SPECIAL_TOKEN_MAP.items()}
37
+
38
+
39
+def _make_tokenizer(sequence: list[tuple[int, str]]) -> MagicMock:
40
+ decode_map: dict[int, str] = dict(SPECIAL_TOKEN_MAP)
41
+ for tid, text in sequence:
42
+ decode_map[tid] = text
43
+
44
+ tokenizer = MagicMock()
45
+ tokenizer.get_vocab.return_value = dict(SPECIAL_TEXT_TO_ID)
46
+ tokenizer.encode.return_value = [tid for tid, _ in sequence]
47
+
48
+ def decode(ids, skip_special_tokens=False):
49
+ parts = []
50
+ for tid in ids:
51
+ if skip_special_tokens and tid in SPECIAL_TOKEN_MAP:
52
+ continue
53
+ text = decode_map.get(tid, f"?{tid}?")
54
+ parts.append(text)
55
+ return "".join(parts)
56
+
57
+ tokenizer.decode.side_effect = decode
58
+ return tokenizer
59
+
60
+
61
+# ── Model output ────────────────────────────────────────────────────
62
+
63
+REASONING_TEXT = (
64
+ "The user is asking for the current weather in Dallas, Texas, "
65
+ "and specifically requests the temperature in Fahrenheit. "
66
+ "I have a tool `get_current_weather` that can provide this "
67
+ "information. I should call this tool with `city='Dallas'`, "
68
+ "`state='TX'`, and `unit='fahrenheit'`."
69
+)
70
+
71
+# Break reasoning into word-level tokens
72
+_reasoning_words = REASONING_TEXT.split(" ")
73
+_REGULAR_TOKEN_START = 1000
74
+REASONING_TOKENS: list[tuple[int, str]] = []
75
+for i, word in enumerate(_reasoning_words):
76
+ prefix = " " if i > 0 else ""
77
+ REASONING_TOKENS.append((_REGULAR_TOKEN_START + i, prefix + word))
78
+
79
+# Tool call body tokens
80
+TOOL_BODY_TOKENS: list[tuple[int, str]] = [
81
+ (2000, "call"),
82
+ (2001, ":"),
83
+ (2002, "get_current_weather"),
84
+ (2003, "{"),
85
+ (2004, "city"),
86
+ (2005, ":"),
87
+ (2006, "Dallas"),
88
+ (2007, ","),
89
+ (2008, "state"),
90
+ (2009, ":"),
91
+ (2010, "TX"),
92
+ (2011, ","),
93
+ (2012, "unit"),
94
+ (2013, ":"),
95
+ (2014, "fahrenheit"),
96
+ (2015, "}"),
97
+]
98
+
99
+FULL_TOKEN_SEQUENCE: list[tuple[int, str]] = []
100
+FULL_TOKEN_SEQUENCE.append((CHANNEL_START_ID, "<|channel>"))
101
+FULL_TOKEN_SEQUENCE.append((3000, "thought"))
102
+FULL_TOKEN_SEQUENCE.append((3001, "\n"))
103
+FULL_TOKEN_SEQUENCE.extend(REASONING_TOKENS)
104
+FULL_TOKEN_SEQUENCE.append((CHANNEL_END_ID, "<channel|>"))
105
+FULL_TOKEN_SEQUENCE.append((TOOL_CALL_START_ID, "<|tool_call>"))
106
+FULL_TOKEN_SEQUENCE.extend(TOOL_BODY_TOKENS[:4])
107
+FULL_TOKEN_SEQUENCE.extend(TOOL_BODY_TOKENS[4:6])
108
+FULL_TOKEN_SEQUENCE.append((QUOTED_ID, '<|"|>'))
109
+FULL_TOKEN_SEQUENCE.append(TOOL_BODY_TOKENS[6])
110
+FULL_TOKEN_SEQUENCE.append((QUOTED_ID, '<|"|>'))
111
+FULL_TOKEN_SEQUENCE.extend(TOOL_BODY_TOKENS[7:10])
112
+FULL_TOKEN_SEQUENCE.append((QUOTED_ID, '<|"|>'))
113
+FULL_TOKEN_SEQUENCE.append(TOOL_BODY_TOKENS[10])
114
+FULL_TOKEN_SEQUENCE.append((QUOTED_ID, '<|"|>'))
115
+FULL_TOKEN_SEQUENCE.extend(TOOL_BODY_TOKENS[11:14])
116
+FULL_TOKEN_SEQUENCE.append((QUOTED_ID, '<|"|>'))
117
+FULL_TOKEN_SEQUENCE.append(TOOL_BODY_TOKENS[14])
118
+FULL_TOKEN_SEQUENCE.append((QUOTED_ID, '<|"|>'))
119
+FULL_TOKEN_SEQUENCE.append(TOOL_BODY_TOKENS[15])
120
+FULL_TOKEN_SEQUENCE.append((TOOL_CALL_END_ID, "<tool_call|>"))
121
+
122
+# Full model output as a single string
123
+FULL_MODEL_OUTPUT = "".join(text for _, text in FULL_TOKEN_SEQUENCE)
124
+
125
+# ── Helpers ──────────────────────────────────────────────────────────
126
+
127
+
128
+def _stream_tokens_batched(
129
+ parser, tokenizer, request, batch_size=10, prompt_token_ids=None
130
+) -> list[DeltaMessage | None]:
131
+ """Feed tokens in batches through parse_delta."""
132
+ token_ids = tokenizer.encode("", add_special_tokens=False)
133
+ results: list[DeltaMessage | None] = []
134
+ n = len(token_ids)
135
+
136
+ for start in range(0, n, batch_size):
137
+ batch_ids = token_ids[start : start + batch_size]
138
+ delta_text = tokenizer.decode(batch_ids)
139
+ result = parser.parse_delta(
140
+ delta_text,
141
+ batch_ids,
142
+ request,
143
+ prompt_token_ids=prompt_token_ids,
144
+ finished=(start + batch_size >= n),
145
+ )
146
+ prompt_token_ids = None
147
+ results.append(result)
148
+ return results
149
+
150
+
151
+def _collect_fields(results):
152
+ reasoning = "".join(r.reasoning for r in results if r and r.reasoning)
153
+ content = "".join(r.content for r in results if r and r.content)
154
+ tool_calls = [tc for r in results if r and r.tool_calls for tc in r.tool_calls]
155
+ return reasoning, content, tool_calls
156
+
157
+
158
+# ── Fixtures ─────────────────────────────────────────────────────────
159
+
160
+
161
+@pytest.fixture
162
+def mock_tokenizer():
163
+ return _make_tokenizer(FULL_TOKEN_SEQUENCE)
164
+
165
+
166
+@pytest.fixture
167
+def parser(mock_tokenizer):
168
+ return Gemma4Parser(mock_tokenizer)
169
+
170
+
171
+@pytest.fixture
172
+def request_obj():
173
+ return ChatCompletionRequest(
174
+ model="test-model",
175
+ messages=[{"role": "user", "content": "hi"}],
176
+ )
177
+
178
+
179
+# ── Tests ────────────────────────────────────────────────────────────
180
+
181
+
182
+class TestGemma4StreamingReasoningThenToolCall:
183
+ """Streaming: reasoning followed by a tool call."""
184
+
185
+ def test_tool_call_extracted(self, parser, mock_tokenizer, request_obj):
186
+ """Tool calls must be extracted from streaming output."""
187
+ results = _stream_tokens_batched(
188
+ parser,
189
+ mock_tokenizer,
190
+ request_obj,
191
+ batch_size=10,
192
+ prompt_token_ids=[],
193
+ )
194
+
195
+ reasoning, content, tool_calls = _collect_fields(results)
196
+
197
+ assert len(tool_calls) > 0, (
198
+ f"Expected tool_calls but got none. "
199
+ f"content={content!r}, reasoning={reasoning[:80]!r}..."
200
+ )
201
+
202
+ names = [
203
+ tc.function.name for tc in tool_calls if tc.function and tc.function.name
204
+ ]
205
+ assert "get_current_weather" in names, (
206
+ f"Expected get_current_weather, got {names}"
207
+ )
208
+
209
+ args_text = "".join(
210
+ tc.function.arguments
211
+ for tc in tool_calls
212
+ if tc.function and tc.function.arguments
213
+ )
214
+ if args_text:
215
+ parsed_args = json.loads(args_text)
216
+ assert parsed_args.get("city") == "Dallas"
217
+ assert parsed_args.get("state") == "TX"
218
+ assert parsed_args.get("unit") == "fahrenheit"
219
+
220
+ def test_tool_call_text_not_in_content(self, parser, mock_tokenizer, request_obj):
221
+ """Tool call body must not leak into content."""
222
+ results = _stream_tokens_batched(
223
+ parser,
224
+ mock_tokenizer,
225
+ request_obj,
226
+ batch_size=10,
227
+ prompt_token_ids=[],
228
+ )
229
+
230
+ _, content, _ = _collect_fields(results)
231
+
232
+ assert "call:" not in content, (
233
+ f"Tool call text leaked into content: {content!r}"
234
+ )
235
+ assert "get_current_weather" not in content, (
236
+ f"Function name leaked into content: {content!r}"
237
+ )
238
+
239
+ def test_reasoning_extracted(self, parser, mock_tokenizer, request_obj):
240
+ """Reasoning content should be captured."""
241
+ results = _stream_tokens_batched(
242
+ parser,
243
+ mock_tokenizer,
244
+ request_obj,
245
+ batch_size=10,
246
+ prompt_token_ids=[],
247
+ )
248
+
249
+ reasoning, _, _ = _collect_fields(results)
250
+
251
+ assert "weather" in reasoning.lower(), (
252
+ f"Expected reasoning about weather, got: {reasoning[:100]!r}"
253
+ )
254
+
255
+
256
+# ── Second model output: two tool calls with holdback ────────────────
257
+
258
+REASONING_TEXT_2 = (
259
+ "The user wants me to:\n"
260
+ "1. Perform some reasoning.\n"
261
+ "2. Call a tool to fetch the hostname.\n"
262
+ "3. Call a tool to fetch the current date.\n"
263
+ "\n"
264
+ "Since I am an AI assistant (opencode), I can use the "
265
+ "`bash` tool to execute commands.\n"
266
+ "To get the hostname, I can run `hostname`.\n"
267
+ "To get the current date, I can run `date`.\n"
268
+ "\n"
269
+ "I should do this in a single response with "
270
+ "multiple tool calls for efficiency."
271
+)
272
+
273
+_reasoning_words_2 = REASONING_TEXT_2.split(" ")
274
+_R2_TOKEN_START = 4000
275
+REASONING_TOKENS_2: list[tuple[int, str]] = []
276
+for i, word in enumerate(_reasoning_words_2):
277
+ prefix = " " if i > 0 else ""
278
+ REASONING_TOKENS_2.append((_R2_TOKEN_START + i, prefix + word))
279
+
280
+TOOL_BODY_TOKENS_2A: list[tuple[int, str]] = [
281
+ (5000, "call"),
282
+ (5001, ":"),
283
+ (5002, "bash"),
284
+ (5003, "{"),
285
+ (5004, "command"),
286
+ (5005, ":"),
287
+ (5006, "hostname"),
288
+ (5007, ","),
289
+ (5008, "description"),
290
+ (5009, ":"),
291
+ (5010, "Fetch the hostname of the system."),
292
+ (5011, "}"),
293
+]
294
+
295
+TOOL_BODY_TOKENS_2B: list[tuple[int, str]] = [
296
+ (6000, "call"),
297
+ (6001, ":"),
298
+ (6002, "bash"),
299
+ (6003, "{"),
300
+ (6004, "command"),
301
+ (6005, ":"),
302
+ (6006, "date"),
303
+ (6007, ","),
304
+ (6008, "description"),
305
+ (6009, ":"),
306
+ (6010, "Fetch the current system date and time."),
307
+ (6011, "}"),
308
+]
309
+
310
+FULL_TOKEN_SEQUENCE_2: list[tuple[int, str]] = []
311
+FULL_TOKEN_SEQUENCE_2.append((CHANNEL_START_ID, "<|channel>"))
312
+FULL_TOKEN_SEQUENCE_2.append((3000, "thought"))
313
+FULL_TOKEN_SEQUENCE_2.append((3001, "\n"))
314
+FULL_TOKEN_SEQUENCE_2.extend(REASONING_TOKENS_2)
315
+FULL_TOKEN_SEQUENCE_2.append((CHANNEL_END_ID, "<channel|>"))
316
+FULL_TOKEN_SEQUENCE_2.append((TOOL_CALL_START_ID, "<|tool_call>"))
317
+FULL_TOKEN_SEQUENCE_2.extend(TOOL_BODY_TOKENS_2A[:6])
318
+FULL_TOKEN_SEQUENCE_2.append((QUOTED_ID, '<|"|>'))
319
+FULL_TOKEN_SEQUENCE_2.append(TOOL_BODY_TOKENS_2A[6])
320
+FULL_TOKEN_SEQUENCE_2.append((QUOTED_ID, '<|"|>'))
321
+FULL_TOKEN_SEQUENCE_2.extend(TOOL_BODY_TOKENS_2A[7:10])
322
+FULL_TOKEN_SEQUENCE_2.append((QUOTED_ID, '<|"|>'))
323
+FULL_TOKEN_SEQUENCE_2.append(TOOL_BODY_TOKENS_2A[10])
324
+FULL_TOKEN_SEQUENCE_2.append((QUOTED_ID, '<|"|>'))
325
+FULL_TOKEN_SEQUENCE_2.append(TOOL_BODY_TOKENS_2A[11])
326
+FULL_TOKEN_SEQUENCE_2.append((TOOL_CALL_END_ID, "<tool_call|>"))
327
+FULL_TOKEN_SEQUENCE_2.append((TOOL_CALL_START_ID, "<|tool_call>"))
328
+FULL_TOKEN_SEQUENCE_2.extend(TOOL_BODY_TOKENS_2B[:6])
329
+FULL_TOKEN_SEQUENCE_2.append((QUOTED_ID, '<|"|>'))
330
+FULL_TOKEN_SEQUENCE_2.append(TOOL_BODY_TOKENS_2B[6])
331
+FULL_TOKEN_SEQUENCE_2.append((QUOTED_ID, '<|"|>'))
332
+FULL_TOKEN_SEQUENCE_2.extend(TOOL_BODY_TOKENS_2B[7:10])
333
+FULL_TOKEN_SEQUENCE_2.append((QUOTED_ID, '<|"|>'))
334
+FULL_TOKEN_SEQUENCE_2.append(TOOL_BODY_TOKENS_2B[10])
335
+FULL_TOKEN_SEQUENCE_2.append((QUOTED_ID, '<|"|>'))
336
+FULL_TOKEN_SEQUENCE_2.append(TOOL_BODY_TOKENS_2B[11])
337
+FULL_TOKEN_SEQUENCE_2.append((TOOL_CALL_END_ID, "<tool_call|>"))
338
+
339
+
340
+def _stream_tokens_with_holdback(
341
+ parser,
342
+ tokenizer,
343
+ request,
344
+ batch_size=10,
345
+ holdback_chars=12,
346
+ prompt_token_ids=None,
347
+) -> list[DeltaMessage | None]:
348
+ """Feed tokens in batches with simulated detokenizer holdback."""
349
+ token_ids = tokenizer.encode("", add_special_tokens=False)
350
+ results: list[DeltaMessage | None] = []
351
+ prev_safe_text = ""
352
+
353
+ for start in range(0, len(token_ids), batch_size):
354
+ batch_end = min(start + batch_size, len(token_ids))
355
+ batch_ids = token_ids[start:batch_end]
356
+
357
+ full_decoded = tokenizer.decode(token_ids[:batch_end])
358
+
359
+ if batch_end < len(token_ids):
360
+ safe_len = max(0, len(full_decoded) - holdback_chars)
361
+ safe_text = full_decoded[:safe_len]
362
+ else:
363
+ safe_text = full_decoded
364
+
365
+ delta_text = safe_text[len(prev_safe_text) :]
366
+ prev_safe_text = safe_text
367
+
368
+ result = parser.parse_delta(
369
+ delta_text,
370
+ batch_ids,
371
+ request,
372
+ prompt_token_ids=prompt_token_ids,
373
+ finished=False,
374
+ )
375
+ prompt_token_ids = None
376
+ results.append(result)
377
+ return results
378
+
379
+
380
+class TestGemma4ReasoningTruncationWithHoldback:
381
+ """Reasoning text must not be truncated when detokenizer holds back text."""
382
+
383
+ @pytest.fixture
384
+ def tokenizer_2(self):
385
+ return _make_tokenizer(FULL_TOKEN_SEQUENCE_2)
386
+
387
+ @pytest.fixture
388
+ def parser_2(self, tokenizer_2):
389
+ return Gemma4Parser(tokenizer_2)
390
+
391
+ def test_reasoning_not_truncated(self, parser_2, tokenizer_2, request_obj):
392
+ """Reasoning must include the full text up to <channel|>."""
393
+ results = _stream_tokens_with_holdback(
394
+ parser_2,
395
+ tokenizer_2,
396
+ request_obj,
397
+ batch_size=10,
398
+ holdback_chars=12,
399
+ prompt_token_ids=[],
400
+ )
401
+
402
+ reasoning, content, tool_calls = _collect_fields(results)
403
+
404
+ assert "efficiency" in reasoning, (
405
+ f"Reasoning truncated — missing 'efficiency'. "
406
+ f"Reasoning ends with: {reasoning[-60:]!r}"
407
+ )
408
+
409
+ def test_both_tool_calls_extracted(self, parser_2, tokenizer_2, request_obj):
410
+ """Both bash tool calls must be extracted."""
411
+ results = _stream_tokens_with_holdback(
412
+ parser_2,
413
+ tokenizer_2,
414
+ request_obj,
415
+ batch_size=10,
416
+ holdback_chars=12,
417
+ prompt_token_ids=[],
418
+ )
419
+
420
+ _, _, tool_calls = _collect_fields(results)
421
+
422
+ names = [
423
+ tc.function.name for tc in tool_calls if tc.function and tc.function.name
424
+ ]
425
+ assert len(names) >= 2, f"Expected 2 tool calls, got {len(names)}: {names}"
426
+ assert names.count("bash") >= 2, f"Expected 2 bash tool calls, got {names}"
427
+
428
+ def test_tool_call_text_not_in_content(self, parser_2, tokenizer_2, request_obj):
429
+ """Tool call body must not leak into content."""
430
+ results = _stream_tokens_with_holdback(
431
+ parser_2,
432
+ tokenizer_2,
433
+ request_obj,
434
+ batch_size=10,
435
+ holdback_chars=12,
436
+ prompt_token_ids=[],
437
+ )
438
+
439
+ _, content, _ = _collect_fields(results)
440
+
441
+ assert "call:" not in content, (
442
+ f"Tool call text leaked into content: {content!r}"
443
+ )
444
+
445
+
446
+# ── Simple mock tokenizer for tool-only tests ────────────────────────
447
+
448
+
449
+@pytest.fixture
450
+def tool_call_tokenizer():
451
+ """Mock tokenizer with Gemma4 special token vocab."""
452
+ tokenizer = MagicMock()
453
+ tokenizer.encode.return_value = [1, 2, 3]
454
+ tokenizer.get_vocab.return_value = {
455
+ "<|tool_call>": TOOL_CALL_START_ID,
456
+ "<tool_call|>": TOOL_CALL_END_ID,
457
+ "<|channel>": CHANNEL_START_ID,
458
+ "<channel|>": CHANNEL_END_ID,
459
+ '<|"|>': QUOTED_ID,
460
+ }
461
+ tokenizer.decode.side_effect = lambda ids: "".join(
462
+ SPECIAL_TOKEN_MAP.get(i, chr(i) if i < 128 else f"<{i}>") for i in ids
463
+ )
464
+ return tokenizer
465
+
466
+
467
+@pytest.fixture
468
+def tool_call_parser(tool_call_tokenizer):
469
+ return Gemma4Parser(tool_call_tokenizer)
470
+
471
+
472
+# ── Non-streaming tool call extraction tests ─────────────────────────
473
+
474
+
475
+class TestNonStreamingToolCalls:
476
+ """Non-streaming tool call extraction via extract_tool_calls()."""
477
+
478
+ def test_no_tool_calls(self, tool_call_parser, mock_request):
479
+ result = tool_call_parser.extract_tool_calls(
480
+ "Hello, how can I help you today?",
481
+ mock_request,
482
+ )
483
+ assert result.tools_called is False
484
+ assert result.tool_calls == []
485
+ assert result.content == "Hello, how can I help you today?"
486
+
487
+ def test_single_tool_call(self, tool_call_parser, mock_request):
488
+ text = '<|tool_call>call:get_weather{location:<|"|>London<|"|>}<tool_call|>'
489
+ result = tool_call_parser.extract_tool_calls(text, mock_request)
490
+
491
+ assert result.tools_called is True
492
+ assert len(result.tool_calls) == 1
493
+ assert result.tool_calls[0].function.name == "get_weather"
494
+ args = json.loads(result.tool_calls[0].function.arguments)
495
+ assert args == {"location": "London"}
496
+
497
+ def test_multiple_arguments(self, tool_call_parser, mock_request):
498
+ text = (
499
+ "<|tool_call>call:get_weather{"
500
+ 'location:<|"|>San Francisco<|"|>,'
501
+ 'unit:<|"|>celsius<|"|>}'
502
+ "<tool_call|>"
503
+ )
504
+ result = tool_call_parser.extract_tool_calls(text, mock_request)
505
+
506
+ assert result.tools_called is True
507
+ assert result.tool_calls[0].function.name == "get_weather"
508
+ args = json.loads(result.tool_calls[0].function.arguments)
509
+ assert args == {"location": "San Francisco", "unit": "celsius"}
510
+
511
+ def test_text_before_tool_call(self, tool_call_parser, mock_request):
512
+ text = (
513
+ "Let me check the weather for you. "
514
+ '<|tool_call>call:get_weather{location:<|"|>Paris<|"|>}'
515
+ "<tool_call|>"
516
+ )
517
+ result = tool_call_parser.extract_tool_calls(text, mock_request)
518
+
519
+ assert result.tools_called is True
520
+ assert result.content is not None
521
+ assert "Let me check the weather" in result.content
522
+ assert result.tool_calls[0].function.name == "get_weather"
523
+
524
+ def test_multiple_tool_calls(self, tool_call_parser, mock_request):
525
+ text = (
526
+ '<|tool_call>call:get_weather{location:<|"|>London<|"|>}'
527
+ "<tool_call|>"
528
+ '<|tool_call>call:get_time{location:<|"|>London<|"|>}'
529
+ "<tool_call|>"
530
+ )
531
+ result = tool_call_parser.extract_tool_calls(text, mock_request)
532
+
533
+ assert result.tools_called is True
534
+ assert len(result.tool_calls) == 2
535
+ assert result.tool_calls[0].function.name == "get_weather"
536
+ assert result.tool_calls[1].function.name == "get_time"
537
+
538
+ def test_nested_arguments(self, tool_call_parser, mock_request):
539
+ text = (
540
+ "<|tool_call>call:complex_function{"
541
+ 'nested:{inner:<|"|>value<|"|>},'
542
+ 'list:[<|"|>a<|"|>,<|"|>b<|"|>]}'
543
+ "<tool_call|>"
544
+ )
545
+ result = tool_call_parser.extract_tool_calls(text, mock_request)
546
+
547
+ assert result.tools_called is True
548
+ assert result.tool_calls[0].function.name == "complex_function"
549
+ args = json.loads(result.tool_calls[0].function.arguments)
550
+ assert args == {"nested": {"inner": "value"}, "list": ["a", "b"]}
551
+
552
+ def test_number_and_boolean(self, tool_call_parser, mock_request):
553
+ text = (
554
+ "<|tool_call>call:set_status{"
555
+ "is_active:true,"
556
+ "count:42,"
557
+ "score:3.14}"
558
+ "<tool_call|>"
559
+ )
560
+ result = tool_call_parser.extract_tool_calls(text, mock_request)
561
+
562
+ assert result.tools_called is True
563
+ args = json.loads(result.tool_calls[0].function.arguments)
564
+ assert args == {"is_active": "true", "count": "42", "score": "3.14"}
565
+
566
+ def test_no_arguments(self, tool_call_parser, mock_request):
567
+ text = "<|tool_call>call:get_status{}<tool_call|>"
568
+ result = tool_call_parser.extract_tool_calls(text, mock_request)
569
+
570
+ assert result.tools_called is True
571
+ assert result.tool_calls[0].function.name == "get_status"
572
+ args = json.loads(result.tool_calls[0].function.arguments)
573
+ assert args == {}
574
+
575
+ def test_hyphenated_function_name(self, tool_call_parser, mock_request):
576
+ text = '<|tool_call>call:get-weather{location:<|"|>London<|"|>}<tool_call|>'
577
+ result = tool_call_parser.extract_tool_calls(text, mock_request)
578
+
579
+ assert result.tools_called is True
580
+ assert result.tool_calls[0].function.name == "get-weather"
581
+
582
+ def test_dotted_function_name(self, tool_call_parser, mock_request):
583
+ text = '<|tool_call>call:weather.get{location:<|"|>London<|"|>}<tool_call|>'
584
+ result = tool_call_parser.extract_tool_calls(text, mock_request)
585
+
586
+ assert result.tools_called is True
587
+ assert result.tool_calls[0].function.name == "weather.get"
588
+
589
+
590
+# ── Streaming tool call edge-case tests ──────────────────────────────
591
+
592
+
593
+class TestStreamingToolCallEdgeCases:
594
+ """Streaming tool call extraction via extract_tool_calls_streaming()."""
595
+
596
+ def test_basic_streaming(self, tool_call_parser, mock_request):
597
+ chunks = [
598
+ "<|tool_call>",
599
+ "call:get_weather{",
600
+ 'location:<|"|>Paris',
601
+ ", France",
602
+ '<|"|>}',
603
+ "<tool_call|>",
604
+ ]
605
+
606
+ results = simulate_tool_streaming(tool_call_parser, mock_request, chunks)
607
+
608
+ name = collect_function_name(results)
609
+ assert name == "get_weather"
610
+
611
+ args_text = collect_tool_arguments(results)
612
+ assert args_text
613
+ parsed = json.loads(args_text)
614
+ assert parsed == {"location": "Paris, France"}
615
+
616
+ def test_streaming_multi_arg(self, tool_call_parser, mock_request):
617
+ chunks = [
618
+ "<|tool_call>",
619
+ "call:get_weather{",
620
+ 'location:<|"|>Tokyo<|"|>,',
621
+ 'unit:<|"|>celsius<|"|>}',
622
+ "<tool_call|>",
623
+ ]
624
+
625
+ results = simulate_tool_streaming(tool_call_parser, mock_request, chunks)
626
+
627
+ name = collect_function_name(results)
628
+ assert name == "get_weather"
629
+
630
+ args_text = collect_tool_arguments(results)
631
+ assert args_text
632
+ parsed = json.loads(args_text)
633
+ assert parsed == {"location": "Tokyo", "unit": "celsius"}
634
+
635
+ def test_streaming_no_extra_brace(self, tool_call_parser, mock_request):
636
+ chunks = [
637
+ "<|tool_call>",
638
+ "call:get_weather{",
639
+ 'location:<|"|>London<|"|>}',
640
+ "<tool_call|>",
641
+ ]
642
+
643
+ results = simulate_tool_streaming(tool_call_parser, mock_request, chunks)
644
+ args_text = collect_tool_arguments(results)
645
+ assert args_text
646
+
647
+ parsed = json.loads(args_text)
648
+ assert parsed == {"location": "London"}
649
+ assert args_text.count("}") <= 1
650
+
651
+ def test_streaming_text_before_tool(self, tool_call_parser, mock_request):
652
+ chunks = [
653
+ "Let me check ",
654
+ "the weather. ",
655
+ "<|tool_call>",
656
+ "call:get_weather{",
657
+ 'location:<|"|>London<|"|>}',
658
+ "<tool_call|>",
659
+ ]
660
+
661
+ results = simulate_tool_streaming(tool_call_parser, mock_request, chunks)
662
+ assert collect_content(results).strip().startswith("Let me check")
663
+
664
+ def test_streaming_numeric_args(self, tool_call_parser, mock_request):
665
+ chunks = [
666
+ "<|tool_call>",
667
+ "call:set_config{",
668
+ "count:42,",
669
+ "active:true}",
670
+ "<tool_call|>",
671
+ ]
672
+
673
+ results = simulate_tool_streaming(tool_call_parser, mock_request, chunks)
674
+ args_text = collect_tool_arguments(results)
675
+ if args_text:
676
+ parsed = json.loads(args_text)
677
+ assert parsed["count"] == "42"
678
+ assert parsed["active"] == "true"
679
+
680
+ def test_streaming_empty_args(self, tool_call_parser, mock_request):
681
+ chunks = [
682
+ "<|tool_call>",
683
+ "call:get_status{}",
684
+ "<tool_call|>",
685
+ ]
686
+
687
+ results = simulate_tool_streaming(tool_call_parser, mock_request, chunks)
688
+ name = collect_function_name(results)
689
+ assert name == "get_status"
690
+
691
+ def test_streaming_split_delimiter(self, tool_call_parser, mock_request):
692
+ """Partial <|"|> delimiter must not leak into JSON."""
693
+ chunks = [
694
+ "<|tool_call>",
695
+ "call:todowrite{",
696
+ 'content:<|"|>Buy milk<|',
697
+ '"|>}',
698
+ "<tool_call|>",
699
+ ]
700
+
701
+ results = simulate_tool_streaming(tool_call_parser, mock_request, chunks)
702
+ args_text = collect_tool_arguments(results)
703
+ assert args_text
704
+ parsed = json.loads(args_text)
705
+ assert parsed["content"] == "Buy milk"
706
+ assert "<|" not in args_text
707
+
708
+ def test_streaming_bool_split(self, tool_call_parser, mock_request):
709
+ chunks = [
710
+ "<|tool_call>",
711
+ "call:search{input:{all:t",
712
+ "rue}}",
713
+ "<tool_call|>",
714
+ ]
715
+
716
+ results = simulate_tool_streaming(tool_call_parser, mock_request, chunks)
717
+ args_text = collect_tool_arguments(results)
718
+ assert args_text
719
+ parsed = json.loads(args_text)
720
+ assert parsed["input"]["all"] == "true"
721
+
722
+ def test_streaming_number_split(self, tool_call_parser, mock_request):
723
+ chunks = [
724
+ "<|tool_call>",
725
+ "call:set{count:4",
726
+ "2}",
727
+ "<tool_call|>",
728
+ ]
729
+
730
+ results = simulate_tool_streaming(tool_call_parser, mock_request, chunks)
731
+ args_text = collect_tool_arguments(results)
732
+ assert args_text
733
+ parsed = json.loads(args_text)
734
+ assert parsed["count"] == "42"
735
+
736
+ def test_streaming_trailing_bare_bool(self, tool_call_parser, mock_request):
737
+ chunks = [
738
+ "<|tool_call>",
739
+ "call:Edit{",
740
+ 'file_path:<|"|>src/env.py<|"|>,',
741
+ 'old_string:<|"|>old_val<|"|>,',
742
+ 'new_string:<|"|>new_val<|"|>,',
743
+ "replace_all:",
744
+ "false}",
745
+ "<tool_call|>",
746
+ ]
747
+
748
+ results = simulate_tool_streaming(tool_call_parser, mock_request, chunks)
749
+ args_text = collect_tool_arguments(results)
750
+ assert args_text
751
+
752
+ parsed = json.loads(args_text)
753
+ assert parsed == {
754
+ "file_path": "src/env.py",
755
+ "old_string": "old_val",
756
+ "new_string": "new_val",
757
+ "replace_all": "false",
758
+ }
759
+
760
+ assert args_text.count("replace_all") == 1
761
+
762
+
763
+# ── Non-streaming reasoning + tool call extraction tests ──────────
764
+
765
+
766
+class TestNonStreamingReasoningPlusToolCalls:
767
+ """Non-streaming extraction with reasoning + tool calls."""
768
+
769
+ def test_extract_tool_calls_from_full_text(self, parser, request_obj):
770
+ """extract_tool_calls on full model output must find tools."""
771
+ model_output = FULL_MODEL_OUTPUT
772
+ result = parser.extract_tool_calls(model_output, request_obj)
773
+
774
+ assert result.tools_called is True
775
+ assert len(result.tool_calls) == 1
776
+ assert result.tool_calls[0].function.name == "get_current_weather"
777
+ args = json.loads(result.tool_calls[0].function.arguments)
778
+ assert args["city"] == "Dallas"
779
+ assert args["state"] == "TX"
780
+ assert args["unit"] == "fahrenheit"
781
+
782
+ def test_extract_reasoning_from_full_text(self, parser, request_obj):
783
+ """extract_reasoning on full model output must find reasoning."""
784
+ model_output = FULL_MODEL_OUTPUT
785
+ reasoning, content = parser.extract_reasoning(model_output, request_obj)
786
+
787
+ assert reasoning is not None
788
+ assert "weather" in reasoning.lower()
789
+ assert not reasoning.startswith("thought")
790
+
791
+ def test_bug_report_scenario(self, tool_call_parser, mock_request):
792
+ """Exact scenario from the bug report: get_weather for Raleigh."""
793
+ model_output = (
794
+ "<|channel>thought\n"
795
+ 'The user wants to get the weather for "Raleigh". '
796
+ "I should use the `get_weather` tool and pass "
797
+ '"Raleigh" as the `city` argument.'
798
+ "<channel|>"
799
+ '<|tool_call>call:get_weather{city:<|"|>Raleigh<|"|>}'
800
+ "<tool_call|>"
801
+ )
802
+ result = tool_call_parser.extract_tool_calls(model_output, mock_request)
803
+
804
+ assert result.tools_called is True, (
805
+ f"No tool calls found. content={result.content!r}"
806
+ )
807
+ assert result.tool_calls[0].function.name == "get_weather"
808
+ args = json.loads(result.tool_calls[0].function.arguments)
809
+ assert args["city"] == "Raleigh"
810
+
811
+ def test_both_extractions_independent(self, parser, request_obj):
812
+ """Calling extract_reasoning then extract_tool_calls on the same
813
+ parser instance should both work (each resets the engine)."""
814
+ model_output = FULL_MODEL_OUTPUT
815
+
816
+ reasoning, _ = parser.extract_reasoning(model_output, request_obj)
817
+ result = parser.extract_tool_calls(model_output, request_obj)
818
+
819
+ assert reasoning is not None
820
+ assert "weather" in reasoning.lower()
821
+ assert result.tools_called is True
822
+ assert result.tool_calls[0].function.name == "get_current_weather"
823
+
824
+
825
+class TestAdapterExtractReasoning:
826
+ """The reasoning adapter's extract_reasoning uses skip_tool_parsing
827
+ so tool call text is preserved as content for the tool adapter."""
828
+
829
+ @pytest.fixture
830
+ def adapter(self, mock_tokenizer):
831
+ from vllm.parser.engine.adapters import make_adapters
832
+
833
+ reasoning_cls, _ = make_adapters(Gemma4Parser)
834
+ return reasoning_cls(mock_tokenizer)
835
+
836
+ def test_preserves_tool_text_in_content(self, adapter, request_obj):
837
+ """Tool call markers must appear in content after extraction."""
838
+ reasoning, content = adapter.extract_reasoning(FULL_MODEL_OUTPUT, request_obj)
839
+
840
+ assert reasoning is not None
841
+ assert "weather" in reasoning.lower()
842
+ assert content is not None
843
+ assert "<|tool_call>" in content
844
+ assert "<tool_call|>" in content
845
+ assert "get_current_weather" in content
846
+
847
+ def test_skip_tool_parsing_restored_after_extraction(self, adapter, request_obj):
848
+ """skip_tool_parsing must be restored to its prior value."""
849
+ engine = adapter._parser_engine._engine
850
+ assert engine.skip_tool_parsing is False
851
+ adapter.extract_reasoning(FULL_MODEL_OUTPUT, request_obj)
852
+ assert engine.skip_tool_parsing is False
853
+
854
+ def test_no_reasoning_returns_none(self, adapter, request_obj):
855
+ """Content-only text returns (None, content)."""
856
+ text = "Hello world, no thinking here."
857
+ reasoning, content = adapter.extract_reasoning(text, request_obj)
858
+ assert reasoning is None
859
+ assert content == text
860
+
861
+
862
+# ── Schema-aware type coercion during streaming ────────────────────
863
+
864
+
865
+class TestGemma4SchemaAwareTypeCoercion:
866
+ """Verify that streaming and non-streaming produce identical
867
+ type-fixed arguments when tool schemas declare string parameters
868
+ but the model outputs bare numbers/booleans."""
869
+
870
+ @pytest.fixture
871
+ def tools(self):
872
+ from vllm.entrypoints.openai.chat_completion.protocol import (
873
+ ChatCompletionToolsParam,
874
+ )
875
+
876
+ return [
877
+ ChatCompletionToolsParam(
878
+ type="function",
879
+ function={
880
+ "name": "update_record",
881
+ "parameters": {
882
+ "type": "object",
883
+ "properties": {
884
+ "zipcode": {"type": "string"},
885
+ "count": {"type": "integer"},
886
+ },
887
+ },
888
+ },
889
+ )
890
+ ]
891
+
892
+ @pytest.fixture
893
+ def parser_with_tools(self, tool_call_tokenizer, tools):
894
+ return Gemma4Parser(tool_call_tokenizer, tools=tools)
895
+
896
+ def test_streaming_string_param_not_coerced(self, parser_with_tools, mock_request):
897
+ """A numeric value for a string-typed param must remain a string
898
+ in the streamed output, matching the non-streaming result."""
899
+ chunks = [
900
+ "<|tool_call>",
901
+ "call:update_record{",
902
+ "zipcode:12345}",
903
+ "<tool_call|>",
904
+ ]
905
+
906
+ results = simulate_tool_streaming(parser_with_tools, mock_request, chunks)
907
+ args_text = collect_tool_arguments(results)
908
+ parsed = json.loads(args_text)
909
+ assert parsed["zipcode"] == "12345"
910
+
911
+ def test_streaming_mixed_types(self, parser_with_tools, mock_request):
912
+ """String params get type-fixed, integer params stay integers."""
913
+ chunks = [
914
+ "<|tool_call>",
915
+ "call:update_record{",
916
+ "zipcode:90210,",
917
+ "count:42}",
918
+ "<tool_call|>",
919
+ ]
920
+
921
+ results = simulate_tool_streaming(parser_with_tools, mock_request, chunks)
922
+ args_text = collect_tool_arguments(results)
923
+ parsed = json.loads(args_text)
924
+ assert parsed["zipcode"] == "90210"
925
+ assert parsed["count"] == 42
926
+
927
+ def test_streaming_matches_non_streaming(self, parser_with_tools, mock_request):
928
+ """Concatenated streaming deltas must produce the same arguments
929
+ as non-streaming extraction."""
930
+ text = "<|tool_call>call:update_record{zipcode:12345}<tool_call|>"
931
+
932
+ non_streaming = parser_with_tools.extract_tool_calls(text, mock_request)
933
+ ns_args = json.loads(non_streaming.tool_calls[0].function.arguments)
934
+
935
+ chunks = [
936
+ "<|tool_call>",
937
+ "call:update_record{",
938
+ "zipcode:1234",
939
+ "5}",
940
+ "<tool_call|>",
941
+ ]
942
+ results = simulate_tool_streaming(parser_with_tools, mock_request, chunks)
943
+ s_args = json.loads(collect_tool_arguments(results))
944
+
945
+ assert s_args == ns_args
946
+
947
+
948
+class TestGemma4SchemaCoercionBoolNumberNull:
949
+ """Verify that _fix_arg_types coerces string values to non-string
950
+ schema types for the Gemma4 parser."""
951
+
952
+ @pytest.fixture
953
+ def tools(self):
954
+ from vllm.entrypoints.openai.chat_completion.protocol import (
955
+ ChatCompletionToolsParam,
956
+ )
957
+
958
+ return [
959
+ ChatCompletionToolsParam(
960
+ type="function",
961
+ function={
962
+ "name": "configure",
963
+ "parameters": {
964
+ "type": "object",
965
+ "properties": {
966
+ "enabled": {"type": "boolean"},
967
+ "ratio": {"type": "number"},
968
+ "label": {"type": "string"},
969
+ "value": {"type": ["string", "null"]},
970
+ },
971
+ },
972
+ },
973
+ )
974
+ ]
975
+
976
+ @pytest.fixture
977
+ def parser_with_tools(self, tool_call_tokenizer, tools):
978
+ return Gemma4Parser(tool_call_tokenizer, tools=tools)
979
+
980
+ def test_bool_param_coerced(self, parser_with_tools, mock_request):
981
+ text = "<|tool_call>call:configure{enabled:true}<tool_call|>"
982
+ result = parser_with_tools.extract_tool_calls(text, mock_request)
983
+ args = json.loads(result.tool_calls[0].function.arguments)
984
+ assert args["enabled"] is True
985
+ assert isinstance(args["enabled"], bool)
986
+
987
+ def test_number_whole_normalized(self, parser_with_tools, mock_request):
988
+ text = "<|tool_call>call:configure{ratio:5.0}<tool_call|>"
989
+ result = parser_with_tools.extract_tool_calls(text, mock_request)
990
+ args = json.loads(result.tool_calls[0].function.arguments)
991
+ assert args["ratio"] == 5
992
+ assert isinstance(args["ratio"], int)
993
+
994
+ def test_null_coerced_when_nullable(self, parser_with_tools, mock_request):
995
+ text = "<|tool_call>call:configure{value:null}<tool_call|>"
996
+ result = parser_with_tools.extract_tool_calls(text, mock_request)
997
+ args = json.loads(result.tool_calls[0].function.arguments)
998
+ assert args["value"] is None
999
+
1000
+ def test_null_stays_string_without_null_schema(
1001
+ self, parser_with_tools, mock_request
1002
+ ):
1003
+ text = "<|tool_call>call:configure{label:null}<tool_call|>"
1004
+ result = parser_with_tools.extract_tool_calls(text, mock_request)
1005
+ args = json.loads(result.tool_calls[0].function.arguments)
1006
+ assert args["label"] == "null"
1007
+ assert isinstance(args["label"], str)
1008
+
1009
+ def test_streaming_type_stability(self, parser_with_tools, mock_request):
1010
+ """Values streamed incrementally must not cause prefix
1011
+ incompatibility when types are coerced."""
1012
+ text = (
1013
+ "<|tool_call>call:configure{"
1014
+ "enabled:true,"
1015
+ "ratio:3.14,"
1016
+ "label:hello}"
1017
+ "<tool_call|>"
1018
+ )
1019
+ non_stream = parser_with_tools.extract_tool_calls(text, mock_request)
1020
+ ns_args = json.loads(non_stream.tool_calls[0].function.arguments)
1021
+
1022
+ chunks = [
1023
+ "<|tool_call>",
1024
+ "call:configure{",
1025
+ "enabled:true,",
1026
+ "ratio:3.14,",
1027
+ "label:hello}",
1028
+ "<tool_call|>",
1029
+ ]
1030
+ results = simulate_tool_streaming(parser_with_tools, mock_request, chunks)
1031
+ s_args = json.loads(collect_tool_arguments(results))
1032
+
1033
+ assert s_args == ns_args
1034
+ assert ns_args == {
1035
+ "enabled": True,
1036
+ "ratio": pytest.approx(3.14),
1037
+ "label": "hello",
1038
+ }
1039
+
1040
+
1041
+class TestGemma4NestedSchemaCoercion:
1042
+ """Verify that _fix_arg_types recurses into nested Gemma4 objects."""
1043
+
1044
+ @pytest.fixture
1045
+ def tools(self):
1046
+ from vllm.entrypoints.openai.chat_completion.protocol import (
1047
+ ChatCompletionToolsParam,
1048
+ )
1049
+
1050
+ return [
1051
+ ChatCompletionToolsParam(
1052
+ type="function",
1053
+ function={
1054
+ "name": "search",
1055
+ "parameters": {
1056
+ "type": "object",
1057
+ "properties": {
1058
+ "query": {"type": "string"},
1059
+ "filters": {
1060
+ "type": "object",
1061
+ "properties": {
1062
+ "language": {"type": "string"},
1063
+ "min_stars": {"type": "integer"},
1064
+ },
1065
+ },
1066
+ },
1067
+ },
1068
+ },
1069
+ )
1070
+ ]
1071
+
1072
+ @pytest.fixture
1073
+ def parser_with_tools(self, tool_call_tokenizer, tools):
1074
+ return Gemma4Parser(tool_call_tokenizer, tools=tools)
1075
+
1076
+ def test_nested_object_coerced(self, parser_with_tools, mock_request):
1077
+ text = (
1078
+ "<|tool_call>call:search{"
1079
+ 'query:<|"|>vllm<|"|>,'
1080
+ "filters:{language:python,min_stars:100}}"
1081
+ "<tool_call|>"
1082
+ )
1083
+ result = parser_with_tools.extract_tool_calls(text, mock_request)
1084
+ args = json.loads(result.tool_calls[0].function.arguments)
1085
+ assert args["query"] == "vllm"
1086
+ assert args["filters"]["language"] == "python"
1087
+ assert args["filters"]["min_stars"] == 100
1088
+ assert isinstance(args["filters"]["min_stars"], int)
1089
+
1090
+
1091
+# ── Tests for bare "thought" without channel opener ──────────────────
1092
+
1093
+BARE_THOUGHT_SEQUENCE: list[tuple[int, str]] = []
1094
+BARE_THOUGHT_SEQUENCE.append((3000, "thought"))
1095
+BARE_THOUGHT_SEQUENCE.append((3001, "\n"))
1096
+BARE_THOUGHT_SEQUENCE.extend(REASONING_TOKENS)
1097
+BARE_THOUGHT_SEQUENCE.append((CHANNEL_END_ID, "<channel|>"))
1098
+BARE_THOUGHT_SEQUENCE.append((TOOL_CALL_START_ID, "<|tool_call>"))
1099
+BARE_THOUGHT_SEQUENCE.extend(TOOL_BODY_TOKENS[:4])
1100
+BARE_THOUGHT_SEQUENCE.extend(TOOL_BODY_TOKENS[4:6])
1101
+BARE_THOUGHT_SEQUENCE.append((QUOTED_ID, '<|"|>'))
1102
+BARE_THOUGHT_SEQUENCE.append(TOOL_BODY_TOKENS[6]) # Dallas
1103
+BARE_THOUGHT_SEQUENCE.append((QUOTED_ID, '<|"|>'))
1104
+BARE_THOUGHT_SEQUENCE.append(TOOL_BODY_TOKENS[15]) # }
1105
+BARE_THOUGHT_SEQUENCE.append((TOOL_CALL_END_ID, "<tool_call|>"))
1106
+
1107
+
1108
+class TestBareThoughtWithoutChannelOpener:
1109
+ """When the model omits <|channel> and starts with bare ``thought``,
1110
+ the parser should auto-inject the channel opener so reasoning is
1111
+ captured correctly."""
1112
+
1113
+ @pytest.fixture
1114
+ def bare_thought_tokenizer(self):
1115
+ return _make_tokenizer(BARE_THOUGHT_SEQUENCE)
1116
+
1117
+ @pytest.fixture
1118
+ def bare_thought_parser(self, bare_thought_tokenizer):
1119
+ return Gemma4Parser(bare_thought_tokenizer)
1120
+
1121
+ def test_bare_thought_reasoning_then_tool_call(
1122
+ self, bare_thought_parser, bare_thought_tokenizer, request_obj
1123
+ ):
1124
+ results = _stream_tokens_batched(
1125
+ bare_thought_parser,
1126
+ bare_thought_tokenizer,
1127
+ request_obj,
1128
+ batch_size=1,
1129
+ prompt_token_ids=[],
1130
+ )
1131
+ reasoning, content, tool_calls = _collect_fields(results)
1132
+
1133
+ assert reasoning == REASONING_TEXT
1134
+ assert content == ""
1135
+ assert len(tool_calls) > 0
1136
+ names = [
1137
+ tc.function.name for tc in tool_calls if tc.function and tc.function.name
1138
+ ]
1139
+ assert "get_current_weather" in names
1140
+
1141
+ def test_bare_thought_larger_batches(
1142
+ self, bare_thought_parser, bare_thought_tokenizer, request_obj
1143
+ ):
1144
+ results = _stream_tokens_batched(
1145
+ bare_thought_parser,
1146
+ bare_thought_tokenizer,
1147
+ request_obj,
1148
+ batch_size=10,
1149
+ prompt_token_ids=[],
1150
+ )
1151
+ reasoning, content, tool_calls = _collect_fields(results)
1152
+
1153
+ assert reasoning == REASONING_TEXT
1154
+ assert content == ""
1155
+ assert len(tool_calls) > 0
1156
+
1157
+ def test_normal_content_not_classified_as_reasoning(self, request_obj):
1158
+ content_seq: list[tuple[int, str]] = [
1159
+ (6000, "The"),
1160
+ (6001, " answer"),
1161
+ (6002, " is"),
1162
+ (6003, " 42."),
1163
+ ]
1164
+ tokenizer = _make_tokenizer(content_seq)
1165
+ parser = Gemma4Parser(tokenizer)
1166
+
1167
+ results = _stream_tokens_batched(
1168
+ parser,
1169
+ tokenizer,
1170
+ request_obj,
1171
+ batch_size=2,
1172
+ prompt_token_ids=[],
1173
+ )
1174
+ reasoning, content, tool_calls = _collect_fields(results)
1175
+
1176
+ assert reasoning == ""
1177
+ assert content == "The answer is 42."
1178
+ assert len(tool_calls) == 0
1179
+
1180
+ def test_bare_thought_token_at_end_of_stream(self, request_obj):
1181
+ """When the stream ends with just "thought" (no \\n), the parser
1182
+ should treat it as the thought prefix token, not real reasoning."""
1183
+ seq: list[tuple[int, str]] = [
1184
+ (CHANNEL_START_ID, "<|channel>"),
1185
+ (3000, "thought"),
1186
+ ]
1187
+ tokenizer = _make_tokenizer(seq)
1188
+ parser = Gemma4Parser(tokenizer)
1189
+
1190
+ results = _stream_tokens_batched(
1191
+ parser,
1192
+ tokenizer,
1193
+ request_obj,
1194
+ batch_size=1,
1195
+ prompt_token_ids=[],
1196
+ )
1197
+ reasoning, content, tool_calls = _collect_fields(results)
1198
+
1199
+ assert reasoning == ""
1200
+ assert content == ""
1201
+ assert len(tool_calls) == 0
tests/parser/engine/test_parser_engine.py
+97
@@ -631,6 +631,103 @@ class TestFixArgTypes:
631
original = '{"name": "Alice"}'
632
assert engine._fix_arg_types(original, "f") == original
633
634
+ @pytest.mark.parametrize(
635
+ "properties, input_json, expected_substr",
636
+ [
637
+ ({"count": {"type": "integer"}}, '{"count": "42"}', '"count": 42'),
638
+ ({"score": {"type": "number"}}, '{"score": "3.14"}', '"score": 3.14'),
639
+ ({"flag": {"type": "boolean"}}, '{"flag": "true"}', '"flag": true'),
640
+ ({"flag": {"type": "boolean"}}, '{"flag": "false"}', '"flag": false'),
641
+ ({"val": {"type": "null"}}, '{"val": "null"}', '"val": null'),
642
+ ({"val": {"type": ["string", "null"]}}, '{"val": "null"}', '"val": null'),
643
+ ({"score": {"type": "number"}}, '{"score": "108."}', '"score": 108'),
644
+ ],
645
+ ids=[
646
+ "string_to_int",
647
+ "string_to_float",
648
+ "string_to_bool_true",
649
+ "string_to_bool_false",
650
+ "string_to_null",
651
+ "string_to_null_union",
652
+ "trailing_dot_float",
653
+ ],
654
+ )
655
+ def test_string_coerced_to_schema_type(
656
+ self,
657
+ properties,
658
+ input_json,
659
+ expected_substr,
660
+ ):
661
+ tool = _make_tool("f", properties)
662
+ engine = _make_engine(tools=[tool])
663
+ result = engine._fix_arg_types(input_json, "f")
664
+ assert expected_substr in result
665
+
666
+ def test_mixed_types_coerced(self):
667
+ tool = _make_tool(
668
+ "f",
669
+ {
670
+ "count": {"type": "integer"},
671
+ "active": {"type": "boolean"},
672
+ "score": {"type": "number"},
673
+ },
674
+ )
675
+ engine = _make_engine(tools=[tool])
676
+ result = engine._fix_arg_types(
677
+ '{"count": "42", "active": "true", "score": "3.14"}', "f"
678
+ )
679
+ parsed = json.loads(result)
680
+ assert parsed["count"] == 42
681
+ assert parsed["active"] is True
682
+ assert parsed["score"] == 3.14
683
+
684
+ def test_nested_object_coercion(self):
685
+ tool = _make_tool(
686
+ "f",
687
+ {
688
+ "inner": {
689
+ "type": "object",
690
+ "properties": {
691
+ "count": {"type": "integer"},
692
+ },
693
+ },
694
+ },
695
+ )
696
+ engine = _make_engine(tools=[tool])
697
+ result = engine._fix_arg_types('{"inner": {"count": "42"}}', "f")
698
+ parsed = json.loads(result)
699
+ assert parsed["inner"]["count"] == 42
700
+
701
+ def test_array_item_coercion(self):
702
+ tool = _make_tool(
703
+ "f",
704
+ {
705
+ "nums": {
706
+ "type": "array",
707
+ "items": {"type": "integer"},
708
+ },
709
+ },
710
+ )
711
+ engine = _make_engine(tools=[tool])
712
+ result = engine._fix_arg_types('{"nums": ["42", "5"]}', "f")
713
+ parsed = json.loads(result)
714
+ assert parsed["nums"] == [42, 5]
715
+
716
+ def test_array_mixed_item_types(self):
717
+ tool = _make_tool(
718
+ "f",
719
+ {
720
+ "vals": {
721
+ "type": "array",
722
+ "items": {"type": "number"},
723
+ },
724
+ },
725
+ )
726
+ engine = _make_engine(tools=[tool])
727
+ result = engine._fix_arg_types('{"vals": ["42", "3.14"]}', "f")
728
+ parsed = json.loads(result)
729
+ assert parsed["vals"] == [42, 3.14]
730
+
731
732
# ── TestBuildExtractedResult ─────────────────────────────────────────
733
tests/parser/engine/test_replay.py
+85
-1
@@ -17,19 +17,25 @@ from tests.parser.engine.replay_harness import (
17
collect_output,
18
make_mock_tokenizer,
19
replay_streaming,
20
+ replay_with_text_holdback,
21
)
22
from tests.parser.engine.trace_builder import build_samples
23
from vllm.parser.abstract_parser import Parser
24
from vllm.parser.engine.registered_adapters import (
25
+ Gemma4Parser,
26
Qwen3Parser,
27
)
28
29
_ENGINE_PARSERS: dict[str, type[Parser]] = {
30
"qwen3_engine": Qwen3Parser,
31
+ "gemma4_engine": Gemma4Parser,
32
}
33
34
+_gemma4_samples = build_samples("gemma4")
35
_qwen3_samples = build_samples("qwen3")
36
37
+_GEMMA4_TERMINALS = ["<|channel>", "<channel|>", "<|tool_call>", "<tool_call|>"]
38
+
39
_QWEN3_TERMINALS = [
40
"<think>",
41
"</think>",
@@ -56,6 +62,7 @@ class TestQwen3ReplayWithHoldback:
62
sample.tokens,
63
chunk_size=chunk_size,
64
holdback_chars=holdback,
65
+ prompt_token_ids=sample.prompt_token_ids,
66
)
67
output = collect_output(deltas)
68
@@ -67,10 +74,85 @@ class TestQwen3ReplayWithHoldback:
74
)
75
76
77
+@pytest.mark.parametrize("holdback", HOLDBACK_CONFIGS, ids=lambda h: f"holdback{h}")
78
+@pytest.mark.parametrize("chunk_size", [3, 5, 10], ids=lambda c: f"chunk{c}")
79
+@pytest.mark.parametrize("sample", _gemma4_samples, ids=lambda s: s.id)
80
+class TestGemma4ReplayWithHoldback:
81
+ """Replay with simulated detokenizer holdback."""
82
+
83
+ def test_replay(self, sample, chunk_size, holdback):
84
+ tokenizer = make_mock_tokenizer(sample)
85
+ parser = Gemma4Parser(tokenizer, sample.tools)
86
+ deltas = replay_streaming(
87
+ parser,
88
+ sample.tokens,
89
+ chunk_size=chunk_size,
90
+ holdback_chars=holdback,
91
+ prompt_token_ids=sample.prompt_token_ids,
92
+ )
93
+ output = collect_output(deltas)
94
+
95
+ assert_parse_output(output, sample)
96
+ assert_no_terminal_leakage(
97
+ output,
98
+ _GEMMA4_TERMINALS,
99
+ context=f"chunk_size={chunk_size}, holdback={holdback}",
100
+ )
101
+
102
+
103
+TEXT_HOLDBACK_DELAYS = [1, 2, 3]
104
+
105
+
106
+@pytest.mark.parametrize("delay", TEXT_HOLDBACK_DELAYS, ids=lambda d: f"delay{d}")
107
+@pytest.mark.parametrize("sample", _gemma4_samples, ids=lambda s: s.id)
108
+class TestGemma4TextHoldback:
109
+ """Replay with production-like text/token-ID misalignment.
110
+
111
+ In production the detokenizer sends token IDs immediately but holds
112
+ back text by N tokens. This exercises the TokenIDScanner deferred
113
+ terminal path that aligned-holdback tests do not cover.
114
+ """
115
+
116
+ def test_replay(self, sample, delay):
117
+ tokenizer = make_mock_tokenizer(sample)
118
+ parser = Gemma4Parser(tokenizer, sample.tools)
119
+ deltas = replay_with_text_holdback(
120
+ parser,
121
+ sample.tokens,
122
+ text_delay=delay,
123
+ prompt_token_ids=sample.prompt_token_ids,
124
+ )
125
+ output = collect_output(deltas)
126
+
127
+ assert_parse_output(output, sample)
128
+ assert_no_terminal_leakage(
129
+ output,
130
+ _GEMMA4_TERMINALS,
131
+ context=f"text_delay={delay}",
132
+ )
133
+
134
+
135
+class TestParserEngineAdjustRequest:
136
+ """Verify ParserEngine and its adapters set skip_special_tokens=False."""
137
+
138
+ def test_adjust_request_disables_skip_special_tokens(self):
139
+ sample = _gemma4_samples[0]
140
+ tokenizer = make_mock_tokenizer(sample)
141
+ parser = Gemma4Parser(tokenizer, sample.tools)
142
+ request = _test_request()
143
+ assert request.skip_special_tokens is True
144
+ adjusted = parser.adjust_request(request)
145
+ assert adjusted.skip_special_tokens is False
146
+
147
+
148
_TOOL_CALL_SAMPLES = [
149
(Qwen3Parser, s)
150
for s in _qwen3_samples
151
if s.expected_tool_calls and s.expected_reasoning
152
+] + [
153
+ (Gemma4Parser, s)
154
+ for s in _gemma4_samples
155
+ if s.expected_tool_calls and s.expected_reasoning
156
]
157
158
@@ -147,7 +229,9 @@ class TestSkipToolParsingReplay:
229
"".join(all_texts[start:end]),
230
all_ids[start:end],
231
request,
150
- prompt_token_ids=[] if start == 0 else None,
232
+ prompt_token_ids=(sample.prompt_token_ids or [])
233
+ if start == 0
234
+ else None,
235
finished=is_last,
236
)
237
results.append(result)
tests/parser/engine/test_token_id_scanner.py
+528
-124
@@ -1,20 +1,19 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-"""Tests for TokenIDScanner, focusing on hold-back text recovery.
4
-
5
-Uses gemma4_config for all end-to-end engine tests, covering
6
-reasoning channels, tool calls, and combined flows."""
3
+"""Tests for TokenIDScanner."""
4
5
from unittest.mock import MagicMock
6
7
import pytest
8
9
from vllm.parser.engine.events import EventType
10
+from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
11
from vllm.parser.engine.token_id_scanner import (
12
PreLexedTerminal,
13
TextChunk,
14
TokenIDScanner,
15
)
16
+from vllm.parser.gemma4 import gemma4_config
17
18
CHANNEL_START = "<|channel>"
19
CHANNEL_END = "<channel|>"
@@ -54,8 +53,7 @@ def scanner(tokenizer):
53
54
55
class TestJoinDecodedTextReturnsStr:
57
- """_join_decoded_text now returns str unconditionally (was
58
- str | None when an isinstance guard made a branch unreachable)."""
56
+ """_join_decoded_text always returns str."""
57
58
@pytest.fixture
59
def bare_scanner(self):
@@ -83,9 +81,7 @@ class TestJoinDecodedTextReturnsStr:
81
82
class TestHoldbackTextRecovery:
83
def test_holdback_text_with_special_token_text_absent(self, scanner):
86
- """delta_text has hold-back text but the special token's text is
87
- NOT in delta_text (held back by the detokenizer). Terminal is
88
- deferred until the text arrives in a subsequent delta."""
84
+ """Terminal deferred when its text is absent from delta_text."""
85
result = scanner.scan(
86
delta_text="processed is appropriate.",
87
delta_token_ids=[CHANNEL_END_ID],
@@ -93,8 +89,6 @@ class TestHoldbackTextRecovery:
89
90
assert len(result) == 0
91
96
- # Second scan: terminal text arrives (detokenizer flushes).
97
- # Deferred terminal resolves with holdback text before it.
92
result2 = scanner.scan(
93
delta_text="<channel|>Understood.",
94
delta_token_ids=[20, 21],
@@ -108,7 +102,7 @@ class TestHoldbackTextRecovery:
102
assert "Understood." in combined
103
104
def test_holdback_text_with_special_token_text_present(self, scanner):
111
- """delta_text includes hold-back text AND the special token text."""
105
+ """Hold-back text + special token text both in delta_text."""
106
result = scanner.scan(
107
delta_text="holdback text<channel|>",
108
delta_token_ids=[CHANNEL_END_ID],
@@ -121,7 +115,7 @@ class TestHoldbackTextRecovery:
115
assert result[1].terminal == "THINK_END"
116
117
def test_no_holdback_text(self, scanner):
124
- """delta_text is exactly the special token text — no hold-back."""
118
+ """delta_text is exactly the special token text."""
119
result = scanner.scan(
120
delta_text="<channel|>",
121
delta_token_ids=[CHANNEL_END_ID],
@@ -132,7 +126,7 @@ class TestHoldbackTextRecovery:
126
assert result[0].terminal == "THINK_END"
127
128
def test_empty_delta_text(self, scanner):
135
- """delta_text is empty — terminal deferred until text arrives."""
129
+ """Empty delta_text defers the terminal until text arrives."""
130
result = scanner.scan(
131
delta_text="",
132
delta_token_ids=[CHANNEL_END_ID],
@@ -146,9 +140,7 @@ class TestHoldbackTextRecovery:
140
assert flushed[0].terminal == "THINK_END"
141
142
def test_empty_delta_text_drops_individual_decode_text(self, tokenizer):
149
- """delta_text="" with multiple tokens including special: all
150
- results deferred — individually-decoded TextChunks are unreliable
151
- and PreLexedTerminals wait for text confirmation."""
143
+ """Empty delta_text with multiple tokens: all results deferred."""
144
tool_start_id = 400
145
tok_a = 201
146
tok_b = 202
@@ -176,7 +168,6 @@ class TestHoldbackTextRecovery:
168
assert flushed[0].terminal == "TOOL_START"
169
170
def test_holdback_before_start_tag(self, scanner):
179
- """Hold-back text before a reasoning start tag."""
171
result = scanner.scan(
172
delta_text="prefix text<|channel>",
173
delta_token_ids=[CHANNEL_START_ID],
@@ -189,8 +180,7 @@ class TestHoldbackTextRecovery:
180
assert result[1].terminal == "THINK_START"
181
182
def test_multi_token_batch_special_in_middle(self, scanner, tokenizer):
192
- """Stream-interval > 1: batch has regular tokens + special token.
193
- delta_text differs from individual decodes (context-dependent)."""
183
+ """Multi-token batch with special token in the middle."""
184
tok_a = 201
185
tok_b = 202
186
tokenizer.decode.side_effect = lambda ids: {
@@ -215,10 +205,7 @@ class TestHoldbackTextRecovery:
205
assert "holdback wordA" in "".join(texts)
206
207
def test_multi_token_batch_special_token_text_absent(self, scanner, tokenizer):
218
- """Stream-interval > 1: batch has regular + special token, but
219
- delta_text doesn't contain the special token text at all
220
- (held back by detokenizer along with trailing regular tokens).
221
- Terminal is deferred until text arrives."""
208
+ """Multi-token batch where special token text is absent."""
209
tok_a = 201
210
tok_b = 202
211
tokenizer.decode.side_effect = lambda ids: {
@@ -239,8 +226,6 @@ class TestHoldbackTextRecovery:
226
227
assert len(result) == 0
228
242
- # Next delta: terminal text arrives (detokenizer flushes).
243
- # Deferred terminal resolves with holdback text before it.
229
result2 = scanner_multi.scan(
230
delta_text="<channel|> more text",
231
delta_token_ids=[300],
@@ -254,8 +239,7 @@ class TestHoldbackTextRecovery:
239
assert "more text" in combined
240
241
def test_holdback_with_content_after_special_token(self, tokenizer):
257
- """delta_text has hold-back + special token + content after,
258
- with corresponding token IDs for all parts."""
242
+ """Hold-back + special token + content after in one delta."""
243
tok_content = 210
244
tokenizer.decode.side_effect = lambda ids: {
245
CHANNEL_END_ID: CHANNEL_END,
@@ -283,8 +267,7 @@ class TestHoldbackTextRecovery:
267
268
class TestDropTokens:
269
def test_drop_token_with_holdback(self, tokenizer):
286
- """Drop tokens stripped from delta_text, hold-back text preserved.
287
- Terminal is deferred when its text is absent from delta_text."""
270
+ """Drop tokens stripped; hold-back text preserved."""
271
drop_id = 300
272
tokenizer.decode.side_effect = lambda ids: {
273
CHANNEL_END_ID: CHANNEL_END,
@@ -304,7 +287,6 @@ class TestDropTokens:
287
288
assert len(result) == 0
289
307
- # Terminal text arrives in next delta; deferred terminal resolves.
290
result2 = scanner.scan(
291
delta_text="<channel|>content",
292
delta_token_ids=[20],
@@ -321,40 +303,10 @@ class TestDropTokens:
303
304
305
class TestEndToEndReasoningHoldback:
324
- """End-to-end tests through the full parser engine simulating
325
- stream-interval > 1 and detokenizer hold-back, using
326
- gemma4_config."""
306
+ """End-to-end engine tests with detokenizer hold-back."""
307
308
def test_reasoning_content_not_truncated(self):
329
- from vllm.parser.engine.parser_engine_config import (
330
- ParserEngineConfig,
331
- ParserState,
332
- Transition,
333
- )
334
- from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
335
-
336
- config = ParserEngineConfig(
337
- name="test_channel",
338
- initial_state=ParserState.CONTENT,
339
- terminals={
340
- "THINK_START": CHANNEL_START,
341
- "THINK_END": CHANNEL_END,
342
- },
343
- token_id_terminals={
344
- "THINK_START": CHANNEL_START,
345
- "THINK_END": CHANNEL_END,
346
- },
347
- transitions={
348
- (ParserState.CONTENT, "THINK_START"): Transition(
349
- ParserState.REASONING,
350
- (EventType.REASONING_START,),
351
- ),
352
- (ParserState.REASONING, "THINK_END"): Transition(
353
- ParserState.CONTENT,
354
- (EventType.REASONING_END,),
355
- ),
356
- },
357
- )
309
+ config = gemma4_config()
310
tok = MagicMock()
311
vocab = {
312
CHANNEL_START: CHANNEL_START_ID,
@@ -369,30 +321,21 @@ class TestEndToEndReasoningHoldback:
321
engine = StreamingParserEngine(config, tok)
322
all_events = []
323
372
- # Delta 1: channel start token (text includes start tag)
324
all_events.extend(engine.feed(CHANNEL_START, [CHANNEL_START_ID]))
374
-
375
- # Delta 2: reasoning text (normal content, no special tokens)
325
all_events.extend(
326
engine.feed(
327
"thought\nThe request was received and ",
328
[10, 11, 12, 13, 14],
329
)
330
)
382
-
383
- # Delta 3: MORE reasoning text, the detokenizer held some back.
384
- # Then channel end token arrives in token_ids, but its text
385
- # is NOT in delta_text (held back by detokenizer).
386
- # delta_text = previously held-back reasoning text only.
331
+ # CHANNEL_END token arrives but its text is held back.
332
all_events.extend(
333
engine.feed(
334
"processed is appropriate.",
335
[CHANNEL_END_ID],
336
)
337
)
393
-
394
- # Delta 4: detokenizer flushes held-back channel end text
395
- # plus new content tokens.
338
+ # Detokenizer flushes the held-back text.
339
all_events.extend(
340
engine.feed(
341
"<channel|>Understood.",
@@ -413,36 +356,7 @@ class TestEndToEndReasoningHoldback:
356
assert "Understood." in content_text
357
358
def test_backtick_content_not_truncated(self):
416
- """Reproduces the hostname backtick truncation case."""
417
- from vllm.parser.engine.parser_engine_config import (
418
- ParserEngineConfig,
419
- ParserState,
420
- Transition,
421
- )
422
- from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
423
-
424
- config = ParserEngineConfig(
425
- name="test_channel",
426
- initial_state=ParserState.CONTENT,
427
- terminals={
428
- "THINK_START": CHANNEL_START,
429
- "THINK_END": CHANNEL_END,
430
- },
431
- token_id_terminals={
432
- "THINK_START": CHANNEL_START,
433
- "THINK_END": CHANNEL_END,
434
- },
435
- transitions={
436
- (ParserState.CONTENT, "THINK_START"): Transition(
437
- ParserState.REASONING,
438
- (EventType.REASONING_START,),
439
- ),
440
- (ParserState.REASONING, "THINK_END"): Transition(
441
- ParserState.CONTENT,
442
- (EventType.REASONING_END,),
443
- ),
444
- },
445
- )
359
+ config = gemma4_config()
360
tok = MagicMock()
361
vocab = {
362
CHANNEL_START: CHANNEL_START_ID,
@@ -464,17 +378,12 @@ class TestEndToEndReasoningHoldback:
378
[10, 11, 12, 13],
379
)
380
)
467
-
468
- # Hold-back text includes backtick content; channel end text
469
- # absent from delta_text.
381
all_events.extend(
382
engine.feed(
383
"`hostname`.\n",
384
[CHANNEL_END_ID],
385
)
386
)
476
-
477
- # Next delta flushes channel end + tool call start
387
all_events.extend(
388
engine.feed(
389
"<channel|>tool output",
@@ -491,10 +400,516 @@ class TestEndToEndReasoningHoldback:
400
assert "`hostname`." in reasoning_text
401
402
403
+_CHANNEL_START_TAG = "<|channel>"
404
+_CHANNEL_END_TAG = "<channel|>"
405
+_TOOL_START_TAG = "<|tool_call>"
406
+_TOOL_END_TAG = "<tool_call|>"
407
+_QUOTE_TAG = '<|"|>'
408
+
409
+_CHANNEL_START_TID = 100
410
+_CHANNEL_END_TID = 101
411
+_TOOL_START_TID = 102
412
+_TOOL_END_TID = 103
413
+_QUOTE_TID = 104
414
+_TOK = list(range(200, 215))
415
+
416
+
417
+def _gemma4_vocab() -> dict[str, int]:
418
+ return {
419
+ _CHANNEL_START_TAG: _CHANNEL_START_TID,
420
+ _CHANNEL_END_TAG: _CHANNEL_END_TID,
421
+ _TOOL_START_TAG: _TOOL_START_TID,
422
+ _TOOL_END_TAG: _TOOL_END_TID,
423
+ _QUOTE_TAG: _QUOTE_TID,
424
+ }
425
+
426
+
427
+def _make_gemma4_tokenizer(
428
+ extra_decode: dict[int, str] | None = None,
429
+) -> MagicMock:
430
+ special = {
431
+ _CHANNEL_START_TID: _CHANNEL_START_TAG,
432
+ _CHANNEL_END_TID: _CHANNEL_END_TAG,
433
+ _TOOL_START_TID: _TOOL_START_TAG,
434
+ _TOOL_END_TID: _TOOL_END_TAG,
435
+ _QUOTE_TID: _QUOTE_TAG,
436
+ }
437
+ decode_map = {**special, **(extra_decode or {})}
438
+
439
+ tok = MagicMock()
440
+ tok.get_vocab.return_value = _gemma4_vocab()
441
+ tok.decode.side_effect = lambda ids: decode_map.get(ids[0], f"tok{ids[0]}")
442
+ return tok
443
+
444
+
445
+def _collect_events(engine, deltas):
446
+ from vllm.parser.engine.events import SemanticEvent
447
+
448
+ all_events: list[SemanticEvent] = []
449
+ for delta_text, delta_token_ids in deltas:
450
+ all_events.extend(engine.feed(delta_text, delta_token_ids))
451
+ all_events.extend(engine.finish())
452
+ return all_events
453
+
454
+
455
+def _reasoning_text(events) -> str:
456
+ return "".join(e.value for e in events if e.type == EventType.REASONING_CHUNK)
457
+
458
+
459
+def _content_text(events) -> str:
460
+ return "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
461
+
462
+
463
+def _arg_text(events) -> str:
464
+ return "".join(e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK)
465
+
466
+
467
+def _has_event(events, event_type) -> bool:
468
+ return any(e.type == event_type for e in events)
469
+
470
+
471
+class TestMultiTokenBoundaryPreservation:
472
+ """No text lost at state boundaries with multi-token deltas."""
473
+
474
+ def test_empty_delta_text_at_channel_end_unified(self):
475
+ """Empty delta_text when CHANNEL_END arrives; text comes later."""
476
+ tok = _make_gemma4_tokenizer()
477
+ engine = StreamingParserEngine(gemma4_config(), tok)
478
+
479
+ events = _collect_events(
480
+ engine,
481
+ [
482
+ ("", [_CHANNEL_START_TID]),
483
+ ("<|channel>thought\nSome reasoning.", [_TOK[0], _TOK[1]]),
484
+ ("", [_CHANNEL_END_TID]),
485
+ ("<channel|>Final answer.", [_TOK[2], _TOK[3]]),
486
+ ],
487
+ )
488
+
489
+ reasoning = _reasoning_text(events)
490
+ content = _content_text(events)
491
+ assert "Some reasoning." in reasoning
492
+ assert "Final answer." in content
493
+ assert _has_event(events, EventType.REASONING_START)
494
+ assert _has_event(events, EventType.REASONING_END)
495
+
496
+ def test_deferred_channel_end_flushed_at_finish_unified(self):
497
+ """Deferred CHANNEL_END flushed at end-of-stream."""
498
+ tok = _make_gemma4_tokenizer()
499
+ engine = StreamingParserEngine(gemma4_config(), tok)
500
+
501
+ events = _collect_events(
502
+ engine,
503
+ [
504
+ (_CHANNEL_START_TAG, [_CHANNEL_START_TID]),
505
+ ("thought\nReasoning text.", [_TOK[0]]),
506
+ (" Final thought.", [_CHANNEL_END_TID]),
507
+ ],
508
+ )
509
+
510
+ reasoning = _reasoning_text(events)
511
+ assert "Reasoning text. Final thought." in reasoning
512
+ assert _has_event(events, EventType.REASONING_END)
513
+
514
+ def test_reasoning_to_tool_call_handoff_unified(self):
515
+ """Full reasoning -> content -> tool call flow."""
516
+ tok = _make_gemma4_tokenizer()
517
+ engine = StreamingParserEngine(gemma4_config(), tok)
518
+
519
+ events = _collect_events(
520
+ engine,
521
+ [
522
+ (_CHANNEL_START_TAG, [_CHANNEL_START_TID]),
523
+ ("thought\nI need to check the weather.", [_TOK[0], _TOK[1], _TOK[2]]),
524
+ (_CHANNEL_END_TAG, [_CHANNEL_END_TID]),
525
+ ("Let me call a tool.", [_TOK[3], _TOK[4]]),
526
+ (_TOOL_START_TAG, [_TOOL_START_TID]),
527
+ ("call:get_weather{city:", [_TOK[5], _TOK[6]]),
528
+ ('<|"|>SF<|"|>}', [_QUOTE_TID, _TOK[7], _QUOTE_TID, _TOK[8]]),
529
+ (_TOOL_END_TAG, [_TOOL_END_TID]),
530
+ ],
531
+ )
532
+
533
+ reasoning = _reasoning_text(events)
534
+ content = _content_text(events)
535
+
536
+ assert "I need to check the weather." in reasoning
537
+ assert "Let me call a tool." in content
538
+ assert _has_event(events, EventType.REASONING_START)
539
+ assert _has_event(events, EventType.REASONING_END)
540
+ assert _has_event(events, EventType.TOOL_CALL_START)
541
+ assert _has_event(events, EventType.TOOL_CALL_END)
542
+ assert "SF" in _arg_text(events)
543
+
544
+ def test_multiple_tool_calls_rapid_transitions_unified(self):
545
+ """Two back-to-back tool calls with correct tool_index tracking."""
546
+ tok = _make_gemma4_tokenizer()
547
+ engine = StreamingParserEngine(gemma4_config(), tok)
548
+
549
+ events = _collect_events(
550
+ engine,
551
+ [
552
+ (_TOOL_START_TAG, [_TOOL_START_TID]),
553
+ ("call:get_weather{city:", [_TOK[0], _TOK[1]]),
554
+ ('<|"|>NYC<|"|>}', [_QUOTE_TID, _TOK[2], _QUOTE_TID, _TOK[3]]),
555
+ (_TOOL_END_TAG, [_TOOL_END_TID]),
556
+ (_TOOL_START_TAG, [_TOOL_START_TID]),
557
+ ("call:get_time{tz:", [_TOK[4], _TOK[5]]),
558
+ ('<|"|>EST<|"|>}', [_QUOTE_TID, _TOK[6], _QUOTE_TID, _TOK[7]]),
559
+ (_TOOL_END_TAG, [_TOOL_END_TID]),
560
+ ],
561
+ )
562
+
563
+ starts = [e for e in events if e.type == EventType.TOOL_CALL_START]
564
+ ends = [e for e in events if e.type == EventType.TOOL_CALL_END]
565
+ assert len(starts) == 2
566
+ assert len(ends) == 2
567
+ assert starts[0].tool_index == 0
568
+ assert starts[1].tool_index == 1
569
+
570
+ names = "".join(e.value for e in events if e.type == EventType.TOOL_NAME)
571
+ assert "get_weather" in names
572
+ assert "get_time" in names
573
+
574
+ def test_deferred_channel_end_before_tool_call_unified(self):
575
+ """Deferred CHANNEL_END followed by a tool call."""
576
+ tok = _make_gemma4_tokenizer()
577
+ engine = StreamingParserEngine(gemma4_config(), tok)
578
+
579
+ events = _collect_events(
580
+ engine,
581
+ [
582
+ (_CHANNEL_START_TAG, [_CHANNEL_START_TID]),
583
+ ("thought\nNeed to call a tool.", [_TOK[0], _TOK[1]]),
584
+ (" Let me proceed.", [_CHANNEL_END_TID]),
585
+ (_CHANNEL_END_TAG, [_TOK[2]]),
586
+ (_TOOL_START_TAG, [_TOOL_START_TID]),
587
+ ("call:get_weather{city:", [_TOK[3], _TOK[4]]),
588
+ ('<|"|>Tokyo<|"|>}', [_QUOTE_TID, _TOK[5], _QUOTE_TID, _TOK[6]]),
589
+ (_TOOL_END_TAG, [_TOOL_END_TID]),
590
+ ],
591
+ )
592
+
593
+ reasoning = _reasoning_text(events)
594
+ assert "Need to call a tool. Let me proceed." in reasoning
595
+ assert _has_event(events, EventType.REASONING_END)
596
+ assert _has_event(events, EventType.TOOL_CALL_START)
597
+ assert _has_event(events, EventType.TOOL_CALL_END)
598
+ assert "Tokyo" in _arg_text(events)
599
+
600
+
601
+class TestStreamInterval10:
602
+ """Tests with stream_interval=10 (large multi-token batches)."""
603
+
604
+ def test_channel_end_mid_batch_text_present(self):
605
+ """<channel|> mid-batch with its text present in delta_text."""
606
+ tok = _make_gemma4_tokenizer({_TOK[i]: f"word{i} " for i in range(15)})
607
+ engine = StreamingParserEngine(gemma4_config(), tok)
608
+
609
+ events: list = []
610
+ events.extend(
611
+ engine.feed(
612
+ "<|channel>thought\nword0 word1 word2 word3 word4 "
613
+ "word5 word6 word7 word8 ",
614
+ [
615
+ _CHANNEL_START_TID,
616
+ _TOK[0],
617
+ _TOK[1],
618
+ _TOK[2],
619
+ _TOK[3],
620
+ _TOK[4],
621
+ _TOK[5],
622
+ _TOK[6],
623
+ _TOK[7],
624
+ _TOK[8],
625
+ ],
626
+ )
627
+ )
628
+
629
+ events.extend(
630
+ engine.feed(
631
+ "word9 word10 word11 <channel|>word12 word13 word14 word0 word1 word2 ",
632
+ [
633
+ _TOK[9],
634
+ _TOK[10],
635
+ _TOK[11],
636
+ _CHANNEL_END_TID,
637
+ _TOK[12],
638
+ _TOK[13],
639
+ _TOK[14],
640
+ _TOK[0],
641
+ _TOK[1],
642
+ _TOK[2],
643
+ ],
644
+ )
645
+ )
646
+
647
+ events.extend(engine.finish())
648
+
649
+ reasoning = _reasoning_text(events)
650
+ content = _content_text(events)
651
+
652
+ for w in ("word9", "word10", "word11"):
653
+ assert w in reasoning, f"{w!r} missing from reasoning"
654
+
655
+ for w in ("word12", "word13", "word14"):
656
+ assert w in content, f"{w!r} missing from content"
657
+
658
+ assert _has_event(events, EventType.REASONING_END)
659
+
660
+ def test_channel_end_and_tool_start_same_batch_unified(self):
661
+ """Both <channel|> and <|tool_call> in a single batch."""
662
+ tok = _make_gemma4_tokenizer({_TOK[i]: f"w{i} " for i in range(15)})
663
+ engine = StreamingParserEngine(gemma4_config(), tok)
664
+
665
+ events: list = []
666
+
667
+ events.extend(
668
+ engine.feed(
669
+ "<|channel>thought\nw0 w1 w2 w3 w4 w5 w6 w7 w8 ",
670
+ [
671
+ _CHANNEL_START_TID,
672
+ _TOK[0],
673
+ _TOK[1],
674
+ _TOK[2],
675
+ _TOK[3],
676
+ _TOK[4],
677
+ _TOK[5],
678
+ _TOK[6],
679
+ _TOK[7],
680
+ _TOK[8],
681
+ ],
682
+ )
683
+ )
684
+
685
+ events.extend(
686
+ engine.feed(
687
+ "w9 w10 <channel|>w11 <|tool_call>",
688
+ [
689
+ _TOK[9],
690
+ _TOK[10],
691
+ _CHANNEL_END_TID,
692
+ _TOK[11],
693
+ _TOOL_START_TID,
694
+ _TOK[12],
695
+ _TOK[13],
696
+ _TOK[14],
697
+ _TOK[0],
698
+ _TOK[1],
699
+ ],
700
+ )
701
+ )
702
+ events.extend(engine.finish())
703
+
704
+ reasoning = _reasoning_text(events)
705
+
706
+ assert "w9" in reasoning
707
+ assert "w10" in reasoning
708
+ assert _has_event(events, EventType.REASONING_END)
709
+ assert _has_event(events, EventType.TOOL_CALL_START)
710
+
711
+ def test_channel_end_mid_batch_text_absent(self):
712
+ """<channel|> mid-batch with its text absent from delta_text."""
713
+ tok = _make_gemma4_tokenizer({_TOK[i]: f"word{i} " for i in range(15)})
714
+ engine = StreamingParserEngine(gemma4_config(), tok)
715
+
716
+ events: list = []
717
+ events.extend(
718
+ engine.feed(
719
+ "<|channel>thought\nword0 word1 word2 word3 word4 "
720
+ "word5 word6 word7 word8 ",
721
+ [
722
+ _CHANNEL_START_TID,
723
+ _TOK[0],
724
+ _TOK[1],
725
+ _TOK[2],
726
+ _TOK[3],
727
+ _TOK[4],
728
+ _TOK[5],
729
+ _TOK[6],
730
+ _TOK[7],
731
+ _TOK[8],
732
+ ],
733
+ )
734
+ )
735
+
736
+ events.extend(
737
+ engine.feed(
738
+ "word9 word10 word11 ",
739
+ [
740
+ _TOK[9],
741
+ _TOK[10],
742
+ _TOK[11],
743
+ _CHANNEL_END_TID,
744
+ _TOK[12],
745
+ _TOK[13],
746
+ _TOK[14],
747
+ _TOK[0],
748
+ _TOK[1],
749
+ _TOK[2],
750
+ ],
751
+ )
752
+ )
753
+
754
+ events.extend(
755
+ engine.feed(
756
+ "<channel|>word12 word13 word14 word0 word1 word2 ",
757
+ [_TOK[3], _TOK[4], _TOK[5]],
758
+ )
759
+ )
760
+
761
+ events.extend(engine.finish())
762
+
763
+ reasoning = _reasoning_text(events)
764
+ content = _content_text(events)
765
+
766
+ for w in ("word9", "word10", "word11"):
767
+ assert w in reasoning, f"{w!r} missing from reasoning"
768
+
769
+ for w in ("word12", "word13", "word14"):
770
+ assert w in content, f"{w!r} missing from content"
771
+
772
+ assert _has_event(events, EventType.REASONING_END)
773
+
774
+ def test_tool_end_mid_batch_text_absent_unified(self):
775
+ """<tool_call|> mid-batch with text absent."""
776
+ tok = _make_gemma4_tokenizer({_TOK[i]: f"w{i}" for i in range(15)})
777
+ engine = StreamingParserEngine(gemma4_config(), tok)
778
+
779
+ events: list = []
780
+ events.extend(
781
+ engine.feed(
782
+ _CHANNEL_START_TAG,
783
+ [_CHANNEL_START_TID],
784
+ )
785
+ )
786
+ events.extend(
787
+ engine.feed(
788
+ "thought\nNeed a tool.",
789
+ [_TOK[0], _TOK[1]],
790
+ )
791
+ )
792
+ events.extend(
793
+ engine.feed(
794
+ _TOOL_START_TAG,
795
+ [_TOOL_START_TID],
796
+ )
797
+ )
798
+ events.extend(
799
+ engine.feed(
800
+ "call:get_weather{city:",
801
+ [_TOK[2], _TOK[3], _TOK[4]],
802
+ )
803
+ )
804
+
805
+ events.extend(
806
+ engine.feed(
807
+ '<|"|>San Francisco<|"|>}',
808
+ [
809
+ _QUOTE_TID,
810
+ _TOK[5],
811
+ _TOK[6],
812
+ _QUOTE_TID,
813
+ _TOK[7],
814
+ _TOOL_END_TID,
815
+ _TOK[8],
816
+ _TOK[9],
817
+ _TOK[10],
818
+ _TOK[11],
819
+ ],
820
+ )
821
+ )
822
+
823
+ events.extend(
824
+ engine.feed(
825
+ "<tool_call|>w8w9w10w11w12",
826
+ [_TOK[12], _TOK[13]],
827
+ )
828
+ )
829
+
830
+ events.extend(engine.finish())
831
+
832
+ assert _has_event(events, EventType.TOOL_CALL_END)
833
+ assert "San Francisco" in _arg_text(events)
834
+
835
+ def test_large_batch_holdback_spans_two_batches(self):
836
+ """Holdback text spanning two batches with <channel|> in the second."""
837
+ tok = _make_gemma4_tokenizer({_TOK[i]: f"w{i} " for i in range(15)})
838
+ engine = StreamingParserEngine(gemma4_config(), tok)
839
+
840
+ events: list = []
841
+
842
+ events.extend(
843
+ engine.feed(
844
+ "<|channel>thought\nThe user asked about machine learning "
845
+ "and I need to think about the best approach to",
846
+ [
847
+ _CHANNEL_START_TID,
848
+ _TOK[0],
849
+ _TOK[1],
850
+ _TOK[2],
851
+ _TOK[3],
852
+ _TOK[4],
853
+ _TOK[5],
854
+ _TOK[6],
855
+ _TOK[7],
856
+ _TOK[8],
857
+ ],
858
+ )
859
+ )
860
+
861
+ events.extend(
862
+ engine.feed(
863
+ " explain this complex topic. Let me organize my thoughts.",
864
+ [
865
+ _TOK[9],
866
+ _TOK[10],
867
+ _TOK[11],
868
+ _TOK[12],
869
+ _TOK[13],
870
+ _TOK[14],
871
+ _CHANNEL_END_TID,
872
+ _TOK[0],
873
+ _TOK[1],
874
+ _TOK[2],
875
+ ],
876
+ )
877
+ )
878
+
879
+ events.extend(
880
+ engine.feed(
881
+ "<channel|>w0 w1 w2 Here is what I recommend: start with "
882
+ "the fundamentals and build up from there.",
883
+ [
884
+ _TOK[3],
885
+ _TOK[4],
886
+ _TOK[5],
887
+ _TOK[6],
888
+ _TOK[7],
889
+ _TOK[8],
890
+ _TOK[9],
891
+ _TOK[10],
892
+ _TOK[11],
893
+ _TOK[12],
894
+ ],
895
+ )
896
+ )
897
+
898
+ events.extend(engine.finish())
899
+
900
+ reasoning = _reasoning_text(events)
901
+ content = _content_text(events)
902
+
903
+ assert "organize my thoughts." in reasoning
904
+ assert "explain" in reasoning
905
+ assert "recommend" in content
906
+
907
+ assert _has_event(events, EventType.REASONING_START)
908
+ assert _has_event(events, EventType.REASONING_END)
909
+
910
+
911
class TestRebuildFromAnchorsLiteralLookalike:
495
- """When delta_text contains a literal mention of a special token's
496
- text before the real special token, _rebuild_from_anchors must
497
- anchor at the real occurrence, not the literal one."""
912
+ """Literal token text in prose must not be consumed as an anchor."""
913
914
@pytest.fixture
915
def tool_scanner(self):
@@ -513,8 +928,6 @@ class TestRebuildFromAnchorsLiteralLookalike:
928
)
929
930
def test_literal_before_real_anchor(self, tool_scanner):
516
- """Literal <tool_call> in prose followed by a real <tool_call>
517
- special token — the scanner must split at the real one."""
931
delta_text = 'Use <tool_call> like this: <tool_call>{"name":"f"}</tool_call>'
932
delta_token_ids = [1, 2, 3, 4, 5, TOOL_START_ID, 6, 7, TOOL_END_ID]
933
items = tool_scanner.scan(delta_text, delta_token_ids)
@@ -526,14 +939,11 @@ class TestRebuildFromAnchorsLiteralLookalike:
939
assert terminals[0].terminal == "TOOL_START"
940
assert terminals[1].terminal == "TOOL_END"
941
529
- # The literal mention must appear in a text chunk, not be
530
- # consumed by the TOOL_START anchor.
942
joined_text = "".join(text_parts)
943
assert "<tool_call>" in joined_text
944
assert '{"name":"f"}' in joined_text
945
946
def test_multiple_tool_calls_with_literal_between(self, tool_scanner):
536
- """Two real tool calls with a literal mention between them."""
947
delta_text = (
948
'<tool_call>{"name":"a"}</tool_call>'
949
" see <tool_call> syntax "
@@ -557,14 +967,11 @@ class TestRebuildFromAnchorsLiteralLookalike:
967
968
text_parts = [it.text for it in items if isinstance(it, TextChunk)]
969
joined_text = "".join(text_parts)
560
- # The literal mention between the two real calls must be in text
970
assert "<tool_call> syntax" in joined_text
971
972
973
class TestRebuildFromAnchorsCascadingDeferral:
565
- """When a middle anchor's text is absent from delta_text,
566
- only that anchor should be deferred — not subsequent ones
567
- with valid positions."""
974
+ """Missing middle anchor defers only itself, not subsequent ones."""
975
976
@pytest.fixture
977
def bare_scanner(self):
@@ -623,9 +1030,6 @@ class TestRebuildFromAnchorsCascadingDeferral:
1030
texts = [r for r in rebuilt if isinstance(r, TextChunk)]
1031
joined = "".join(t.text for t in texts)
1032
assert "text" in joined
626
- # "more" is deferred along with the missing terminal —
627
- # it will be resolved in the next scan when the terminal
628
- # text arrives.
1033
assert bare_scanner._deferred_post_text == "more"
1034
assert len(bare_scanner._deferred_terminals) == 1
1035
assert bare_scanner._deferred_terminals[0].terminal == "THINK_END"
tests/parser/engine/trace_builder.py
+113
-2
@@ -29,6 +29,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
29
ChatCompletionToolsParam,
30
)
31
from vllm.parser.engine.registered_adapters import (
32
+ Gemma4Parser,
33
Qwen3Parser,
34
)
35
@@ -48,6 +49,7 @@ class Scenario:
49
reasoning: str | None = None
50
content: str | None = None
51
tool_calls: list[ToolCallSpec] | None = None
52
+ after_tool_response: bool = False
53
54
55
# ── Scenarios ────────────────────────────────────────────────────────
@@ -132,6 +134,12 @@ SCENARIOS: list[Scenario] = [
134
reasoning="",
135
content="The epoch timestamp is 1779111346.",
136
),
137
+ Scenario(
138
+ id="tool-after-tool-response",
139
+ description="Tool call immediately after tool response (agentic flow)",
140
+ tool_calls=[_READ_TOOL],
141
+ after_tool_response=True,
142
+ ),
143
]
144
145
@@ -250,7 +258,13 @@ def _validate_sample(sample: Sample, parser_cls: type, **kwargs) -> None:
258
"""Replay sample through the real parser and assert correctness."""
259
tokenizer = MockTokenizer(vocab=dict(sample.vocab), tokens=sample.tokens)
260
parser = parser_cls(tokenizer, sample.tools, **kwargs)
253
- deltas = replay_streaming(parser, sample.tokens, chunk_size=1, tools=sample.tools)
261
+ deltas = replay_streaming(
262
+ parser,
263
+ sample.tokens,
264
+ chunk_size=1,
265
+ tools=sample.tools,
266
+ prompt_token_ids=sample.prompt_token_ids,
267
+ )
268
output = collect_output(deltas)
269
assert_parse_output(output, sample)
270
@@ -273,6 +287,7 @@ def _make_sample(
287
expected_tool_calls: list[dict] | None,
288
tools: list[dict] | None,
289
chat_template_kwargs: dict | None = None,
290
+ prompt_token_ids: list[int] | None = None,
291
) -> Sample:
292
tokens = _tokenize(segments, vocab)
293
return Sample(
@@ -286,10 +301,11 @@ def _make_sample(
301
expected_tool_calls=expected_tool_calls,
302
tools=_validate_tools(tools),
303
chat_template_kwargs=chat_template_kwargs,
304
+ prompt_token_ids=prompt_token_ids,
305
)
306
307
292
-# ── Qwen3 / NemotronV3 (XML tool format, starts in REASONING) ───────
308
+# ── Qwen3 (XML tool format, starts in REASONING) ────────────────────
309
310
_QWEN3_VOCAB: dict[str, int] = {
311
"<think>": 50,
@@ -376,10 +392,105 @@ def _build_qwen3(
392
return sample
393
394
395
+# ── Gemma4 (channel reasoning, custom arg format) ────────────────────
396
+
397
+_GEMMA4_VOCAB: dict[str, int] = {
398
+ "<|channel>": 50,
399
+ "<channel|>": 51,
400
+ "<|tool_call>": 48,
401
+ "<tool_call|>": 49,
402
+ '<|"|>': 52,
403
+ "<|turn>": 53,
404
+ "<|tool_response>": 54,
405
+}
406
+_GEMMA4_THOUGHT_PREFIX = "thought\n"
407
+_GEMMA4_QUOTE = '<|"|>'
408
+
409
+
410
+def _gemma4_value_segments(value: Any) -> list[tuple[str, bool]]:
411
+ """Render a value in Gemma4 arg format as segments."""
412
+ if isinstance(value, str):
413
+ return [(_GEMMA4_QUOTE, True), (value, False), (_GEMMA4_QUOTE, True)]
414
+ if isinstance(value, bool):
415
+ return [("true" if value else "false", False)]
416
+ if isinstance(value, (int, float)):
417
+ return [(str(value), False)]
418
+ if isinstance(value, dict):
419
+ segs: list[tuple[str, bool]] = [("{", False)]
420
+ for i, (k, v) in enumerate(value.items()):
421
+ if i > 0:
422
+ segs.append((",", False))
423
+ segs.append((f"{k}:", False))
424
+ segs.extend(_gemma4_value_segments(v))
425
+ segs.append(("}", False))
426
+ return segs
427
+ if isinstance(value, list):
428
+ segs = [("[", False)]
429
+ for i, item in enumerate(value):
430
+ if i > 0:
431
+ segs.append((",", False))
432
+ segs.extend(_gemma4_value_segments(item))
433
+ segs.append(("]", False))
434
+ return segs
435
+ return [(json.dumps(value, ensure_ascii=False), False)]
436
+
437
+
438
+def _gemma4_tool_segments(tc: ToolCallSpec) -> list[tuple[str, bool]]:
439
+ segs: list[tuple[str, bool]] = [
440
+ ("<|tool_call>", True),
441
+ (f"call:{tc.name}", False),
442
+ ("{", False),
443
+ ]
444
+ for i, (key, value) in enumerate(tc.arguments.items()):
445
+ if i > 0:
446
+ segs.append((",", False))
447
+ segs.append((f"{key}:", False))
448
+ segs.extend(_gemma4_value_segments(value))
449
+ segs.append(("}", False))
450
+ segs.append(("<tool_call|>", True))
451
+ return segs
452
+
453
+
454
+def _gemma4_segments(scenario: Scenario) -> list[tuple[str, bool]]:
455
+ segs: list[tuple[str, bool]] = []
456
+ if scenario.reasoning is not None:
457
+ segs.append(("<|channel>", True))
458
+ segs.append((_GEMMA4_THOUGHT_PREFIX, False))
459
+ segs.append((scenario.reasoning, False))
460
+ segs.append(("<channel|>", True))
461
+ if scenario.content is not None:
462
+ segs.append((scenario.content, False))
463
+ if scenario.tool_calls:
464
+ for tc in scenario.tool_calls:
465
+ segs.extend(_gemma4_tool_segments(tc))
466
+ return segs
467
+
468
+
469
+def _build_gemma4(scenario: Scenario, validate: bool = True) -> Sample:
470
+ prompt_token_ids = None
471
+ if scenario.after_tool_response:
472
+ prompt_token_ids = [_GEMMA4_VOCAB["<|tool_response>"]]
473
+ sample = _make_sample(
474
+ sample_id=f"gemma4-{scenario.id}",
475
+ description=scenario.description,
476
+ vocab=_GEMMA4_VOCAB,
477
+ segments=_gemma4_segments(scenario),
478
+ expected_reasoning=scenario.reasoning,
479
+ expected_content=_qwen3_expected_content(scenario),
480
+ expected_tool_calls=_expected_tc(scenario),
481
+ tools=_expected_tools(scenario),
482
+ prompt_token_ids=prompt_token_ids,
483
+ )
484
+ if validate:
485
+ _validate_sample(sample, Gemma4Parser)
486
+ return sample
487
+
488
+
489
# ── Registry and public API ──────────────────────────────────────────
490
491
_BUILDERS: dict[str, Any] = {
492
"qwen3": _build_qwen3,
493
+ "gemma4": _build_gemma4,
494
}
495
496
tests/reasoning/test_gemma4_reasoning_parser.py
+4
-4
@@ -83,15 +83,15 @@ CHANNEL_NO_END = {
83
EMPTY = {
84
"output": "",
85
"reasoning": None,
86
- "content": "",
87
- "is_reasoning_end": False,
86
+ "content": None,
87
+ "is_reasoning_end": True,
88
}
89
NEW_LINE_NONSTREAMING = {
90
"output": (
91
"Before\n<|channel>This is a reasoning section<channel|>\nThis is the rest"
92
),
93
"reasoning": "This is a reasoning section",
94
- "content": "\nThis is the rest",
94
+ "content": "Before\n\nThis is the rest",
95
"is_reasoning_end": True,
96
}
97
NEW_LINE_STREAMING = {
@@ -111,7 +111,7 @@ THOUGHT_PREFIX = {
111
}
112
THOUGHT_PREFIX_ONLY = {
113
"output": "<|channel>thought\n<channel|>",
114
- "reasoning": "",
114
+ "reasoning": None,
115
"content": None,
116
"is_reasoning_end": True,
117
}
tests/tool_parsers/test_gemma4_tool_parser.py
+138
-51
@@ -8,31 +8,105 @@ from unittest.mock import MagicMock
8
import pytest
9
10
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
11
-from vllm.tool_parsers.gemma4_tool_parser import (
11
+from vllm.parser.gemma4 import (
12
TOOL_CALL_END,
13
TOOL_CALL_START,
14
- Gemma4ToolParser,
14
_parse_gemma4_args,
15
_parse_gemma4_array,
16
)
17
+from vllm.tool_parsers.gemma4_engine_tool_parser import Gemma4EngineToolParser
18
19
# ---------------------------------------------------------------------------
20
# Fixtures
21
# ---------------------------------------------------------------------------
22
23
24
+TOOL_CALL_START_ID = 48
25
+TOOL_CALL_END_ID = 49
26
+CHANNEL_START = "<|channel>"
27
+CHANNEL_END = "<channel|>"
28
+CHANNEL_START_ID = 50
29
+CHANNEL_END_ID = 51
30
+
31
+
32
+def _make_tool(name, properties):
33
+ from vllm.entrypoints.openai.chat_completion.protocol import (
34
+ ChatCompletionToolsParam,
35
+ )
36
+
37
+ return ChatCompletionToolsParam(
38
+ type="function",
39
+ function={
40
+ "name": name,
41
+ "parameters": {"type": "object", "properties": properties},
42
+ },
43
+ )
44
+
45
+
46
+_TOOLS = [
47
+ _make_tool(
48
+ "set_status",
49
+ {
50
+ "is_active": {"type": "boolean"},
51
+ "count": {"type": "integer"},
52
+ "score": {"type": "number"},
53
+ },
54
+ ),
55
+ _make_tool(
56
+ "set_config",
57
+ {
58
+ "count": {"type": "integer"},
59
+ "active": {"type": "boolean"},
60
+ },
61
+ ),
62
+ _make_tool(
63
+ "search",
64
+ {
65
+ "input": {
66
+ "type": "object",
67
+ "properties": {"all": {"type": "boolean"}},
68
+ },
69
+ },
70
+ ),
71
+ _make_tool(
72
+ "set",
73
+ {
74
+ "flag": {"type": "boolean"},
75
+ "count": {"type": "integer"},
76
+ },
77
+ ),
78
+ _make_tool(
79
+ "Edit",
80
+ {
81
+ "file_path": {"type": "string"},
82
+ "old_string": {"type": "string"},
83
+ "new_string": {"type": "string"},
84
+ "replace_all": {"type": "boolean"},
85
+ },
86
+ ),
87
+]
88
+
89
+
90
@pytest.fixture
91
def mock_tokenizer():
92
+ vocab = {
93
+ TOOL_CALL_START: TOOL_CALL_START_ID,
94
+ TOOL_CALL_END: TOOL_CALL_END_ID,
95
+ CHANNEL_START: CHANNEL_START_ID,
96
+ CHANNEL_END: CHANNEL_END_ID,
97
+ }
98
+ decode_map = {v: k for k, v in vocab.items()}
99
+
100
tokenizer = MagicMock()
101
tokenizer.encode.return_value = [1, 2, 3]
28
- # Include the tool call start token in the vocab for the parser
29
- tokenizer.get_vocab.return_value = {TOOL_CALL_START: 48, TOOL_CALL_END: 49}
102
+ tokenizer.get_vocab.return_value = vocab
103
+ tokenizer.decode.side_effect = lambda ids: decode_map.get(ids[0], f"tok{ids[0]}")
104
return tokenizer
105
106
107
@pytest.fixture
108
def parser(mock_tokenizer):
35
- return Gemma4ToolParser(mock_tokenizer)
109
+ return Gemma4EngineToolParser(mock_tokenizer, tools=_TOOLS)
110
111
112
@pytest.fixture
@@ -49,6 +123,9 @@ def mock_request():
123
124
125
class TestParseGemma4Args:
126
+ """Values are returned as strings; type coercion to proper JSON types
127
+ happens at the engine layer."""
128
+
129
def test_empty_string(self):
130
assert _parse_gemma4_args("") == {}
131
@@ -71,27 +148,23 @@ class TestParseGemma4Args:
148
149
def test_integer_value(self):
150
result = _parse_gemma4_args("count:42")
74
- assert result == {"count": 42}
151
+ assert result == {"count": "42"}
152
153
def test_float_value(self):
154
result = _parse_gemma4_args("score:3.14")
78
- assert result == {"score": 3.14}
155
+ assert result == {"score": "3.14"}
156
157
def test_boolean_true(self):
158
result = _parse_gemma4_args("flag:true")
82
- assert result == {"flag": True}
159
+ assert result == {"flag": "true"}
160
161
def test_boolean_false(self):
162
result = _parse_gemma4_args("flag:false")
86
- assert result == {"flag": False}
163
+ assert result == {"flag": "false"}
164
165
def test_null_value(self):
89
- # Bare `null` must parse as None (Python), not the string "null".
90
- # Without this, tool_choice=auto would emit `{"param": "null"}`
91
- # instead of `{"param": null}` for nullable tool parameters.
166
result = _parse_gemma4_args("param:null")
93
- assert result == {"param": None}
94
- assert json.dumps(result) == '{"param": null}'
167
+ assert result == {"param": "null"}
168
169
def test_mixed_types(self):
170
result = _parse_gemma4_args(
@@ -99,9 +172,9 @@ class TestParseGemma4Args:
172
)
173
assert result == {
174
"name": "test",
102
- "count": 42,
103
- "active": True,
104
- "score": 3.14,
175
+ "count": "42",
176
+ "active": "true",
177
+ "score": "3.14",
178
}
179
180
def test_nested_object(self):
@@ -112,6 +185,17 @@ class TestParseGemma4Args:
185
result = _parse_gemma4_args('items:[<|"|>a<|"|>,<|"|>b<|"|>]')
186
assert result == {"items": ["a", "b"]}
187
188
+ def test_delimited_keys_stripped(self):
189
+ """Keys wrapped in <|"|> delimiters are stripped."""
190
+ result = _parse_gemma4_args('<|"|>location<|"|>:<|"|>Paris<|"|>')
191
+ assert result == {"location": "Paris"}
192
+
193
+ result = _parse_gemma4_args('outer:{<|"|>inner<|"|>:<|"|>val<|"|>}')
194
+ assert result == {"outer": {"inner": "val"}}
195
+
196
+ result = _parse_gemma4_args('<|"|>name<|"|>:<|"|>Alice<|"|>,count:42')
197
+ assert result == {"name": "Alice", "count": "42"}
198
+
199
def test_unterminated_string(self):
200
"""Unterminated strings should take everything after the delimiter."""
201
result = _parse_gemma4_args('key:<|"|>unterminated')
@@ -153,7 +237,7 @@ class TestParseGemma4Args:
237
238
# Non-partial mode parses trailing dot normally
239
result = _parse_gemma4_args("left:108.,right:22.8", partial=False)
156
- assert result == {"left": 108.0, "right": 22.8}
240
+ assert result == {"left": "108.", "right": "22.8"}
241
242
@pytest.mark.timeout(5)
243
def test_malformed_partial_array(self):
@@ -172,7 +256,7 @@ class TestParseGemma4Array:
256
257
def test_bare_values(self):
258
result = _parse_gemma4_array("42,true,3.14")
175
- assert result == [42, True, 3.14]
259
+ assert result == ["42", "true", "3.14"]
260
261
@pytest.mark.timeout(5)
262
def test_string_element_with_closing_bracket(self):
@@ -182,7 +266,7 @@ class TestParseGemma4Array:
266
@pytest.mark.timeout(5)
267
def test_stray_closing_bracket(self):
268
result = _parse_gemma4_array("42,]trailing")
185
- assert result == [42]
269
+ assert result == ["42"]
270
271
def test_trailing_dot_float_partial_withheld(self):
272
"""Array elements with trailing dot withheld in partial mode."""
@@ -191,7 +275,7 @@ class TestParseGemma4Array:
275
276
# Stable elements before trailing-dot element are kept
277
result = _parse_gemma4_array("42,108.,3", partial=True)
194
- assert result == [42]
278
+ assert result == ["42"]
279
280
281
# ---------------------------------------------------------------------------
@@ -297,9 +381,11 @@ class TestExtractToolCalls:
381
model_output = '<|tool_call>call:get_weather{location:<|"|>London'
382
result = parser.extract_tool_calls(model_output, mock_request)
383
300
- # Incomplete — no <tool_call|> end marker, regex won't match
301
- assert result.tools_called is False
302
- assert result.content == model_output
384
+ assert result.tools_called is True
385
+ assert len(result.tool_calls) == 1
386
+ assert result.tool_calls[0].function.name == "get_weather"
387
+ args = json.loads(result.tool_calls[0].function.arguments)
388
+ assert args == {"location": "London"}
389
390
def test_hyphenated_function_name(self, parser, mock_request):
391
"""Ensure function names with hyphens are parsed correctly."""
@@ -345,8 +431,15 @@ class TestStreamingExtraction:
431
verifying that the accumulated argument deltas form valid JSON.
432
"""
433
434
+ _SPECIAL_TOKEN_IDS = {
435
+ TOOL_CALL_START: TOOL_CALL_START_ID,
436
+ TOOL_CALL_END: TOOL_CALL_END_ID,
437
+ CHANNEL_START: CHANNEL_START_ID,
438
+ CHANNEL_END: CHANNEL_END_ID,
439
+ }
440
+
441
def _simulate_streaming(
349
- self, parser: Gemma4ToolParser, mock_request: Any, chunks: list[str]
442
+ self, parser: Any, mock_request: Any, chunks: list[str]
443
) -> list[tuple[Any, str]]:
444
"""Feed chunks through the streaming parser and collect results.
445
@@ -358,14 +451,17 @@ class TestStreamingExtraction:
451
452
for chunk in chunks:
453
current_text = previous_text + chunk
361
- # Use token ID 48 for tool_call start, 49 for end, 0 otherwise
362
- delta_token_ids: list[int] = []
363
- if TOOL_CALL_START in chunk:
364
- delta_token_ids.append(48)
365
- elif TOOL_CALL_END in chunk:
366
- delta_token_ids.append(49)
367
- else:
368
- delta_token_ids.append(0)
454
+ found: list[tuple[int, int]] = []
455
+ for token, tid in self._SPECIAL_TOKEN_IDS.items():
456
+ pos = 0
457
+ while True:
458
+ idx = chunk.find(token, pos)
459
+ if idx < 0:
460
+ break
461
+ found.append((idx, tid))
462
+ pos = idx + len(token)
463
+ found.sort()
464
+ delta_token_ids: list[int] = [tid for _, tid in found] if found else [0]
465
466
current_token_ids = previous_token_ids + delta_token_ids
467
@@ -551,10 +647,10 @@ class TestStreamingExtraction:
647
648
results = self._simulate_streaming(parser, mock_request, chunks)
649
args_text = self._collect_arguments(results)
554
- if args_text:
555
- parsed_args = json.loads(args_text)
556
- assert parsed_args["count"] == 42
557
- assert parsed_args["active"] is True
650
+ assert args_text is not None
651
+ parsed_args = json.loads(args_text)
652
+ assert parsed_args["count"] == 42
653
+ assert parsed_args["active"] is True
654
655
def test_streaming_boolean_split_across_chunks(self, parser, mock_request):
656
"""Boolean value split across token boundaries must not corrupt JSON."""
@@ -643,23 +739,15 @@ class TestStreamingExtraction:
739
)
740
741
def test_streaming_does_not_duplicate_plain_text_after_tool_call(
646
- self, parser, mock_request, monkeypatch
742
+ self, parser, mock_request
743
):
648
- """Buffered plain text after a tool call must not corrupt current_text."""
649
- captured_current_texts: list[str] = []
650
- original_extract_streaming = parser._extract_streaming
651
-
652
- def wrapped_extract_streaming(previous_text, current_text, delta_text):
653
- captured_current_texts.append(current_text)
654
- return original_extract_streaming(previous_text, current_text, delta_text)
655
-
656
- monkeypatch.setattr(parser, "_extract_streaming", wrapped_extract_streaming)
657
-
744
+ """Buffered plain text after a tool call must not corrupt content."""
745
chunks = [
746
"<|tool_call>",
747
"call:get_weather{",
748
'location:<|"|>Paris<|"|>}',
662
- "<tool_call|><",
749
+ "<tool_call|>",
750
+ "<",
751
"div>",
752
]
753
@@ -668,8 +756,7 @@ class TestStreamingExtraction:
756
delta.content for delta, _ in results if delta is not None and delta.content
757
]
758
assert "".join(content_parts) == "<div>"
671
- assert captured_current_texts[-1].endswith("<tool_call|><div>")
672
- assert not captured_current_texts[-1].endswith("<tool_call|><<div>")
759
+ assert "<<div>" not in "".join(content_parts)
760
761
def test_streaming_html_argument_does_not_duplicate_tag_prefixes(
762
self, parser, mock_request
tests/tool_use/test_gemma4_responses_adjust_request.py
+13
-6
@@ -30,7 +30,9 @@ from openai.types.responses.tool_param import FunctionToolParam
30
31
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
32
from vllm.tool_parsers.abstract_tool_parser import ToolParser
33
-from vllm.tool_parsers.gemma4_tool_parser import Gemma4ToolParser
33
+from vllm.tool_parsers.gemma4_engine_tool_parser import (
34
+ Gemma4EngineToolParser as Gemma4ToolParser,
35
+)
36
37
38
def _get_weather_tool() -> FunctionToolParam:
@@ -59,10 +61,16 @@ def _build_responses_request(*, tool_choice: str) -> ResponsesRequest:
61
62
63
class _StubTokenizer:
62
- """Minimal tokenizer stub to satisfy ``Gemma4ToolParser.__init__``."""
64
+ """Minimal tokenizer stub to satisfy ``Gemma4EngineToolParser.__init__``."""
65
66
def get_vocab(self) -> dict[str, int]:
65
- return {"<|tool_call>": 256_000, "<tool_call|>": 256_001, '<|"|>': 52}
67
+ return {
68
+ "<|tool_call>": 256_000,
69
+ "<tool_call|>": 256_001,
70
+ '<|"|>': 52,
71
+ "<|channel>": 256_002,
72
+ "<channel|>": 256_003,
73
+ }
74
75
76
def test_gemma4_adjust_request_sets_skip_special_tokens_on_responses() -> None:
@@ -74,15 +82,14 @@ def test_gemma4_adjust_request_sets_skip_special_tokens_on_responses() -> None:
82
path, causing raw ``call:fn{...}`` text to leak via
83
``response.output_text.delta``.
84
"""
77
- parser = Gemma4ToolParser.__new__(Gemma4ToolParser)
78
- parser.model_tokenizer = _StubTokenizer()
85
+ parser = Gemma4ToolParser(_StubTokenizer())
86
87
request = _build_responses_request(tool_choice="auto")
88
assert request.skip_special_tokens is True, (
89
"Precondition: ResponsesRequest.skip_special_tokens default is True"
90
)
91
85
- Gemma4ToolParser.adjust_request(parser, request)
92
+ parser.adjust_request(request)
93
94
assert request.skip_special_tokens is False
95
vllm/parser/engine/parser_engine.py
+20
-5
@@ -182,6 +182,21 @@ class ParserEngine(Parser):
182
request.skip_special_tokens = False
183
return request
184
185
+ def _preprocess_feed(
186
+ self,
187
+ delta_text: str,
188
+ delta_token_ids: Sequence[int],
189
+ ) -> tuple[str, Sequence[int]]:
190
+ return delta_text, delta_token_ids
191
+
192
+ def _feed(
193
+ self,
194
+ delta_text: str,
195
+ delta_token_ids: Sequence[int],
196
+ ) -> list[SemanticEvent]:
197
+ delta_text, delta_token_ids = self._preprocess_feed(delta_text, delta_token_ids)
198
+ return self._engine.feed(delta_text, delta_token_ids)
199
+
200
# ── Schema-aware type correction ─────────────────────────────────
201
202
@staticmethod
@@ -340,7 +355,7 @@ class ParserEngine(Parser):
355
finished: bool,
356
) -> DeltaMessage | None:
357
self._check_skip_tool_parsing(request)
343
- events = self._engine.feed(delta_text, delta_token_ids)
358
+ events = self._feed(delta_text, delta_token_ids)
359
if finished:
360
events.extend(self._engine.finish())
361
result = self._events_to_delta(events, finished=finished)
@@ -384,7 +399,7 @@ class ParserEngine(Parser):
399
request: ChatCompletionRequest | ResponsesRequest,
400
) -> tuple[str | None, str | None]:
401
self._reset()
387
- events = self._engine.feed(model_output, [])
402
+ events = self._feed(model_output, [])
403
events.extend(self._engine.finish())
404
405
reasoning_parts: list[str] = []
@@ -417,7 +432,7 @@ class ParserEngine(Parser):
432
delta_token_ids: Sequence[int],
433
) -> DeltaMessage | None:
434
self.initialize_streaming()
420
- events = self._engine.feed(delta_text, delta_token_ids)
435
+ events = self._feed(delta_text, delta_token_ids)
436
return self._strip_trailing_reasoning(self._events_to_delta(events))
437
438
# ── Non-streaming: extract_tool_calls ─────────────────────────────
@@ -477,7 +492,7 @@ class ParserEngine(Parser):
492
) -> DeltaMessage | None:
493
self.initialize_streaming()
494
self._check_skip_tool_parsing(request)
480
- events = self._engine.feed(delta_text, delta_token_ids)
495
+ events = self._feed(delta_text, delta_token_ids)
496
return self._strip_trailing_reasoning(self._events_to_delta(events))
497
498
# ── Reasoning state queries ───────────────────────────────────────
@@ -537,7 +552,7 @@ class ParserEngine(Parser):
552
state that ``_build_extracted_result`` reads.
553
"""
554
self._reset(initial_state=initial_state)
540
- events = self._engine.feed(text, token_ids)
555
+ events = self._feed(text, token_ids)
556
events.extend(self._engine.finish())
557
558
delta = self._events_to_delta(events)
vllm/parser/engine/registered_adapters.py
+6
@@ -8,8 +8,14 @@ names so that :class:`ReasoningParserManager` and
8
"""
9
10
from vllm.parser.engine.adapters import make_adapters
11
+from vllm.parser.gemma4 import Gemma4Parser
12
from vllm.parser.qwen3 import Qwen3Parser
13
14
+(
15
+ Gemma4ParserReasoningAdapter,
16
+ Gemma4ParserToolAdapter,
17
+) = make_adapters(Gemma4Parser)
18
+
19
(
20
Qwen3ParserReasoningAdapter,
21
Qwen3ParserToolAdapter,
vllm/parser/gemma4.py
new
+557
@@ -0,0 +1,557 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Gemma4 parser.
4
+
5
+Handles channel-based reasoning plus custom tool call format in a single
6
+state machine::
7
+
8
+ <|channel>thought
9
+ ...reasoning...<channel|>
10
+ <|tool_call>call:func_name{key:<|"|>value<|"|>,num:42}<tool_call|>
11
+"""
12
+
13
+from __future__ import annotations
14
+
15
+import functools
16
+import json
17
+from collections.abc import Sequence
18
+from typing import TYPE_CHECKING
19
+
20
+from vllm.entrypoints.openai.engine.protocol import DeltaMessage
21
+from vllm.logger import init_logger
22
+from vllm.parser.engine.events import EventType, SemanticEvent
23
+from vllm.parser.engine.parser_engine import ParserEngine
24
+from vllm.parser.engine.parser_engine_config import (
25
+ ParserEngineConfig,
26
+ ParserState,
27
+ Transition,
28
+)
29
+
30
+if TYPE_CHECKING:
31
+ from vllm.entrypoints.openai.chat_completion.protocol import (
32
+ ChatCompletionRequest,
33
+ )
34
+ from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
35
+ from vllm.tokenizers import TokenizerLike
36
+ from vllm.tool_parsers.abstract_tool_parser import Tool
37
+
38
+# Tokens the model generates that must not leak into response content.
39
+_GEMMA4_MODEL_DROP_TOKENS: set[str] = {
40
+ # Turn boundaries
41
+ "<|turn>",
42
+ "<turn|>",
43
+ # Channel / reasoning
44
+ "<|channel>",
45
+ "<channel|>",
46
+ # Tool protocol tokens
47
+ "<|tool>",
48
+ "<tool|>",
49
+ "<|tool_call>",
50
+ "<tool_call|>",
51
+ "<|tool_response>",
52
+ "<tool_response|>",
53
+ '<|"|>',
54
+ # Thinking
55
+ "<|think|>",
56
+ # Multi-modal (defensive — not expected during text completion)
57
+ "<|image>",
58
+ "<|image|>",
59
+ "<image|>",
60
+ "<|audio>",
61
+ "<|audio|>",
62
+ "<audio|>",
63
+ "<|video|>",
64
+}
65
+
66
+CHANNEL_START = "<|channel>"
67
+CHANNEL_END = "<channel|>"
68
+TOOL_CALL_START = "<|tool_call>"
69
+TOOL_CALL_END = "<tool_call|>"
70
+STRING_DELIM = '<|"|>'
71
+_DELIM_LEN = len(STRING_DELIM)
72
+
73
+logger = init_logger(__name__)
74
+
75
+
76
+# ---------------------------------------------------------------------------
77
+# Gemma4 argument parser
78
+# ---------------------------------------------------------------------------
79
+
80
+_PARTIAL_DELIM_SUFFIXES = tuple(
81
+ STRING_DELIM[:k] for k in range(len(STRING_DELIM), 0, -1)
82
+)
83
+
84
+
85
+def _strip_partial_delim(value: str) -> str:
86
+ """Strip a trailing partial ``STRING_DELIM`` prefix from *value*.
87
+
88
+ Prevents partial delimiters from leaking into the streamed JSON diff.
89
+ """
90
+ for suffix in _PARTIAL_DELIM_SUFFIXES:
91
+ if value.endswith(suffix):
92
+ return value[: -len(suffix)]
93
+ return value
94
+
95
+
96
+def _parse_gemma4_args(args_str: str, *, partial: bool = False) -> dict:
97
+ """Parse Gemma4's custom key:value format into a Python dict.
98
+
99
+ Format examples::
100
+
101
+ location:<|"|>Tokyo<|"|>
102
+ location:<|"|>San Francisco<|"|>,unit:<|"|>celsius<|"|>
103
+ count:42,flag:true
104
+ nested:{inner_key:<|"|>val<|"|>}
105
+ items:[<|"|>a<|"|>,<|"|>b<|"|>]
106
+
107
+ Args:
108
+ args_str: The raw Gemma4 argument string.
109
+ partial: When True (streaming), bare values at end of string are
110
+ omitted because they may be incomplete and type-unstable
111
+ (e.g. partial boolean parsed as bare string).
112
+
113
+ Returns a dict ready for ``json.dumps()``.
114
+ """
115
+ if not args_str or not args_str.strip():
116
+ return {}
117
+
118
+ result: dict = {}
119
+ i = 0
120
+ n = len(args_str)
121
+
122
+ while i < n:
123
+ while i < n and args_str[i] in (" ", ",", "\n", "\t"):
124
+ i += 1
125
+ if i >= n:
126
+ break
127
+
128
+ key_start = i
129
+ while i < n and args_str[i] != ":":
130
+ i += 1
131
+ if i >= n:
132
+ break
133
+ key = args_str[key_start:i].strip()
134
+ if key.startswith(STRING_DELIM) and key.endswith(STRING_DELIM):
135
+ key = key[_DELIM_LEN:-_DELIM_LEN]
136
+ i += 1
137
+
138
+ if i >= n:
139
+ if not partial:
140
+ result[key] = ""
141
+ break
142
+
143
+ while i < n and args_str[i] in (" ", "\n", "\t"):
144
+ i += 1
145
+ if i >= n:
146
+ if not partial:
147
+ result[key] = ""
148
+ break
149
+
150
+ if args_str[i : i + _DELIM_LEN] == STRING_DELIM:
151
+ i += _DELIM_LEN
152
+ val_start = i
153
+ end_pos = args_str.find(STRING_DELIM, i)
154
+ if end_pos == -1:
155
+ # Unterminated string — take rest, strip partial delimiter.
156
+ value = args_str[val_start:]
157
+ if partial:
158
+ value = _strip_partial_delim(value)
159
+ result[key] = value
160
+ break
161
+ result[key] = args_str[val_start:end_pos]
162
+ i = end_pos + _DELIM_LEN
163
+
164
+ elif args_str[i] == "{":
165
+ depth = 1
166
+ obj_start = i + 1
167
+ i += 1
168
+ while i < n and depth > 0:
169
+ if args_str[i : i + _DELIM_LEN] == STRING_DELIM:
170
+ # Skip over string contents to avoid counting { inside strings
171
+ i += _DELIM_LEN
172
+ next_delim = args_str.find(STRING_DELIM, i)
173
+ i = n if next_delim == -1 else next_delim + _DELIM_LEN
174
+ continue
175
+ if args_str[i] == "{":
176
+ depth += 1
177
+ elif args_str[i] == "}":
178
+ depth -= 1
179
+ i += 1
180
+ if depth > 0:
181
+ # Incomplete nested object — use i (not i-1) to avoid
182
+ # dropping the last char, and recurse as partial.
183
+ result[key] = _parse_gemma4_args(args_str[obj_start:i], partial=True)
184
+ else:
185
+ result[key] = _parse_gemma4_args(args_str[obj_start : i - 1])
186
+
187
+ elif args_str[i] == "[":
188
+ depth = 1
189
+ arr_start = i + 1
190
+ i += 1
191
+ while i < n and depth > 0:
192
+ if args_str[i : i + _DELIM_LEN] == STRING_DELIM:
193
+ i += _DELIM_LEN
194
+ next_delim = args_str.find(STRING_DELIM, i)
195
+ i = n if next_delim == -1 else next_delim + _DELIM_LEN
196
+ continue
197
+ if args_str[i] == "[":
198
+ depth += 1
199
+ elif args_str[i] == "]":
200
+ depth -= 1
201
+ i += 1
202
+ if depth > 0:
203
+ result[key] = _parse_gemma4_array(args_str[arr_start:i], partial=True)
204
+ else:
205
+ result[key] = _parse_gemma4_array(args_str[arr_start : i - 1])
206
+
207
+ else:
208
+ val_start = i
209
+ while i < n and args_str[i] not in (",", "}", "]"):
210
+ i += 1
211
+ if partial and i >= n:
212
+ # Value may be incomplete (e.g. partial boolean) —
213
+ # withhold to avoid type instability during streaming.
214
+ break
215
+ if i == val_start:
216
+ logger.warning(
217
+ "Gemma4 args parser made no progress at position %d; "
218
+ "aborting on malformed input.",
219
+ i,
220
+ )
221
+ break
222
+ raw_val = args_str[val_start:i].strip()
223
+ if partial and raw_val.endswith("."):
224
+ # Digits may still arrive (e.g. "108." -> "108.2");
225
+ # withhold to avoid corrupting the streaming diff.
226
+ break
227
+ result[key] = raw_val
228
+
229
+ return result
230
+
231
+
232
+def _parse_gemma4_array(arr_str: str, *, partial: bool = False) -> list:
233
+ items: list = []
234
+ i = 0
235
+ n = len(arr_str)
236
+
237
+ while i < n:
238
+ while i < n and arr_str[i] in (" ", ",", "\n", "\t"):
239
+ i += 1
240
+ if i >= n:
241
+ break
242
+
243
+ if arr_str[i : i + _DELIM_LEN] == STRING_DELIM:
244
+ i += _DELIM_LEN
245
+ end_pos = arr_str.find(STRING_DELIM, i)
246
+ if end_pos == -1:
247
+ items.append(arr_str[i:])
248
+ break
249
+ items.append(arr_str[i:end_pos])
250
+ i = end_pos + _DELIM_LEN
251
+
252
+ elif arr_str[i] == "{":
253
+ depth = 1
254
+ obj_start = i + 1
255
+ i += 1
256
+ while i < n and depth > 0:
257
+ if arr_str[i : i + _DELIM_LEN] == STRING_DELIM:
258
+ i += _DELIM_LEN
259
+ nd = arr_str.find(STRING_DELIM, i)
260
+ i = nd + _DELIM_LEN if nd != -1 else n
261
+ continue
262
+ if arr_str[i] == "{":
263
+ depth += 1
264
+ elif arr_str[i] == "}":
265
+ depth -= 1
266
+ i += 1
267
+ if depth > 0:
268
+ items.append(_parse_gemma4_args(arr_str[obj_start:i], partial=True))
269
+ else:
270
+ items.append(_parse_gemma4_args(arr_str[obj_start : i - 1]))
271
+
272
+ elif arr_str[i] == "[":
273
+ depth = 1
274
+ sub_start = i + 1
275
+ i += 1
276
+ while i < n and depth > 0:
277
+ if arr_str[i : i + _DELIM_LEN] == STRING_DELIM:
278
+ i += _DELIM_LEN
279
+ nd = arr_str.find(STRING_DELIM, i)
280
+ i = nd + _DELIM_LEN if nd != -1 else n
281
+ continue
282
+ if arr_str[i] == "[":
283
+ depth += 1
284
+ elif arr_str[i] == "]":
285
+ depth -= 1
286
+ i += 1
287
+ if depth > 0:
288
+ items.append(_parse_gemma4_array(arr_str[sub_start:i], partial=True))
289
+ else:
290
+ items.append(_parse_gemma4_array(arr_str[sub_start : i - 1]))
291
+
292
+ else:
293
+ val_start = i
294
+ while i < n and arr_str[i] not in (",", "]"):
295
+ i += 1
296
+ if partial and i >= n:
297
+ break
298
+ if i == val_start:
299
+ logger.warning(
300
+ "Gemma4 array parser made no progress at position %d; "
301
+ "aborting on malformed input.",
302
+ i,
303
+ )
304
+ break
305
+ raw_val = arr_str[val_start:i].strip()
306
+ if partial and raw_val.endswith("."):
307
+ break
308
+ items.append(raw_val)
309
+
310
+ return items
311
+
312
+
313
+def _gemma4_arg_converter(raw_args: str, partial: bool) -> str:
314
+ """Convert Gemma4 custom arg format to a JSON string."""
315
+ text = raw_args.strip()
316
+ if text.endswith("}"):
317
+ text = text[:-1]
318
+
319
+ parsed = _parse_gemma4_args(text, partial=partial)
320
+ return json.dumps(parsed, ensure_ascii=False)
321
+
322
+
323
+@functools.cache
324
+def gemma4_config() -> ParserEngineConfig:
325
+ used_tokens = {
326
+ CHANNEL_START,
327
+ CHANNEL_END,
328
+ TOOL_CALL_START,
329
+ TOOL_CALL_END,
330
+ '<|"|>',
331
+ }
332
+
333
+ return ParserEngineConfig(
334
+ name="gemma4",
335
+ initial_state=ParserState.CONTENT,
336
+ terminals={
337
+ "THINK_START": CHANNEL_START,
338
+ "THINK_END": CHANNEL_END,
339
+ "TOOL_START": TOOL_CALL_START,
340
+ "TOOL_END": TOOL_CALL_END,
341
+ "CALL_PREFIX": "call:",
342
+ "OPEN_BRACE": "{",
343
+ },
344
+ token_id_terminals={
345
+ "THINK_START": CHANNEL_START,
346
+ "THINK_END": CHANNEL_END,
347
+ "TOOL_START": TOOL_CALL_START,
348
+ "TOOL_END": TOOL_CALL_END,
349
+ },
350
+ transitions={
351
+ # -- Reasoning transitions --
352
+ (ParserState.CONTENT, "THINK_START"): Transition(
353
+ ParserState.REASONING,
354
+ (EventType.REASONING_START,),
355
+ ),
356
+ (ParserState.REASONING, "THINK_END"): Transition(
357
+ ParserState.CONTENT,
358
+ (EventType.REASONING_END,),
359
+ ),
360
+ # Tool call directly from reasoning (no explicit <channel|>)
361
+ (ParserState.REASONING, "TOOL_START"): Transition(
362
+ ParserState.TOOL_PREAMBLE,
363
+ (EventType.REASONING_END, EventType.TOOL_CALL_START),
364
+ ),
365
+ # -- Tool call transitions --
366
+ (ParserState.CONTENT, "TOOL_START"): Transition(
367
+ ParserState.TOOL_PREAMBLE,
368
+ (EventType.REASONING_END, EventType.TOOL_CALL_START),
369
+ ),
370
+ (ParserState.TOOL_PREAMBLE, "CALL_PREFIX"): Transition(
371
+ ParserState.TOOL_NAME,
372
+ (),
373
+ ),
374
+ (ParserState.TOOL_NAME, "OPEN_BRACE"): Transition(
375
+ ParserState.TOOL_ARGS,
376
+ (),
377
+ ),
378
+ (ParserState.TOOL_ARGS, "TOOL_END"): Transition(
379
+ ParserState.CONTENT,
380
+ (EventType.TOOL_CALL_END,),
381
+ ),
382
+ # Back-to-back tool calls
383
+ (ParserState.CONTENT, "TOOL_END"): Transition(
384
+ ParserState.CONTENT,
385
+ (),
386
+ ),
387
+ # Absorb a bare <channel|> that arrives after we already
388
+ # returned to CONTENT; prevents leaking it as TEXT_CHUNK.
389
+ (ParserState.CONTENT, "THINK_END"): Transition(
390
+ ParserState.CONTENT,
391
+ (),
392
+ ),
393
+ },
394
+ content_events={
395
+ ParserState.CONTENT: EventType.TEXT_CHUNK,
396
+ ParserState.REASONING: EventType.REASONING_CHUNK,
397
+ ParserState.TOOL_NAME: EventType.TOOL_NAME,
398
+ ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
399
+ },
400
+ arg_converter=_gemma4_arg_converter,
401
+ tool_args_json=False,
402
+ arg_structural_chars=frozenset(",:{}[]<"),
403
+ drop_tokens=frozenset(_GEMMA4_MODEL_DROP_TOKENS - used_tokens),
404
+ )
405
+
406
+
407
+_GEMMA4_THOUGHT_PREFIX = "thought\n"
408
+_GEMMA4_THOUGHT_TOKEN = "thought"
409
+
410
+
411
+class Gemma4Parser(ParserEngine):
412
+ """Gemma4 parser: ``<|channel>`` reasoning + ``<|tool_call>``
413
+ tool calls in a single engine.
414
+
415
+ - Strips the ``thought\\n`` prefix from reasoning content
416
+ - Sets ``skip_special_tokens=False`` so boundary tokens are visible
417
+ - Detects ``<|tool_call>`` token as implicit reasoning end
418
+ """
419
+
420
+ def __init__(
421
+ self,
422
+ tokenizer: TokenizerLike,
423
+ tools: list[Tool] | None = None,
424
+ **kwargs,
425
+ ) -> None:
426
+ super().__init__(
427
+ tokenizer,
428
+ tools,
429
+ parser_engine_config=gemma4_config(),
430
+ **kwargs,
431
+ )
432
+ vocab = self.vocab
433
+ self._tool_call_token_id: int | None = vocab.get("<|tool_call>")
434
+ self._new_turn_token_id: int | None = vocab.get("<|turn>")
435
+ self._tool_response_token_id: int | None = vocab.get("<|tool_response>")
436
+ self._reasoning_text: str = ""
437
+ self._prefix_stripped: bool = False
438
+ self._is_first_feed: bool = True
439
+
440
+ def _reset(self, initial_state=None) -> None:
441
+ super()._reset(initial_state=initial_state)
442
+ self._reasoning_text = ""
443
+ self._prefix_stripped = False
444
+ self._is_first_feed = True
445
+
446
+ def _preprocess_feed(
447
+ self,
448
+ delta_text: str,
449
+ delta_token_ids: Sequence[int],
450
+ ) -> tuple[str, Sequence[int]]:
451
+ if not self._is_first_feed:
452
+ return delta_text, delta_token_ids
453
+ self._is_first_feed = False
454
+
455
+ if (
456
+ not delta_text
457
+ or self._engine.state != ParserState.CONTENT
458
+ or self._reasoning_start_token_id is None
459
+ or self._reasoning_end_token_id is None
460
+ ):
461
+ return delta_text, delta_token_ids
462
+
463
+ if CHANNEL_START in delta_text:
464
+ return delta_text, delta_token_ids
465
+
466
+ needs_injection = (
467
+ CHANNEL_END in delta_text
468
+ or delta_text.startswith(_GEMMA4_THOUGHT_PREFIX)
469
+ or delta_text == _GEMMA4_THOUGHT_TOKEN
470
+ )
471
+ if not needs_injection:
472
+ return delta_text, delta_token_ids
473
+
474
+ delta_text = CHANNEL_START + delta_text
475
+ if delta_token_ids:
476
+ delta_token_ids = [self._reasoning_start_token_id, *delta_token_ids]
477
+
478
+ return delta_text, delta_token_ids
479
+
480
+ def is_reasoning_end(self, input_ids: list[int]) -> bool:
481
+ end_id = self._reasoning_end_token_id
482
+ start_id = self._reasoning_start_token_id
483
+ tool_call_id = self._tool_call_token_id
484
+ new_turn_id = self._new_turn_token_id
485
+ tool_response_id = self._tool_response_token_id
486
+
487
+ if end_id is not None and not input_ids:
488
+ return self.parser_engine_config.initial_state != ParserState.REASONING
489
+
490
+ for i in range(len(input_ids) - 1, -1, -1):
491
+ tid = input_ids[i]
492
+ if start_id is not None and tid == start_id:
493
+ return False
494
+ if tool_call_id is not None and tid == tool_call_id:
495
+ return True
496
+ if new_turn_id is not None and tid == new_turn_id:
497
+ return False
498
+ if tool_response_id is not None and tid == tool_response_id:
499
+ return False
500
+ if end_id is not None and tid == end_id:
501
+ return True
502
+ return self._reasoning_ended
503
+
504
+ def _events_to_delta(
505
+ self,
506
+ events: list[SemanticEvent],
507
+ finished: bool = False,
508
+ ) -> DeltaMessage | None:
509
+ delta = super()._events_to_delta(events, finished=finished)
510
+ if delta is None or delta.reasoning is None:
511
+ return delta
512
+
513
+ if self._prefix_stripped:
514
+ return delta
515
+ self._reasoning_text += delta.reasoning
516
+
517
+ if self._reasoning_text.startswith(_GEMMA4_THOUGHT_PREFIX):
518
+ prefix_len = len(_GEMMA4_THOUGHT_PREFIX)
519
+ prev_reasoning_len = len(self._reasoning_text) - len(delta.reasoning)
520
+ if prev_reasoning_len >= prefix_len:
521
+ self._prefix_stripped = True
522
+ return delta
523
+ chars_of_prefix_in_delta = prefix_len - prev_reasoning_len
524
+ stripped = delta.reasoning[chars_of_prefix_in_delta:]
525
+ if stripped:
526
+ self._prefix_stripped = True
527
+ delta.reasoning = stripped
528
+ return delta
529
+ if len(self._reasoning_text) >= prefix_len:
530
+ self._prefix_stripped = True
531
+ delta.reasoning = None
532
+ if delta.content is not None or delta.tool_calls:
533
+ return delta
534
+ return None
535
+ return None
536
+
537
+ if _GEMMA4_THOUGHT_PREFIX.startswith(self._reasoning_text):
538
+ if finished:
539
+ self._prefix_stripped = True
540
+ return None
541
+
542
+ self._prefix_stripped = True
543
+ delta.reasoning = self._reasoning_text
544
+ return delta
545
+
546
+ def extract_reasoning(
547
+ self,
548
+ model_output: str,
549
+ request: ChatCompletionRequest | ResponsesRequest,
550
+ ) -> tuple[str | None, str | None]:
551
+ reasoning, content = super().extract_reasoning(model_output, request)
552
+ if reasoning:
553
+ if reasoning.startswith(_GEMMA4_THOUGHT_PREFIX):
554
+ reasoning = reasoning[len(_GEMMA4_THOUGHT_PREFIX) :]
555
+ elif reasoning == _GEMMA4_THOUGHT_PREFIX.rstrip():
556
+ reasoning = None
557
+ return reasoning or None, content
vllm/parser/qwen3.py
+1
-1
@@ -118,7 +118,7 @@ def qwen3_config(thinking: bool = True) -> ParserEngineConfig:
118
# -- Tool call transitions --
119
(ParserState.CONTENT, "TOOL_START"): Transition(
120
ParserState.TOOL_PREAMBLE,
121
- (EventType.TOOL_CALL_START,),
121
+ (EventType.REASONING_END, EventType.TOOL_CALL_START),
122
),
123
# Fallback: <function= without a preceding <tool_call>
124
(ParserState.CONTENT, "FUNC_PREFIX"): Transition(
vllm/reasoning/__init__.py
+2
-2
@@ -49,8 +49,8 @@ _REASONING_PARSERS_TO_REGISTER = {
49
"Ernie45ReasoningParser",
50
),
51
"gemma4": (
52
- "gemma4_reasoning_parser",
53
- "Gemma4ReasoningParser",
52
+ "gemma4_engine_reasoning_parser",
53
+ "Gemma4ParserReasoningAdapter",
54
),
55
"glm45": (
56
"deepseek_v3_reasoning_parser",
vllm/reasoning/gemma4_engine_reasoning_parser.py
new
+6
@@ -0,0 +1,6 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from vllm.parser.engine.registered_adapters import Gemma4ParserReasoningAdapter
5
+
6
+__all__ = ["Gemma4ParserReasoningAdapter"]
vllm/reasoning/gemma4_reasoning_parser.py
deleted
-225
@@ -1,225 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-from collections.abc import Sequence
5
-from typing import TYPE_CHECKING
6
-
7
-from vllm.entrypoints.openai.engine.protocol import DeltaMessage
8
-from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
9
-from vllm.tokenizers import TokenizerLike
10
-
11
-if TYPE_CHECKING:
12
- from vllm.entrypoints.openai.chat_completion.protocol import (
13
- ChatCompletionRequest,
14
- )
15
- from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
16
-
17
-# Role label that Gemma4 emits at the start of the thinking channel.
18
-# The model generates: <|channel>thought\n...reasoning...<channel|>
19
-# This prefix must be stripped to expose only the actual reasoning content.
20
-_THOUGHT_PREFIX = "thought\n"
21
-
22
-
23
-class Gemma4ReasoningParser(BaseThinkingReasoningParser):
24
- """
25
- Reasoning parser for Google Gemma4 thinking models.
26
-
27
- Gemma4 uses <|channel>...<channel|> tokens to delimit reasoning/thinking
28
- content within its output. Thinking mode is activated by passing
29
- ``enable_thinking=True`` in the chat template kwargs, which injects a
30
- system turn containing <|think|> (token 98) to trigger chain-of-thought
31
- reasoning.
32
-
33
- Output pattern when thinking is enabled::
34
-
35
- <|channel>thought
36
- ...chain of thought reasoning...<channel|>
37
- Final answer text here.
38
-
39
- The ``thought\\n`` role label inside the channel delimiters is a
40
- structural artefact (analogous to ``user\\n`` in ``<|turn>user\\n...``).
41
- This parser strips it so that downstream consumers see only the
42
- actual reasoning text, consistent with the offline parser
43
- (``vllm.reasoning.gemma4_utils._strip_thought_label``).
44
- """
45
-
46
- def __init__(self, tokenizer: TokenizerLike, *args, **kwargs):
47
- super().__init__(tokenizer, *args, **kwargs)
48
- # Instance state for streaming prefix stripping.
49
- # Tracks only the reasoning text received from the base parser,
50
- # independent of current_text (which may contain pre-reasoning
51
- # content and lacks special token text due to
52
- # skip_special_tokens=True).
53
- self._reasoning_text: str = ""
54
- self._prefix_stripped: bool = False
55
- self.new_turn_token_id = self.vocab["<|turn>"]
56
- self.tool_call_token_id = self.vocab["<|tool_call>"]
57
- self.tool_response_token_id = self.vocab["<|tool_response>"]
58
-
59
- def adjust_request(
60
- self, request: "ChatCompletionRequest | ResponsesRequest"
61
- ) -> "ChatCompletionRequest | ResponsesRequest":
62
- """Disable special-token stripping to preserve boundary tokens."""
63
- request.skip_special_tokens = False
64
- return request
65
-
66
- @property
67
- def start_token(self) -> str:
68
- """The token that starts reasoning content."""
69
- return "<|channel>"
70
-
71
- @property
72
- def end_token(self) -> str:
73
- """The token that ends reasoning content."""
74
- return "<channel|>"
75
-
76
- def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
77
- start_token_id = self.start_token_id
78
- end_token_id = self.end_token_id
79
- new_turn_token_id = self.new_turn_token_id
80
- tool_call_token_id = self.tool_call_token_id
81
- tool_response_token_id = self.tool_response_token_id
82
-
83
- # Search from the end of input_ids to find the last match.
84
- for i in range(len(input_ids) - 1, -1, -1):
85
- if input_ids[i] == start_token_id:
86
- return False
87
- if input_ids[i] == tool_call_token_id:
88
- # We're generating a tool call, so reasoning must be ended.
89
- return True
90
- if input_ids[i] in (new_turn_token_id, tool_response_token_id):
91
- # We found a new turn or tool response token so don't consider
92
- # reasoning ended yet, since the model starts new reasoning
93
- # after these tokens.
94
- return False
95
- if input_ids[i] == end_token_id:
96
- return True
97
- return False
98
-
99
- # ------------------------------------------------------------------
100
- # Non-streaming path
101
- # ------------------------------------------------------------------
102
-
103
- def extract_reasoning(
104
- self,
105
- model_output: str,
106
- request: "ChatCompletionRequest | ResponsesRequest",
107
- ) -> tuple[str | None, str | None]:
108
- """Extract reasoning, stripping the ``thought\\n`` role label."""
109
- if self.start_token not in model_output and self.end_token not in model_output:
110
- # Default to content history if no tags are present
111
- # (or if they were stripped)
112
- return None, model_output
113
-
114
- reasoning, content = super().extract_reasoning(model_output, request)
115
- if reasoning is not None:
116
- reasoning = _strip_thought_label(reasoning)
117
- return reasoning, content
118
-
119
- # ------------------------------------------------------------------
120
- # Streaming path
121
- # ------------------------------------------------------------------
122
-
123
- def extract_reasoning_streaming(
124
- self,
125
- previous_text: str,
126
- current_text: str,
127
- delta_text: str,
128
- previous_token_ids: Sequence[int],
129
- current_token_ids: Sequence[int],
130
- delta_token_ids: Sequence[int],
131
- ) -> DeltaMessage | None:
132
- """Extract streaming reasoning, stripping ``thought\\n`` from the
133
- first reasoning delta(s).
134
-
135
- The ``thought\\n`` prefix may arrive as a single delta or split
136
- across multiple deltas (e.g. ``"thought"`` then ``"\\n"``). We
137
- buffer early reasoning tokens until we can determine whether the
138
- prefix is present, then emit the buffered content minus the
139
- prefix.
140
-
141
- Unlike the previous implementation which reconstructed accumulated
142
- reasoning from ``current_text``, this uses instance state
143
- (``_reasoning_text``) to track only the reasoning content returned
144
- by the base parser. This is necessary because
145
- ``skip_special_tokens=True`` (the vLLM default) causes the
146
- ``<|channel>`` delimiter to be invisible in ``current_text``,
147
- making it impossible to separate pre-reasoning content from
148
- reasoning content via string matching.
149
- """
150
- result = super().extract_reasoning_streaming(
151
- previous_text,
152
- current_text,
153
- delta_text,
154
- previous_token_ids,
155
- current_token_ids,
156
- delta_token_ids,
157
- )
158
- if result is None:
159
- return None
160
-
161
- if result.reasoning is None:
162
- return result
163
-
164
- # Accumulate ONLY the reasoning text from base parser results.
165
- # This is immune to pre-reasoning content pollution.
166
- self._reasoning_text += result.reasoning
167
-
168
- # Once the prefix has been handled, all subsequent reasoning
169
- # deltas pass through unchanged.
170
- if self._prefix_stripped:
171
- return result
172
-
173
- # ---- Prefix stripping logic ----
174
-
175
- # Case 1: We've accumulated enough to confirm the prefix is
176
- # present. Strip it and pass through the remainder.
177
- if self._reasoning_text.startswith(_THOUGHT_PREFIX):
178
- prefix_len = len(_THOUGHT_PREFIX)
179
- # How much reasoning was accumulated before this delta?
180
- prev_reasoning_len = len(self._reasoning_text) - len(result.reasoning)
181
- if prev_reasoning_len >= prefix_len:
182
- # Prefix was already consumed by prior deltas; this
183
- # delta is entirely real content — pass through.
184
- self._prefix_stripped = True
185
- return result
186
- else:
187
- # Part or all of the prefix is in this delta.
188
- chars_of_prefix_in_delta = prefix_len - prev_reasoning_len
189
- stripped = result.reasoning[chars_of_prefix_in_delta:]
190
- if stripped:
191
- self._prefix_stripped = True
192
- result.reasoning = stripped
193
- return result
194
- else:
195
- if len(self._reasoning_text) >= prefix_len:
196
- self._prefix_stripped = True
197
- result.reasoning = ""
198
- return result
199
- return None
200
-
201
- # Case 2: Accumulated text is a strict prefix of
202
- # _THOUGHT_PREFIX (e.g. we've only seen "thou" so far).
203
- # Buffer by suppressing — we can't yet tell if this will
204
- # become the full prefix or diverge.
205
- if _THOUGHT_PREFIX.startswith(self._reasoning_text):
206
- return None
207
-
208
- # Case 3: Accumulated text doesn't match the thought prefix
209
- # at all. This means prior deltas were buffered (suppressed
210
- # by Case 2) but the text diverged. Re-emit the full
211
- # accumulated text to avoid data loss.
212
- self._prefix_stripped = True
213
- result.reasoning = self._reasoning_text
214
- return result
215
-
216
-
217
-def _strip_thought_label(text: str) -> str:
218
- """Remove the ``thought\\n`` role label from the beginning of text.
219
-
220
- Mirrors ``vllm.reasoning.gemma4_utils._strip_thought_label`` from the
221
- offline parser.
222
- """
223
- if text.startswith(_THOUGHT_PREFIX):
224
- return text[len(_THOUGHT_PREFIX) :]
225
- return text
vllm/tool_parsers/__init__.py
+2
-2
@@ -191,8 +191,8 @@ _TOOL_PARSERS_TO_REGISTER = {
191
"FunctionGemmaToolParser",
192
),
193
"gemma4": (
194
- "gemma4_tool_parser",
195
- "Gemma4ToolParser",
194
+ "gemma4_engine_tool_parser",
195
+ "Gemma4EngineToolParser",
196
),
197
"apertus": (
198
"apertus_tool_parser",
vllm/tool_parsers/gemma4_engine_tool_parser.py
new
+8
@@ -0,0 +1,8 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from vllm.parser.engine.registered_adapters import Gemma4ParserToolAdapter
5
+
6
+
7
+class Gemma4EngineToolParser(Gemma4ParserToolAdapter): # type: ignore[valid-type, misc]
8
+ supports_required_and_named = False
vllm/tool_parsers/gemma4_tool_parser.py
deleted
-896
@@ -1,896 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-"""
4
-Tool call parser for Google Gemma4 models.
5
-
6
-Gemma4 uses a custom serialization format (not JSON) for tool calls::
7
-
8
- <|tool_call>call:func_name{key:<|"|>value<|"|>,num:42}<tool_call|>
9
-
10
-Strings are delimited by ``<|"|>`` (token 52), keys are unquoted, and
11
-multiple tool calls are concatenated without separators.
12
-
13
-Used when ``--enable-auto-tool-choice --tool-call-parser gemma4`` are set.
14
-
15
-For offline inference tool call parsing (direct ``tokenizer.decode()`` output),
16
-see ``vllm.tool_parsers.gemma4_utils.parse_tool_calls``.
17
-"""
18
-
19
-import json
20
-from collections.abc import Sequence
21
-
22
-import regex as re
23
-from openai.types.responses import ToolChoiceFunction
24
-
25
-from vllm.entrypoints.chat_utils import make_tool_call_id
26
-from vllm.entrypoints.openai.chat_completion.protocol import (
27
- ChatCompletionNamedToolChoiceParam,
28
- ChatCompletionRequest,
29
-)
30
-from vllm.entrypoints.openai.engine.protocol import (
31
- DeltaFunctionCall,
32
- DeltaMessage,
33
- DeltaToolCall,
34
- ExtractedToolCallInformation,
35
- FunctionCall,
36
- ToolCall,
37
-)
38
-from vllm.entrypoints.openai.responses.protocol import (
39
- ResponsesRequest,
40
-)
41
-from vllm.logger import init_logger
42
-from vllm.tokenizers import TokenizerLike
43
-from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser
44
-from vllm.tool_parsers.utils import find_common_prefix
45
-
46
-logger = init_logger(__name__)
47
-
48
-# Gemma4 special tokens for tool calls
49
-TOOL_CALL_START = "<|tool_call>"
50
-TOOL_CALL_END = "<tool_call|>"
51
-STRING_DELIM = '<|"|>'
52
-
53
-
54
-# ---------------------------------------------------------------------------
55
-# Gemma4 argument parser (used by both streaming and non-streaming paths)
56
-# ---------------------------------------------------------------------------
57
-
58
-
59
-def _parse_gemma4_value(value_str: str) -> object:
60
- """Parse a single Gemma4 value (after key:) into a Python object."""
61
- value_str = value_str.strip()
62
- if not value_str:
63
- return value_str
64
-
65
- # Boolean
66
- if value_str == "true":
67
- return True
68
- if value_str == "false":
69
- return False
70
-
71
- # Null
72
- if value_str.lower() in ("null", "none", "nil"):
73
- return None
74
-
75
- # Number (int or float)
76
- try:
77
- if "." in value_str:
78
- return float(value_str)
79
- return int(value_str)
80
- except ValueError:
81
- pass
82
-
83
- # Bare string (no <|"|> delimiters — shouldn't happen but be safe)
84
- return value_str
85
-
86
-
87
-def _parse_gemma4_args(args_str: str, *, partial: bool = False) -> dict:
88
- """Parse Gemma4's custom key:value format into a Python dict.
89
-
90
- Format examples::
91
-
92
- location:<|"|>Tokyo<|"|>
93
- location:<|"|>San Francisco<|"|>,unit:<|"|>celsius<|"|>
94
- count:42,flag:true
95
- nested:{inner_key:<|"|>val<|"|>}
96
- items:[<|"|>a<|"|>,<|"|>b<|"|>]
97
-
98
- Args:
99
- args_str: The raw Gemma4 argument string.
100
- partial: When True (streaming), bare values at end of string are
101
- omitted because they may be incomplete and type-unstable
102
- (e.g. partial boolean parsed as bare string).
103
-
104
- Returns a dict ready for ``json.dumps()``.
105
- """
106
- if not args_str or not args_str.strip():
107
- return {}
108
-
109
- result: dict = {}
110
- i = 0
111
- n = len(args_str)
112
-
113
- while i < n:
114
- # Skip whitespace and commas
115
- while i < n and args_str[i] in (" ", ",", "\n", "\t"):
116
- i += 1
117
- if i >= n:
118
- break
119
-
120
- # Parse key (unquoted, ends at ':')
121
- key_start = i
122
- while i < n and args_str[i] != ":":
123
- i += 1
124
- if i >= n:
125
- break
126
- key = args_str[key_start:i].strip()
127
- i += 1 # skip ':'
128
-
129
- # Parse value
130
- if i >= n:
131
- if not partial:
132
- result[key] = ""
133
- break
134
-
135
- # Skip whitespace after ':'
136
- while i < n and args_str[i] in (" ", "\n", "\t"):
137
- i += 1
138
- if i >= n:
139
- if not partial:
140
- result[key] = ""
141
- break
142
-
143
- # String value: <|"|>...<|"|>
144
- if args_str[i:].startswith(STRING_DELIM):
145
- i += len(STRING_DELIM)
146
- val_start = i
147
- end_pos = args_str.find(STRING_DELIM, i)
148
- if end_pos == -1:
149
- # Unterminated string — take rest
150
- result[key] = args_str[val_start:]
151
- break
152
- result[key] = args_str[val_start:end_pos]
153
- i = end_pos + len(STRING_DELIM)
154
-
155
- # Nested object: {...}
156
- elif args_str[i] == "{":
157
- depth = 1
158
- obj_start = i + 1
159
- i += 1
160
- while i < n and depth > 0:
161
- if args_str[i:].startswith(STRING_DELIM):
162
- # Skip over string contents to avoid counting { inside strings
163
- i += len(STRING_DELIM)
164
- next_delim = args_str.find(STRING_DELIM, i)
165
- i = n if next_delim == -1 else next_delim + len(STRING_DELIM)
166
- continue
167
- if args_str[i] == "{":
168
- depth += 1
169
- elif args_str[i] == "}":
170
- depth -= 1
171
- i += 1
172
- if depth > 0:
173
- # Incomplete nested object — use i (not i-1) to avoid
174
- # dropping the last char, and recurse as partial.
175
- result[key] = _parse_gemma4_args(args_str[obj_start:i], partial=True)
176
- else:
177
- result[key] = _parse_gemma4_args(args_str[obj_start : i - 1])
178
-
179
- # Array: [...]
180
- elif args_str[i] == "[":
181
- depth = 1
182
- arr_start = i + 1
183
- i += 1
184
- while i < n and depth > 0:
185
- if args_str[i:].startswith(STRING_DELIM):
186
- i += len(STRING_DELIM)
187
- next_delim = args_str.find(STRING_DELIM, i)
188
- i = n if next_delim == -1 else next_delim + len(STRING_DELIM)
189
- continue
190
- if args_str[i] == "[":
191
- depth += 1
192
- elif args_str[i] == "]":
193
- depth -= 1
194
- i += 1
195
- if depth > 0:
196
- result[key] = _parse_gemma4_array(args_str[arr_start:i], partial=True)
197
- else:
198
- result[key] = _parse_gemma4_array(args_str[arr_start : i - 1])
199
-
200
- # Bare value (number, boolean, etc.)
201
- else:
202
- val_start = i
203
- while i < n and args_str[i] not in (",", "}", "]"):
204
- i += 1
205
- if partial and i >= n:
206
- # Value may be incomplete (e.g. partial boolean) —
207
- # withhold to avoid type instability during streaming.
208
- break
209
- if i == val_start:
210
- logger.warning(
211
- "Gemma4 args parser made no progress at position %d; "
212
- "aborting on malformed input.",
213
- i,
214
- )
215
- break
216
- if partial:
217
- raw_val = args_str[val_start:i].strip()
218
- if raw_val.endswith("."):
219
- # Trailing dot means decimal digits may still arrive
220
- # (e.g. "108." may become "108.2"). Parsing now would
221
- # yield float("108.") == 108.0, whose json repr "108.0"
222
- # corrupts the streaming diff when the true digit lands.
223
- break
224
- result[key] = _parse_gemma4_value(args_str[val_start:i])
225
-
226
- return result
227
-
228
-
229
-def _parse_gemma4_array(arr_str: str, *, partial: bool = False) -> list:
230
- """Parse a Gemma4 array content string into a Python list."""
231
- items: list = []
232
- i = 0
233
- n = len(arr_str)
234
-
235
- while i < n:
236
- while i < n and arr_str[i] in (" ", ",", "\n", "\t"):
237
- i += 1
238
- if i >= n:
239
- break
240
-
241
- # String element
242
- if arr_str[i:].startswith(STRING_DELIM):
243
- i += len(STRING_DELIM)
244
- end_pos = arr_str.find(STRING_DELIM, i)
245
- if end_pos == -1:
246
- items.append(arr_str[i:])
247
- break
248
- items.append(arr_str[i:end_pos])
249
- i = end_pos + len(STRING_DELIM)
250
-
251
- # Nested object
252
- elif arr_str[i] == "{":
253
- depth = 1
254
- obj_start = i + 1
255
- i += 1
256
- while i < n and depth > 0:
257
- if arr_str[i:].startswith(STRING_DELIM):
258
- i += len(STRING_DELIM)
259
- nd = arr_str.find(STRING_DELIM, i)
260
- i = nd + len(STRING_DELIM) if nd != -1 else n
261
- continue
262
- if arr_str[i] == "{":
263
- depth += 1
264
- elif arr_str[i] == "}":
265
- depth -= 1
266
- i += 1
267
- if depth > 0:
268
- items.append(_parse_gemma4_args(arr_str[obj_start:i], partial=True))
269
- else:
270
- items.append(_parse_gemma4_args(arr_str[obj_start : i - 1]))
271
-
272
- # Nested array
273
- elif arr_str[i] == "[":
274
- depth = 1
275
- sub_start = i + 1
276
- i += 1
277
- while i < n and depth > 0:
278
- if arr_str[i:].startswith(STRING_DELIM):
279
- i += len(STRING_DELIM)
280
- nd = arr_str.find(STRING_DELIM, i)
281
- i = nd + len(STRING_DELIM) if nd != -1 else n
282
- continue
283
- if arr_str[i] == "[":
284
- depth += 1
285
- elif arr_str[i] == "]":
286
- depth -= 1
287
- i += 1
288
- if depth > 0:
289
- items.append(_parse_gemma4_array(arr_str[sub_start:i], partial=True))
290
- else:
291
- items.append(_parse_gemma4_array(arr_str[sub_start : i - 1]))
292
-
293
- # Bare value
294
- else:
295
- val_start = i
296
- while i < n and arr_str[i] not in (",", "]"):
297
- i += 1
298
- if partial and i >= n:
299
- break
300
- if i == val_start:
301
- logger.warning(
302
- "Gemma4 array parser made no progress at position %d; "
303
- "aborting on malformed input.",
304
- i,
305
- )
306
- break
307
- if partial:
308
- raw_val = arr_str[val_start:i].strip()
309
- if raw_val.endswith("."):
310
- break
311
- items.append(_parse_gemma4_value(arr_str[val_start:i]))
312
-
313
- return items
314
-
315
-
316
-# ---------------------------------------------------------------------------
317
-# Parser
318
-# ---------------------------------------------------------------------------
319
-
320
-
321
-class Gemma4ToolParser(ToolParser):
322
- """
323
- Tool call parser for Google Gemma4 models.
324
-
325
- Handles the Gemma4 function call format::
326
-
327
- <|tool_call>call:func_name{key:<|"|>value<|"|>}<tool_call|>
328
-
329
- Used when ``--enable-auto-tool-choice --tool-call-parser gemma4``
330
- are set.
331
-
332
- Streaming strategy: **accumulate-then-parse-then-diff**
333
-
334
- Instead of trying to convert Gemma4's custom format to JSON
335
- token-by-token (which fails because Gemma4 uses bare keys, custom
336
- delimiters, and structural braces that differ from JSON), this parser:
337
-
338
- 1. Accumulates the raw Gemma4 argument string during streaming
339
- 2. Parses it with ``_parse_gemma4_args()`` into a Python dict
340
- 3. Converts to JSON with ``json.dumps()``
341
- 4. Diffs against the previously-streamed JSON string
342
- 5. Emits only the new JSON fragment as the delta
343
-
344
- This follows the same pattern used by FunctionGemma, Hermes, and Llama
345
- tool parsers.
346
- """
347
-
348
- # Gemma4 emits native special-token tool calls, not generic JSON calls.
349
- supports_required_and_named = False
350
-
351
- def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
352
- super().__init__(tokenizer, tools)
353
-
354
- if not self.model_tokenizer:
355
- raise ValueError(
356
- "The model tokenizer must be passed to the ToolParser "
357
- "constructor during construction."
358
- )
359
-
360
- # Token strings
361
- self.tool_call_start_token = TOOL_CALL_START
362
- self.tool_call_end_token = TOOL_CALL_END
363
-
364
- # Token IDs
365
- self.tool_call_start_token_id = self.vocab.get(TOOL_CALL_START)
366
- self.tool_call_end_token_id = self.vocab.get(TOOL_CALL_END)
367
-
368
- if self.tool_call_start_token_id is None:
369
- raise RuntimeError(
370
- "Gemma4 ToolParser could not locate the tool call start "
371
- f"token '{TOOL_CALL_START}' in the tokenizer!"
372
- )
373
-
374
- # Regex for non-streaming: extract complete tool calls.
375
- # Supports function names with letters, digits, underscores,
376
- # hyphens, and dots (e.g. "get-weather", "module.func").
377
- self.tool_call_regex = re.compile(
378
- r"<\|tool_call>call:([\w\-\.]+)\{(.*?)\}<tool_call\|>",
379
- re.DOTALL,
380
- )
381
-
382
- # Streaming state — reset per-request via _reset_streaming_state()
383
- self._reset_streaming_state()
384
-
385
- # Delta buffer for handling multi-token special sequences
386
- self.buffered_delta_text = ""
387
-
388
- def _reset_streaming_state(self) -> None:
389
- """Reset all streaming state for a new request."""
390
- self.current_tool_id = -1
391
- self.current_tool_name_sent = False
392
- self.prev_tool_call_arr: list[dict] = []
393
- self.streamed_args_for_tool: list[str] = []
394
-
395
- def adjust_request(
396
- self, request: ChatCompletionRequest | ResponsesRequest
397
- ) -> ChatCompletionRequest | ResponsesRequest:
398
- if request.tools:
399
- tc = request.tool_choice
400
- if tc == "required" or isinstance(
401
- tc,
402
- (ChatCompletionNamedToolChoiceParam, ToolChoiceFunction),
403
- ):
404
- # Do NOT call super().adjust_request() for required/named tool
405
- # choice. The base implementation injects a JSON-array
406
- # `structured_outputs` schema and forces xgrammar guided
407
- # decoding, which conflicts with Gemma4's native
408
- # `<|tool_call>call:...` (non-JSON) tool syntax and crashes
409
- # EngineCore under MTP spec decode. The streaming/extraction
410
- # parser already handles the native output, so guided decoding
411
- # is skipped here (mirrors the GLM4 precedent).
412
- if request.tool_choice != "none":
413
- request.skip_special_tokens = False
414
- return request
415
- request = super().adjust_request(request)
416
- if request.tools and request.tool_choice != "none":
417
- # Don't skip special tokens — <|tool_call> etc. are needed for
418
- # the parser to detect tool calls. Apply to BOTH
419
- # ChatCompletionRequest and ResponsesRequest (the previous
420
- # isinstance(ChatCompletionRequest) guard caused tool-call
421
- # delimiters to be stripped on /v1/responses, leaking raw
422
- # `call:fn{...}` text via output_text.delta).
423
- request.skip_special_tokens = False
424
- return request
425
-
426
- # ------------------------------------------------------------------
427
- # Delta buffering for multi-token special sequences
428
- # ------------------------------------------------------------------
429
-
430
- def _buffer_delta_text(self, delta_text: str) -> str:
431
- """Buffer incoming delta text to handle multi-token special sequences.
432
-
433
- Accumulates partial tokens that could be the start of
434
- ``<|tool_call>`` or ``<tool_call|>`` and only flushes them
435
- when the complete sequence is recognized or the sequence breaks.
436
-
437
- This prevents partial special tokens (e.g., ``<|tool``) from being
438
- emitted prematurely as content text.
439
- """
440
- combined = self.buffered_delta_text + delta_text
441
-
442
- # Check if combined ends with a complete special token
443
- if combined.endswith(TOOL_CALL_START) or combined.endswith(TOOL_CALL_END):
444
- self.buffered_delta_text = ""
445
- return combined
446
-
447
- # Check if combined ends with a partial prefix of a special token
448
- for tag in [TOOL_CALL_START, TOOL_CALL_END]:
449
- for i in range(1, len(tag)):
450
- if combined.endswith(tag[:i]):
451
- self.buffered_delta_text = combined[-i:]
452
- return combined[:-i]
453
-
454
- # No partial match — flush everything
455
- self.buffered_delta_text = ""
456
- return combined
457
-
458
- # ------------------------------------------------------------------
459
- # Non-streaming extraction
460
- # ------------------------------------------------------------------
461
-
462
- def extract_tool_calls(
463
- self,
464
- model_output: str,
465
- request: ChatCompletionRequest,
466
- ) -> ExtractedToolCallInformation:
467
- if self.tool_call_start_token not in model_output:
468
- return ExtractedToolCallInformation(
469
- tools_called=False, tool_calls=[], content=model_output
470
- )
471
-
472
- try:
473
- matches = self.tool_call_regex.findall(model_output)
474
- if not matches:
475
- return ExtractedToolCallInformation(
476
- tools_called=False, tool_calls=[], content=model_output
477
- )
478
-
479
- tool_calls: list[ToolCall] = []
480
- for func_name, args_str in matches:
481
- arguments = _parse_gemma4_args(args_str)
482
- tool_calls.append(
483
- ToolCall(
484
- type="function",
485
- function=FunctionCall(
486
- name=func_name,
487
- arguments=json.dumps(arguments, ensure_ascii=False),
488
- ),
489
- )
490
- )
491
-
492
- # Content = text before first tool call (if any)
493
- content_end = model_output.find(self.tool_call_start_token)
494
- content = model_output[:content_end].strip() if content_end > 0 else None
495
-
496
- return ExtractedToolCallInformation(
497
- tools_called=True,
498
- tool_calls=tool_calls,
499
- content=content if content else None,
500
- )
501
-
502
- except Exception:
503
- logger.exception("Error extracting tool calls from Gemma4 response")
504
- return ExtractedToolCallInformation(
505
- tools_called=False, tool_calls=[], content=model_output
506
- )
507
-
508
- # ------------------------------------------------------------------
509
- # Streaming extraction — accumulate-then-parse-then-diff
510
- # ------------------------------------------------------------------
511
-
512
- def extract_tool_calls_streaming(
513
- self,
514
- previous_text: str,
515
- current_text: str,
516
- delta_text: str,
517
- previous_token_ids: Sequence[int],
518
- current_token_ids: Sequence[int],
519
- delta_token_ids: Sequence[int],
520
- request: ChatCompletionRequest,
521
- ) -> DeltaMessage | None:
522
- # Buffer delta text to handle multi-token special sequences
523
- delta_text = self._buffer_delta_text(delta_text)
524
- # Keep current_text from the upstream stream state. The buffered delta
525
- # is only for emission, and must not be stitched back into the
526
- # accumulated model text or normal content like "<div>" can be
527
- # duplicated into "<<div>" when a tool call just ended.
528
-
529
- # If no tool call token seen yet, emit as content
530
- if self.tool_call_start_token not in current_text:
531
- if delta_text:
532
- return DeltaMessage(content=delta_text)
533
- return None
534
-
535
- try:
536
- return self._extract_streaming(
537
- previous_text=previous_text,
538
- current_text=current_text,
539
- delta_text=delta_text,
540
- )
541
- except Exception:
542
- logger.exception("Error in Gemma4 streaming tool call extraction")
543
- return None
544
-
545
- def _extract_streaming(
546
- self,
547
- previous_text: str,
548
- current_text: str,
549
- delta_text: str,
550
- ) -> DeltaMessage | None:
551
- """Tag-counting streaming parser.
552
-
553
- Uses the proven approach from FunctionGemma/Hermes: count start/end
554
- tags in previous vs current text to determine phase, then
555
- accumulate-parse-diff for arguments.
556
-
557
- Format: ``<|tool_call>call:name{args}<tool_call|>``
558
- """
559
- start_count = current_text.count(self.tool_call_start_token)
560
- end_count = current_text.count(self.tool_call_end_token)
561
- prev_start_count = previous_text.count(self.tool_call_start_token)
562
- prev_end_count = previous_text.count(self.tool_call_end_token)
563
-
564
- # Case 1: Not inside any tool call — emit as content
565
- if (
566
- start_count == end_count
567
- and prev_end_count == end_count
568
- and self.tool_call_end_token not in delta_text
569
- ):
570
- if delta_text:
571
- return DeltaMessage(content=delta_text)
572
- return None
573
-
574
- # Case 2: One or more new tool calls started in this delta.
575
- # A single delta can batch several complete calls, so advance the
576
- # tool id once per newly-seen start token and allocate a tracking
577
- # slot for each.
578
- if start_count > prev_start_count:
579
- num_new = start_count - prev_start_count
580
- for _ in range(num_new):
581
- self.current_tool_id += 1
582
- self.streamed_args_for_tool.append("")
583
- self.prev_tool_call_arr.append({})
584
- self.current_tool_name_sent = False
585
- logger.debug(
586
- "Started %d new tool call(s); current_tool_id=%d",
587
- num_new,
588
- self.current_tool_id,
589
- )
590
- # Don't return yet if this delta also contains call payload or
591
- # the end marker; backends can batch one or more complete tool
592
- # calls into a single streaming chunk. Only wait for more text
593
- # when the delta is just the start token itself.
594
- if start_count > end_count and len(delta_text) <= len(
595
- self.tool_call_start_token
596
- ):
597
- return None
598
-
599
- # Case 3: One or more tool calls just ended (possibly several in a
600
- # single batched delta) — drain every newly-completed call.
601
- if end_count > prev_end_count:
602
- return self._handle_tool_call_end(
603
- current_text,
604
- prev_end_count=prev_end_count,
605
- end_count=end_count,
606
- start_count=start_count,
607
- )
608
-
609
- # Case 4: In the middle of a tool call — parse partial content
610
- if start_count > end_count:
611
- return self._handle_tool_call_middle(current_text)
612
-
613
- # Default: generate text outside tool calls
614
- if delta_text:
615
- text = delta_text.replace(self.tool_call_start_token, "")
616
- text = text.replace(self.tool_call_end_token, "")
617
- if text:
618
- return DeltaMessage(content=text)
619
- return None
620
-
621
- def _extract_partial_call(self, current_text: str) -> tuple[str | None, str]:
622
- """Extract function name and raw argument string from partial text.
623
-
624
- Returns (func_name, raw_args_str) or (None, "") if not parseable yet.
625
- """
626
- # Get the text after the last <|tool_call> token
627
- last_start = current_text.rfind(self.tool_call_start_token)
628
- if last_start == -1:
629
- return None, ""
630
-
631
- partial_call = current_text[last_start + len(self.tool_call_start_token) :]
632
-
633
- # Strip end token if present
634
- if self.tool_call_end_token in partial_call:
635
- partial_call = partial_call.split(self.tool_call_end_token)[0]
636
-
637
- # Expect "call:name{args...}" or "call:name{args...}"
638
- if not partial_call.startswith("call:"):
639
- return None, ""
640
-
641
- func_part = partial_call[5:] # skip "call:"
642
-
643
- if "{" not in func_part:
644
- # Still accumulating function name, not ready yet
645
- return None, ""
646
-
647
- func_name, _, args_part = func_part.partition("{")
648
- func_name = func_name.strip()
649
-
650
- # Strip trailing '}' if present (Gemma4 structural brace)
651
- if args_part.endswith("}"):
652
- args_part = args_part[:-1]
653
-
654
- return func_name, args_part
655
-
656
- def _handle_tool_call_middle(self, current_text: str) -> DeltaMessage | None:
657
- """Handle streaming when we're inside an active tool call.
658
-
659
- Accumulates the raw Gemma4 arguments, parses them into JSON, and
660
- diffs against the previously-streamed JSON to emit only the new
661
- fragment.
662
- """
663
- func_name, args_part = self._extract_partial_call(current_text)
664
-
665
- if func_name is None:
666
- return None
667
-
668
- # Step 1: Send function name (once)
669
- if not self.current_tool_name_sent and func_name:
670
- self.current_tool_name_sent = True
671
- self.prev_tool_call_arr[self.current_tool_id] = {
672
- "name": func_name,
673
- "arguments": {},
674
- }
675
- return DeltaMessage(
676
- tool_calls=[
677
- DeltaToolCall(
678
- index=self.current_tool_id,
679
- type="function",
680
- id=make_tool_call_id(),
681
- function=DeltaFunctionCall(
682
- name=func_name,
683
- arguments="",
684
- ).model_dump(exclude_none=True),
685
- )
686
- ]
687
- )
688
-
689
- # Step 2: Parse and diff arguments
690
- if self.current_tool_name_sent and args_part:
691
- return self._emit_argument_diff(args_part)
692
-
693
- return None
694
-
695
- def _handle_tool_call_end(
696
- self,
697
- current_text: str,
698
- prev_end_count: int,
699
- end_count: int,
700
- start_count: int,
701
- ) -> DeltaMessage | None:
702
- """Handle streaming when one or more tool calls have just completed.
703
-
704
- A single streaming delta can batch several complete tool calls
705
- (``<|tool_call>...<tool_call|><|tool_call>...<tool_call|>``). Every
706
- call whose ``<tool_call|>`` end marker arrived in this delta — i.e.
707
- those with index in ``[prev_end_count, end_count)`` — is drained and
708
- emitted, with one ``DeltaToolCall`` per call in a single
709
- ``DeltaMessage`` (this matches the OpenAI streaming wire format, and
710
- the serving layer iterates over ``delta.tool_calls``).
711
-
712
- Per call:
713
-
714
- * If the function name was already streamed incrementally (the
715
- token-by-token path), only the remaining argument fragment is
716
- flushed as a diff.
717
- * If the call is seen complete for the first time in this delta (the
718
- batched-complete path), the id + name + full arguments JSON are
719
- emitted exactly once.
720
- """
721
- # Parse the complete tool calls using regex for accuracy.
722
- all_matches = self.tool_call_regex.findall(current_text)
723
- if not all_matches:
724
- logger.debug("Tool call end detected but no complete tool call parsed yet.")
725
- return None
726
-
727
- deltas: list[DeltaToolCall] = []
728
- for idx in range(prev_end_count, end_count):
729
- if idx >= len(all_matches):
730
- break
731
- # Ensure the tracking arrays have a slot for this index (defensive;
732
- # Case 2 normally allocates these when the start token arrives).
733
- while len(self.prev_tool_call_arr) <= idx:
734
- self.prev_tool_call_arr.append({})
735
- self.streamed_args_for_tool.append("")
736
-
737
- func_name, args_str = all_matches[idx]
738
- final_args = _parse_gemma4_args(args_str)
739
- final_args_json = json.dumps(final_args, ensure_ascii=False)
740
-
741
- # The name is sent exactly once per call. We track that via the
742
- # per-call entry in prev_tool_call_arr (set either by the middle
743
- # path or by the batched-complete branch below), which is robust
744
- # even when several calls are drained in one delta.
745
- name_already_sent = bool(self.prev_tool_call_arr[idx].get("name"))
746
-
747
- if not name_already_sent:
748
- # Batched-complete call: emit id + name + full arguments once.
749
- self.streamed_args_for_tool[idx] = final_args_json
750
- self.prev_tool_call_arr[idx] = {
751
- "name": func_name,
752
- "arguments": final_args,
753
- }
754
- deltas.append(
755
- DeltaToolCall(
756
- index=idx,
757
- type="function",
758
- id=make_tool_call_id(),
759
- function=DeltaFunctionCall(
760
- name=func_name, arguments=final_args_json
761
- ).model_dump(exclude_none=True),
762
- )
763
- )
764
- else:
765
- # Incrementally-streamed call: flush the remaining argument
766
- # tail that was withheld during the middle phase.
767
- prev_streamed = self.streamed_args_for_tool[idx]
768
- if len(final_args_json) > len(prev_streamed):
769
- diff = final_args_json[len(prev_streamed) :]
770
- self.streamed_args_for_tool[idx] = final_args_json
771
- self.prev_tool_call_arr[idx]["arguments"] = final_args
772
- deltas.append(
773
- DeltaToolCall(
774
- index=idx,
775
- function=DeltaFunctionCall(arguments=diff).model_dump(
776
- exclude_none=True
777
- ),
778
- )
779
- )
780
-
781
- # Advance streaming state past the calls completed in this delta. If a
782
- # further tool call is still being accumulated (start without a
783
- # matching end), point current_tool_id at it so the middle path can
784
- # stream its arguments next; otherwise settle on the last completed
785
- # call.
786
- if start_count > end_count:
787
- self.current_tool_id = end_count
788
- while len(self.prev_tool_call_arr) <= self.current_tool_id:
789
- self.prev_tool_call_arr.append({})
790
- self.streamed_args_for_tool.append("")
791
- self.current_tool_name_sent = bool(
792
- self.prev_tool_call_arr[self.current_tool_id].get("name")
793
- )
794
- else:
795
- self.current_tool_id = end_count - 1
796
- self.current_tool_name_sent = True
797
-
798
- if deltas:
799
- return DeltaMessage(tool_calls=deltas)
800
- return None
801
-
802
- def _emit_argument_diff(self, raw_args_str: str) -> DeltaMessage | None:
803
- """Parse raw Gemma4 arguments, convert to JSON, diff, and emit.
804
-
805
- This is the core of the accumulate-then-parse-then-diff strategy:
806
- 1. Parse ``raw_args_str`` with ``_parse_gemma4_args()``
807
- 2. Convert to JSON string with ``json.dumps()``
808
- 3. Withhold trailing closing characters (``"}``) that may move
809
- as more tokens arrive
810
- 4. Diff against previously streamed JSON and emit only new chars
811
-
812
- **Why withholding is necessary:**
813
-
814
- Gemma4's custom format produces *structurally incomplete* JSON
815
- during streaming. For example, when ``<|"|>Paris`` arrives
816
- without a closing delimiter, ``_parse_gemma4_args`` treats it
817
- as a complete value and produces ``{"location": "Paris"}``. But
818
- when ``, France<|"|>`` arrives next, the JSON becomes
819
- ``{"location": "Paris, France"}``. If we had sent the closing
820
- ``"}`` from the first parse, the concatenated client output
821
- would be ``{"location": "Paris"}France"}``, which is garbage.
822
-
823
- The solution: **never send trailing closing chars during
824
- streaming**. They get flushed by ``_handle_tool_call_end()``
825
- when the ``<tool_call|>`` end marker arrives.
826
-
827
- Args:
828
- raw_args_str: The raw Gemma4 argument text accumulated so far
829
- (without the surrounding ``{`` ``}``).
830
-
831
- Returns:
832
- DeltaMessage with the argument diff, or None if no new content.
833
- """
834
- try:
835
- current_args = _parse_gemma4_args(raw_args_str, partial=True)
836
- except Exception:
837
- logger.debug(
838
- "Could not parse partial Gemma4 args yet: %s",
839
- raw_args_str[:100],
840
- )
841
- return None
842
-
843
- if not current_args:
844
- return None
845
-
846
- current_args_json = json.dumps(current_args, ensure_ascii=False)
847
-
848
- # Withhold trailing closing characters that may shift as more
849
- # tokens arrive. Strip trailing '}', '"', ']' and partial
850
- # STRING_DELIM fragments ('<', '|', '\\', '>') to get the
851
- # "safe prefix".
852
- safe_json = current_args_json
853
- while safe_json and safe_json[-1] in ("}", '"', "]", "<", "|", "\\", ">"):
854
- safe_json = safe_json[:-1]
855
-
856
- prev_streamed = self.streamed_args_for_tool[self.current_tool_id]
857
-
858
- if not safe_json or safe_json == prev_streamed:
859
- return None
860
-
861
- # Use find_common_prefix to handle cases where the value changed
862
- # structurally (e.g., a string grew).
863
- if prev_streamed:
864
- prefix = find_common_prefix(prev_streamed, safe_json)
865
- sent_len = len(prev_streamed)
866
- prefix_len = len(prefix)
867
-
868
- if prefix_len < sent_len:
869
- # Structure changed — we sent too much. Truncate our
870
- # tracking to the common prefix and wait for the final
871
- # flush in _handle_tool_call_end.
872
- self.streamed_args_for_tool[self.current_tool_id] = prefix
873
- return None
874
-
875
- # Stream the new stable portion
876
- diff = safe_json[sent_len:]
877
- else:
878
- # First emission
879
- diff = safe_json
880
-
881
- if diff:
882
- self.streamed_args_for_tool[self.current_tool_id] = safe_json
883
- self.prev_tool_call_arr[self.current_tool_id]["arguments"] = current_args
884
-
885
- return DeltaMessage(
886
- tool_calls=[
887
- DeltaToolCall(
888
- index=self.current_tool_id,
889
- function=DeltaFunctionCall(arguments=diff).model_dump(
890
- exclude_none=True
891
- ),
892
- )
893
- ]
894
- )
895
-
896
- return None