@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f3858d5422
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

[Frontend] [Parser] Migrate Nemotron V3 to streaming parser engine (#45755)

Signed-off-by: Ben Browning <bbrownin@redhat.com>

Ben Browning committed Jun 16, 2026 at 01:31 UTC f3858d5422f0353f4a1f7763b7fd6909a3712e69
12 files changed +768 -169
tests/parser/engine/test_delegating_replay.py
+91 -37
@@ -5,70 +5,124 @@
5 Exercises DelegatingParser in engine-adapter mode to verify that delegated
6 routing produces correct output across chunk sizes.
7 See test_replay.py for tests that target engine parsers directly.
8 +
9 +Parser discovery is automatic: any engine parser in ``registered_adapters``
10 +that has both tool and reasoning adapters and a builder in
11 +``trace_builder._BUILDERS`` is picked up with zero manual wiring.
12 """
13
14 from __future__ import annotations
15
12 -from functools import lru_cache
16 +from typing import NamedTuple
17
18 import pytest
19 from pydantic import TypeAdapter
20
21 from tests.parser.engine.replay_harness import (
22 + MockTokenizer,
23 assert_parse_output,
24 collect_output,
25 make_mock_tokenizer,
26 replay_streaming,
27 )
23 -from tests.parser.engine.trace_builder import build_samples
28 +from tests.parser.engine.trace_builder import _BUILDERS, build_samples
29 from vllm.entrypoints.openai.chat_completion.protocol import (
30 ChatCompletionToolsParam,
31 )
27 -from vllm.parser.abstract_parser import Parser
28 -from vllm.parser.parser_manager import ParserManager
32 +from vllm.parser.abstract_parser import DelegatingParser, Parser
33 +from vllm.parser.engine import registered_adapters as _adapters_mod
34 +from vllm.parser.engine.adapters import (
35 + ParserEngineReasoningAdapter,
36 + ParserEngineToolAdapter,
37 +)
38
39 _TOOLS_VALIDATOR = TypeAdapter(list[ChatCompletionToolsParam])
40
32 -_PAIRINGS: dict[str, tuple[str, str, str]] = {
33 - "engine": ("qwen3_coder", "qwen3", "qwen3"),
34 - "gemma4_engine": ("gemma4", "gemma4", "gemma4"),
35 -}
41 +# ── Pairing discovery ────────────────────────────────────────────────
42 +
43 +
44 +class _PairingInfo(NamedTuple):
45 + parser_cls: type[Parser]
46 + name: str
47 + samples: tuple
48 +
49 +
50 +def _discover_pairings() -> list[_PairingInfo]:
51 + """Discover valid delegating pairings from registered engine adapters.
52 +
53 + Groups tool and reasoning adapters by their engine class, then builds
54 + a DelegatingParser subclass for each engine that has both adapters
55 + and a test builder.
56 + """
57 + bare_tok = MockTokenizer(vocab={}, tokens=[])
58 + engines: dict[type, dict[str, type]] = {}
59 + for obj in vars(_adapters_mod).values():
60 + if not isinstance(obj, type):
61 + continue
62 + if (
63 + issubclass(obj, ParserEngineToolAdapter)
64 + and obj is not ParserEngineToolAdapter
65 + ):
66 + tool_adapter: type[ParserEngineToolAdapter] = obj
67 + engines.setdefault(tool_adapter._parser_engine_cls, {})["tool"] = obj
68 + elif (
69 + issubclass(obj, ParserEngineReasoningAdapter)
70 + and obj is not ParserEngineReasoningAdapter
71 + ):
72 + reasoning_adapter: type[ParserEngineReasoningAdapter] = obj
73 + engines.setdefault(reasoning_adapter._parser_engine_cls, {})[
74 + "reasoning"
75 + ] = obj
76 +
77 + found: list[_PairingInfo] = []
78 + missing_builders: list[str] = []
79 + for engine_cls, adapters in engines.items():
80 + if "tool" not in adapters or "reasoning" not in adapters:
81 + continue
82 + cfg = engine_cls(bare_tok, None).parser_engine_config
83 + if cfg.name not in _BUILDERS:
84 + missing_builders.append(f"{engine_cls.__name__} (config.name={cfg.name!r})")
85 + continue
86 +
87 + parser_cls = type(
88 + f"_Delegating{engine_cls.__name__}",
89 + (DelegatingParser,),
90 + {
91 + "reasoning_parser_cls": adapters["reasoning"],
92 + "tool_parser_cls": adapters["tool"],
93 + },
94 + )
95 + found.append(
96 + _PairingInfo(
97 + parser_cls=parser_cls,
98 + name=cfg.name,
99 + samples=build_samples(cfg.name),
100 + )
101 + )
102 + if missing_builders:
103 + raise RuntimeError(
104 + f"Engine adapters in registered_adapters have no test builder "
105 + f"in trace_builder._BUILDERS: {', '.join(missing_builders)}. "
106 + f"Add a builder to _BUILDERS for each new parser."
107 + )
108 + found.sort(key=lambda p: p.name)
109 + return found
110 +
111 +
112 +_PAIRINGS = _discover_pairings()
113 +
114 +_ALL_SAMPLES = [(p.parser_cls, s) for p in _PAIRINGS for s in p.samples]
115
116 CHUNK_SIZES = [1, 2, 3, 5, 11, 23, None]
117
118
40 -@lru_cache
41 -def _get_delegating_parser_cls(pairings: str) -> type[Parser]:
42 - tool_name, reasoning_name, _ = _PAIRINGS[pairings]
43 - parser_cls = ParserManager.get_parser(
44 - tool_parser_name=tool_name,
45 - reasoning_parser_name=reasoning_name,
46 - enable_auto_tools=True,
47 - )
48 - assert parser_cls is not None
49 - return parser_cls
50 -
51 -
52 -def _pairing_samples() -> list[tuple[str, object]]:
53 - items: list[tuple[str, object]] = []
54 - for pairing_name, (_, _, model) in _PAIRINGS.items():
55 - for sample in build_samples(model):
56 - items.append((pairing_name, sample))
57 - return items
58 -
59 -
60 -_all_pairing_samples = _pairing_samples()
61 -
62 -
119 @pytest.mark.parametrize("chunk_size", CHUNK_SIZES, ids=lambda c: f"chunk={c}")
120 @pytest.mark.parametrize(
65 - "pairings,sample",
66 - _all_pairing_samples,
67 - ids=lambda v: v.id if hasattr(v, "id") else v,
121 + "parser_cls,sample",
122 + _ALL_SAMPLES,
123 + ids=lambda v: v.id if hasattr(v, "id") else "",
124 )
69 -def test_delegating_replay(sample, chunk_size, pairings):
70 - parser_cls = _get_delegating_parser_cls(pairings=pairings)
71 -
125 +def test_delegating_replay(parser_cls, sample, chunk_size):
126 tokenizer = make_mock_tokenizer(sample)
127 validated_tools = (
128 _TOOLS_VALIDATOR.validate_python(sample.tools) if sample.tools else None
tests/parser/engine/test_nemotron_v3.py new
+302
@@ -0,0 +1,302 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Tests for the engine-based Nemotron V3 parser.
4 +
5 +Validates that ``NemotronV3Parser`` correctly handles:
6 +- ``<think>``/``</think>`` reasoning with ``<tool_call>`` XML tool calls
7 + (same format as Qwen3)
8 +- Nemotron-specific reasoning/content swap when ``enable_thinking=False``
9 + or ``force_nonempty_content=True``
10 +"""
11 +
12 +import json
13 +from unittest.mock import MagicMock
14 +
15 +import pytest
16 +
17 +from tests.parser.engine.conftest import make_mock_tokenizer
18 +from tests.parser.engine.streaming_helpers import (
19 + collect_function_name,
20 + collect_tool_arguments,
21 + simulate_tool_streaming,
22 +)
23 +from vllm.entrypoints.openai.chat_completion.protocol import (
24 + ChatCompletionRequest,
25 +)
26 +from vllm.parser.nemotron_v3 import NemotronV3Parser
27 +
28 +_THINK_START_ID = 50
29 +_THINK_END_ID = 51
30 +_TOOL_CALL_ID = 60
31 +_TOOL_CALL_END_ID = 61
32 +_TEXT_ID = 100
33 +
34 +_VOCAB = {
35 + "<think>": _THINK_START_ID,
36 + "</think>": _THINK_END_ID,
37 + "<tool_call>": _TOOL_CALL_ID,
38 + "</tool_call>": _TOOL_CALL_END_ID,
39 +}
40 +
41 +
42 +def _make_request(**chat_template_kwargs):
43 + request = MagicMock(spec=ChatCompletionRequest)
44 + request.tools = []
45 + request.tool_choice = "auto"
46 + request.chat_template_kwargs = chat_template_kwargs or None
47 + return request
48 +
49 +
50 +@pytest.fixture
51 +def parser():
52 + return NemotronV3Parser(make_mock_tokenizer(_VOCAB))
53 +
54 +
55 +class TestNemotronSwap:
56 + def test_enable_thinking_false_swaps(self, parser):
57 + """When enable_thinking=False, model output without think tags
58 + should have reasoning swapped to content."""
59 + text = "The answer is 42."
60 + request = _make_request(enable_thinking=False)
61 + reasoning, content = parser.extract_reasoning(text, request)
62 + assert content == "The answer is 42."
63 + assert reasoning is None
64 +
65 + def test_force_nonempty_content_swaps(self, parser):
66 + """force_nonempty_content=True triggers swap when content empty."""
67 + text = "The answer is 42."
68 + request = _make_request(force_nonempty_content=True)
69 + reasoning, content = parser.extract_reasoning(text, request)
70 + assert content == "The answer is 42."
71 + assert reasoning is None
72 +
73 + def test_no_swap_when_content_exists(self, parser):
74 + """With enable_thinking=False but real </think> giving content,
75 + no swap occurs."""
76 + text = "Some reasoning.</think>Actual content here."
77 + request = _make_request(enable_thinking=False)
78 + reasoning, content = parser.extract_reasoning(text, request)
79 + assert reasoning == "Some reasoning."
80 + assert content == "Actual content here."
81 +
82 + def test_no_swap_when_enable_thinking_true(self, parser):
83 + """Normal thinking mode: no swap, even when content is empty."""
84 + text = "Still thinking..."
85 + request = _make_request(enable_thinking=True)
86 + reasoning, content = parser.extract_reasoning(text, request)
87 + assert reasoning == "Still thinking..."
88 + assert content is None
89 +
90 + def test_no_swap_with_none_request(self, parser):
91 + """Graceful handling when request is None."""
92 + text = "Some text."
93 + reasoning, content = parser.extract_reasoning(text, None)
94 + assert reasoning == "Some text."
95 + assert content is None
96 +
97 + def test_no_swap_with_no_kwargs(self, parser):
98 + """No swap when chat_template_kwargs is absent."""
99 + text = "Some text."
100 + request = _make_request()
101 + reasoning, content = parser.extract_reasoning(text, request)
102 + assert reasoning == "Some text."
103 + assert content is None
104 +
105 + def test_swap_with_whitespace_only_content(self, parser):
106 + """Swap occurs when content is whitespace-only."""
107 + text = "The answer.</think> "
108 + request = _make_request(enable_thinking=False)
109 + reasoning, content = parser.extract_reasoning(text, request)
110 + assert content == "The answer."
111 + assert reasoning == " "
112 +
113 +
114 +class TestNonStreamingToolCalls:
115 + def test_single_tool_call(self, parser):
116 + text = (
117 + "<tool_call>\n"
118 + "<function=get_weather>\n"
119 + "<parameter=city>Tokyo</parameter>\n"
120 + "</function>\n"
121 + "</tool_call>"
122 + )
123 + request = _make_request()
124 + result = parser.extract_tool_calls(text, request)
125 + assert result.tools_called is True
126 + assert len(result.tool_calls) == 1
127 + assert result.tool_calls[0].function.name == "get_weather"
128 + args = json.loads(result.tool_calls[0].function.arguments)
129 + assert args == {"city": "Tokyo"}
130 +
131 + def test_parallel_tool_calls(self, parser):
132 + text = (
133 + "<tool_call>\n"
134 + "<function=get_weather>\n"
135 + "<parameter=city>Tokyo</parameter>\n"
136 + "</function>\n"
137 + "</tool_call>"
138 + "<tool_call>\n"
139 + "<function=get_time>\n"
140 + "<parameter=timezone>Asia/Tokyo</parameter>\n"
141 + "</function>\n"
142 + "</tool_call>"
143 + )
144 + request = _make_request()
145 + result = parser.extract_tool_calls(text, request)
146 + assert result.tools_called is True
147 + assert len(result.tool_calls) == 2
148 + assert result.tool_calls[0].function.name == "get_weather"
149 + assert result.tool_calls[1].function.name == "get_time"
150 +
151 + def test_no_tool_calls(self, parser):
152 + request = _make_request()
153 + result = parser.extract_tool_calls("Hello, how can I help?", request)
154 + assert result.tools_called is False
155 + # Parser starts in REASONING state, so plain text is classified
156 + # as reasoning (not content) when there are no tool calls.
157 + assert result.content is None
158 +
159 +
160 +class TestStreaming:
161 + def test_streaming_tool_calls(self, parser):
162 + request = _make_request()
163 + chunks = [
164 + "<tool_call>\n",
165 + "<function=get_weather>\n",
166 + "<parameter=city>Tokyo",
167 + "</parameter>\n",
168 + "</function>\n",
169 + "</tool_call>",
170 + ]
171 + results = simulate_tool_streaming(parser, request, chunks)
172 + name = collect_function_name(results)
173 + assert name == "get_weather"
174 + args_text = collect_tool_arguments(results)
175 + assert args_text
176 + parsed = json.loads(args_text)
177 + assert parsed == {"city": "Tokyo"}
178 +
179 +
180 +class TestParseDeltaTokenIdFiltering:
181 + """parse_delta must not trigger tool call parsing when <tool_call>
182 + appears as regular text rather than as a special token ID."""
183 +
184 + def test_tool_call_text_in_reasoning_is_not_parsed(self, parser):
185 + """Literal <tool_call> in model reasoning should be content,
186 + not a tool call."""
187 + request = _make_request()
188 +
189 + text = (
190 + "The test uses <tool_call> syntax:\n"
191 + "<tool_call>\n"
192 + "<function=Bash>\n"
193 + "<parameter=command>ls</parameter>\n"
194 + "</function>\n"
195 + "</tool_call>"
196 + )
197 + result = parser.parse_delta(
198 + delta_text=text,
199 + delta_token_ids=[_TEXT_ID] * 6,
200 + request=request,
201 + prompt_token_ids=[],
202 + finished=True,
203 + )
204 +
205 + assert result is not None
206 + assert result.reasoning is not None
207 + assert "<tool_call>" in result.reasoning
208 + assert not result.tool_calls
209 +
210 + def test_special_token_id_still_triggers_tool_call(self, parser):
211 + """When the scanner matches a special token ID, the tool call
212 + must still be parsed correctly."""
213 + request = _make_request()
214 +
215 + parser.parse_delta(
216 + delta_text="Let me check.",
217 + delta_token_ids=[_TEXT_ID, _TEXT_ID, _TEXT_ID],
218 + request=request,
219 + prompt_token_ids=[],
220 + finished=False,
221 + )
222 +
223 + parser.parse_delta(
224 + delta_text="<tool_call>",
225 + delta_token_ids=[_TOOL_CALL_ID],
226 + request=request,
227 + finished=False,
228 + )
229 +
230 + parser.parse_delta(
231 + delta_text=(
232 + "\n<function=get_weather>\n"
233 + "<parameter=city>Tokyo</parameter>\n"
234 + "</function>\n"
235 + ),
236 + delta_token_ids=[_TEXT_ID] * 5,
237 + request=request,
238 + finished=False,
239 + )
240 +
241 + parser.parse_delta(
242 + delta_text="</tool_call>",
243 + delta_token_ids=[_TOOL_CALL_END_ID],
244 + request=request,
245 + finished=True,
246 + )
247 +
248 + assert any(s.name == "get_weather" for s in parser._tool_slots)
249 +
250 + def test_text_discussion_then_real_tool_call(self, parser):
251 + """Model discusses tool syntax in reasoning, then makes a real
252 + tool call via special tokens."""
253 + request = _make_request()
254 +
255 + r1 = parser.parse_delta(
256 + delta_text="Use <tool_call> to invoke tools.",
257 + delta_token_ids=[_TEXT_ID] * 6,
258 + request=request,
259 + prompt_token_ids=[],
260 + finished=False,
261 + )
262 +
263 + r2 = parser.parse_delta(
264 + delta_text="</think>",
265 + delta_token_ids=[_THINK_END_ID],
266 + request=request,
267 + finished=False,
268 + )
269 +
270 + r3 = parser.parse_delta(
271 + delta_text="<tool_call>",
272 + delta_token_ids=[_TOOL_CALL_ID],
273 + request=request,
274 + finished=False,
275 + )
276 +
277 + r4 = parser.parse_delta(
278 + delta_text=("\n<function=test>\n<parameter=x>1</parameter>\n</function>\n"),
279 + delta_token_ids=[_TEXT_ID] * 4,
280 + request=request,
281 + finished=False,
282 + )
283 +
284 + r5 = parser.parse_delta(
285 + delta_text="</tool_call>",
286 + delta_token_ids=[_TOOL_CALL_END_ID],
287 + request=request,
288 + finished=True,
289 + )
290 +
291 + results = [r1, r2, r3, r4, r5]
292 + reasoning = "".join(r.reasoning for r in results if r and r.reasoning)
293 + assert "<tool_call>" in reasoning
294 +
295 + names = [
296 + tc.function.name
297 + for r in results
298 + if r and r.tool_calls
299 + for tc in r.tool_calls
300 + if tc.function and tc.function.name
301 + ]
302 + assert "test" in names
tests/parser/engine/test_replay.py
+214 -80
@@ -4,13 +4,21 @@
4
5 Replays dynamically built token sequences at different chunk sizes and
6 holdback depths to verify chunk-size invariance and terminal-token hygiene.
7 +
8 +Parser discovery is automatic: any ``ParserEngine`` subclass registered in
9 +``registered_adapters`` that also has a builder in ``trace_builder._BUILDERS``
10 +is picked up with zero manual wiring.
11 """
12
13 from __future__ import annotations
14
15 +import dataclasses
16 +from typing import NamedTuple
17 +
18 import pytest
19
20 from tests.parser.engine.replay_harness import (
21 + MockTokenizer,
22 _test_request,
23 assert_no_terminal_leakage,
24 assert_parse_output,
@@ -19,70 +27,96 @@ from tests.parser.engine.replay_harness import (
27 replay_streaming,
28 replay_with_text_holdback,
29 )
22 -from tests.parser.engine.trace_builder import build_samples
23 -from vllm.parser.abstract_parser import Parser
24 -from vllm.parser.engine.registered_adapters import (
25 - Gemma4Parser,
26 - Qwen3Parser,
27 -)
30 +from tests.parser.engine.trace_builder import _BUILDERS, build_samples
31 +from vllm.parser.engine import registered_adapters as _adapters_mod
32 +from vllm.parser.engine.parser_engine import ParserEngine
33
29 -_ENGINE_PARSERS: dict[str, type[Parser]] = {
30 - "qwen3_engine": Qwen3Parser,
31 - "gemma4_engine": Gemma4Parser,
32 -}
34 +# ── Parser discovery ─────────────────────────────────────────────────
35
34 -_gemma4_samples = build_samples("gemma4")
35 -_qwen3_samples = build_samples("qwen3")
36
37 -_GEMMA4_TERMINALS = ["<|channel>", "<channel|>", "<|tool_call>", "<tool_call|>"]
37 +class _ParserInfo(NamedTuple):
38 + parser_cls: type[ParserEngine]
39 + name: str
40 + samples: tuple
41 + terminals: list[str]
42 + tool_end: str
43 + think_end: str
44 + tool_start: str
45
39 -_QWEN3_TERMINALS = [
40 - "<think>",
41 - "</think>",
42 - "<tool_call>",
43 - "</tool_call>",
44 - "<function=",
45 - "</function>",
46 -]
46
48 -HOLDBACK_CONFIGS = [6, 12, 24]
47 +def _discover_parsers() -> list[_ParserInfo]:
48 + """Discover engine parsers from registered_adapters that have test builders.
49
50 + Returns one ``_ParserInfo`` per parser, sorted by config name.
51 + Raises ``RuntimeError`` if any registered parser lacks a builder.
52 + """
53 + bare_tok = MockTokenizer(vocab={}, tokens=[])
54 + found: list[_ParserInfo] = []
55 + missing_builders: list[str] = []
56 + for obj in vars(_adapters_mod).values():
57 + if not (
58 + isinstance(obj, type)
59 + and issubclass(obj, ParserEngine)
60 + and obj is not ParserEngine
61 + ):
62 + continue
63 + cfg = obj(bare_tok, None).parser_engine_config
64 + if cfg.name not in _BUILDERS:
65 + missing_builders.append(f"{obj.__name__} (config.name={cfg.name!r})")
66 + continue
67 + tool_end = cfg.token_id_terminals.get("TOOL_END")
68 + if not tool_end:
69 + raise RuntimeError(
70 + f"{obj.__name__} config missing 'TOOL_END' in token_id_terminals"
71 + )
72 + all_vals = set(cfg.terminals.values()) | set(cfg.token_id_terminals.values())
73 + found.append(
74 + _ParserInfo(
75 + parser_cls=obj,
76 + name=cfg.name,
77 + samples=build_samples(cfg.name),
78 + terminals=sorted(v for v in all_vals if len(v) > 1),
79 + tool_end=tool_end,
80 + think_end=cfg.terminals.get("THINK_END", ""),
81 + tool_start=cfg.terminals.get("TOOL_START", ""),
82 + )
83 + )
84 + if missing_builders:
85 + raise RuntimeError(
86 + f"Engine parsers in registered_adapters have no test builder "
87 + f"in trace_builder._BUILDERS: {', '.join(missing_builders)}. "
88 + f"Add a builder to _BUILDERS for each new parser."
89 + )
90 + found.sort(key=lambda p: p.name)
91 + return found
92
51 -@pytest.mark.parametrize("holdback", HOLDBACK_CONFIGS, ids=lambda h: f"holdback{h}")
52 -@pytest.mark.parametrize("chunk_size", [5, 10], ids=lambda c: f"chunk{c}")
53 -@pytest.mark.parametrize("sample", _qwen3_samples, ids=lambda s: s.id)
54 -class TestQwen3ReplayWithHoldback:
55 - """Replay Qwen3 with simulated detokenizer holdback."""
93
57 - def test_replay(self, sample, chunk_size, holdback):
58 - tokenizer = make_mock_tokenizer(sample)
59 - parser = Qwen3Parser(tokenizer, sample.tools)
60 - deltas = replay_streaming(
61 - parser,
62 - sample.tokens,
63 - chunk_size=chunk_size,
64 - holdback_chars=holdback,
65 - prompt_token_ids=sample.prompt_token_ids,
66 - )
67 - output = collect_output(deltas)
94 +_PARSERS = _discover_parsers()
95
69 - assert_parse_output(output, sample)
70 - assert_no_terminal_leakage(
71 - output,
72 - _QWEN3_TERMINALS,
73 - context=f"chunk_size={chunk_size}, holdback={holdback}",
74 - )
96 +_ENGINE_PARSERS: dict[str, type[ParserEngine]] = {
97 + f"{p.name}_engine": p.parser_cls for p in _PARSERS
98 +}
99 +
100 +# ── Parametrize sample lists ─────────────────────────────────────────
101 +
102 +HOLDBACK_CONFIGS = [6, 12, 24]
103 +
104 +_REPLAY_SAMPLES = [(p.parser_cls, s, p.terminals) for p in _PARSERS for s in p.samples]
105
106
107 @pytest.mark.parametrize("holdback", HOLDBACK_CONFIGS, ids=lambda h: f"holdback{h}")
108 @pytest.mark.parametrize("chunk_size", [3, 5, 10], ids=lambda c: f"chunk{c}")
79 -@pytest.mark.parametrize("sample", _gemma4_samples, ids=lambda s: s.id)
80 -class TestGemma4ReplayWithHoldback:
81 - """Replay with simulated detokenizer holdback."""
109 +@pytest.mark.parametrize(
110 + "parser_cls,sample,terminals",
111 + _REPLAY_SAMPLES,
112 + ids=lambda v: v.id if hasattr(v, "id") else "",
113 +)
114 +class TestReplayWithHoldback:
115 + """Replay all parsers with simulated detokenizer holdback."""
116
83 - def test_replay(self, sample, chunk_size, holdback):
117 + def test_replay(self, parser_cls, sample, terminals, chunk_size, holdback):
118 tokenizer = make_mock_tokenizer(sample)
85 - parser = Gemma4Parser(tokenizer, sample.tools)
119 + parser = parser_cls(tokenizer, sample.tools)
120 deltas = replay_streaming(
121 parser,
122 sample.tokens,
@@ -95,7 +129,7 @@ class TestGemma4ReplayWithHoldback:
129 assert_parse_output(output, sample)
130 assert_no_terminal_leakage(
131 output,
98 - _GEMMA4_TERMINALS,
132 + terminals,
133 context=f"chunk_size={chunk_size}, holdback={holdback}",
134 )
135
@@ -104,8 +138,12 @@ TEXT_HOLDBACK_DELAYS = [1, 2, 3]
138
139
140 @pytest.mark.parametrize("delay", TEXT_HOLDBACK_DELAYS, ids=lambda d: f"delay{d}")
107 -@pytest.mark.parametrize("sample", _gemma4_samples, ids=lambda s: s.id)
108 -class TestGemma4TextHoldback:
141 +@pytest.mark.parametrize(
142 + "parser_cls,sample,terminals",
143 + _REPLAY_SAMPLES,
144 + ids=lambda v: v.id if hasattr(v, "id") else "",
145 +)
146 +class TestTextHoldback:
147 """Replay with production-like text/token-ID misalignment.
148
149 In production the detokenizer sends token IDs immediately but holds
@@ -113,9 +151,9 @@ class TestGemma4TextHoldback:
151 terminal path that aligned-holdback tests do not cover.
152 """
153
116 - def test_replay(self, sample, delay):
154 + def test_replay(self, parser_cls, sample, terminals, delay):
155 tokenizer = make_mock_tokenizer(sample)
118 - parser = Gemma4Parser(tokenizer, sample.tools)
156 + parser = parser_cls(tokenizer, sample.tools)
157 deltas = replay_with_text_holdback(
158 parser,
159 sample.tokens,
@@ -127,18 +165,114 @@ class TestGemma4TextHoldback:
165 assert_parse_output(output, sample)
166 assert_no_terminal_leakage(
167 output,
130 - _GEMMA4_TERMINALS,
168 + terminals,
169 context=f"text_delay={delay}",
170 )
171
172
173 +@pytest.mark.parametrize(
174 + "chunk_size", [1, 2, 3, 5, 10, 19, 20, None], ids=lambda c: f"chunk{c}"
175 +)
176 +@pytest.mark.parametrize(
177 + "parser_cls,sample,terminals",
178 + _REPLAY_SAMPLES,
179 + ids=lambda v: v.id if hasattr(v, "id") else "",
180 +)
181 +class TestReplay:
182 + """Replay all parsers at varied chunk sizes without holdback."""
183 +
184 + def test_replay(self, parser_cls, sample, terminals, chunk_size):
185 + tokenizer = make_mock_tokenizer(sample)
186 + parser = parser_cls(tokenizer, sample.tools)
187 + deltas = replay_streaming(
188 + parser,
189 + sample.tokens,
190 + chunk_size=chunk_size,
191 + prompt_token_ids=sample.prompt_token_ids,
192 + )
193 + output = collect_output(deltas)
194 +
195 + assert_parse_output(output, sample)
196 + assert_no_terminal_leakage(output, terminals)
197 +
198 +
199 +_DEFERRAL_SAMPLES = [
200 + (p.parser_cls, s, p.tool_end)
201 + for p in _PARSERS
202 + for s in p.samples
203 + if s.expected_tool_calls
204 +]
205 +
206 +
207 +@pytest.mark.parametrize(
208 + "parser_cls,sample,tool_end_text",
209 + _DEFERRAL_SAMPLES,
210 + ids=lambda v: v.id if hasattr(v, "id") else getattr(v, "__name__", ""),
211 +)
212 +class TestDeferralFinish:
213 + """Test that parse_delta(finished=True) resolves deferred scanner state.
214 +
215 + Simulates a production failure where delta_text is missing the
216 + tool-call-end text but delta_token_ids has the token, causing the
217 + scanner to defer it. Without finish(), the deferred state is lost
218 + and tool call arguments are empty.
219 + """
220 +
221 + def test_misaligned_last_delta_with_finish(self, parser_cls, sample, tool_end_text):
222 + tokenizer = make_mock_tokenizer(sample)
223 + parser = parser_cls(tokenizer, sample.tools)
224 +
225 + request = _test_request()
226 +
227 + all_ids = [tid for tid, _ in sample.tokens]
228 + all_texts = [text for _, text in sample.tokens]
229 +
230 + tool_end_id = sample.vocab.get(tool_end_text)
231 + split_idx = None
232 + for i in range(len(all_ids) - 1, -1, -1):
233 + if all_ids[i] == tool_end_id:
234 + split_idx = i
235 + break
236 +
237 + if split_idx is None:
238 + pytest.skip(f"no {tool_end_text} token found")
239 +
240 + first_ids = all_ids[:split_idx]
241 + first_text = "".join(all_texts[:split_idx])
242 +
243 + last_ids = all_ids[split_idx:]
244 + last_text_missing = "".join(all_texts[split_idx:]).replace(tool_end_text, "")
245 +
246 + result1 = parser.parse_delta(
247 + first_text,
248 + first_ids,
249 + request,
250 + prompt_token_ids=[],
251 + finished=False,
252 + )
253 + result2 = parser.parse_delta(
254 + last_text_missing, last_ids, request, finished=True
255 + )
256 +
257 + output = collect_output([result1, result2])
258 +
259 + tool_calls_only = dataclasses.replace(
260 + sample, expected_reasoning=None, expected_content=None
261 + )
262 + assert_parse_output(output, tool_calls_only)
263 +
264 +
265 +@pytest.mark.parametrize(
266 + "parser_cls,sample",
267 + [(p.parser_cls, p.samples[0]) for p in _PARSERS],
268 + ids=[p.name for p in _PARSERS],
269 +)
270 class TestParserEngineAdjustRequest:
271 """Verify ParserEngine and its adapters set skip_special_tokens=False."""
272
138 - def test_adjust_request_disables_skip_special_tokens(self):
139 - sample = _gemma4_samples[0]
273 + def test_adjust_request_disables_skip_special_tokens(self, parser_cls, sample):
274 tokenizer = make_mock_tokenizer(sample)
141 - parser = Gemma4Parser(tokenizer, sample.tools)
275 + parser = parser_cls(tokenizer, sample.tools)
276 request = _test_request()
277 assert request.skip_special_tokens is True
278 adjusted = parser.adjust_request(request)
@@ -146,25 +280,23 @@ class TestParserEngineAdjustRequest:
280
281
282 _TOOL_CALL_SAMPLES = [
149 - (Qwen3Parser, s)
150 - for s in _qwen3_samples
151 - if s.expected_tool_calls and s.expected_reasoning
152 -] + [
153 - (Gemma4Parser, s)
154 - for s in _gemma4_samples
283 + (p.parser_cls, s, p.think_end, p.tool_start)
284 + for p in _PARSERS
285 + for s in p.samples
286 if s.expected_tool_calls and s.expected_reasoning
287 ]
288
289
159 -def _suppressed_expectations(sample) -> tuple[str, str]:
290 +def _suppressed_expectations(
291 + sample, think_end: str, tool_start: str
292 +) -> tuple[str, str]:
293 """Compute expected (reasoning, content) when tools are suppressed.
294
162 - When an explicit reasoning-end delimiter (``</think>``, ``<channel|>``)
163 - is present, reasoning ends there and the tool call block becomes content.
164 - When reasoning ends implicitly (the tool-start token triggers both
165 - REASONING_END and TOOL_CALL_START), reasoning still ends at the tool
166 - start and the raw tool call block becomes content text — only the
167 - structured tool parsing is suppressed, not the reasoning boundary.
295 + When an explicit reasoning-end delimiter is present, reasoning ends
296 + there and the tool call block becomes content. When reasoning ends
297 + implicitly (the tool-start token triggers both REASONING_END and
298 + TOOL_CALL_START), reasoning still ends at the tool start and the raw
299 + tool call block becomes content text.
300 """
301 full_text = "".join(text for _, text in sample.tokens)
302 reasoning = sample.expected_reasoning
@@ -172,12 +304,12 @@ def _suppressed_expectations(sample) -> tuple[str, str]:
304 if idx < 0:
305 return (full_text, "")
306 after_reasoning = full_text[idx + len(reasoning) :]
175 - for delim in ("</think>", "<channel|>"):
176 - pos = after_reasoning.find(delim)
307 + if think_end:
308 + pos = after_reasoning.find(think_end)
309 if pos >= 0:
178 - return (reasoning, after_reasoning[pos + len(delim) :])
179 - for delim in ("<tool_call>",):
180 - pos = after_reasoning.find(delim)
310 + return (reasoning, after_reasoning[pos + len(think_end) :])
311 + if tool_start:
312 + pos = after_reasoning.find(tool_start)
313 if pos >= 0:
314 return (reasoning, after_reasoning[pos:])
315 return (full_text, "")
@@ -193,9 +325,9 @@ _DUMMY_TOOLS = [
325
326 @pytest.mark.parametrize("chunk_size", [1, 5, None], ids=lambda c: f"chunk{c}")
327 @pytest.mark.parametrize(
196 - "parser_cls,sample",
328 + "parser_cls,sample,think_end,tool_start",
329 _TOOL_CALL_SAMPLES,
198 - ids=lambda v: v.id if hasattr(v, "id") else v.__name__,
330 + ids=lambda v: v.id if hasattr(v, "id") else getattr(v, "__name__", ""),
331 )
332 class TestSkipToolParsingReplay:
333 """Replay with skip_tool_parsing=True (tool_choice='none').
@@ -204,7 +336,7 @@ class TestSkipToolParsingReplay:
336 block appears as content text with no tool calls parsed.
337 """
338
207 - def test_replay(self, parser_cls, sample, chunk_size):
339 + def test_replay(self, parser_cls, sample, think_end, tool_start, chunk_size):
340 tokenizer = make_mock_tokenizer(sample)
341 kwargs = {}
342 if sample.chat_template_kwargs:
@@ -238,7 +370,9 @@ class TestSkipToolParsingReplay:
370
371 output = collect_output(results)
372
241 - expected_reasoning, expected_content = _suppressed_expectations(sample)
373 + expected_reasoning, expected_content = _suppressed_expectations(
374 + sample, think_end, tool_start
375 + )
376
377 assert output.reasoning == expected_reasoning, (
378 f"Reasoning mismatch:\n"
tests/parser/engine/trace_builder.py
+12
@@ -30,6 +30,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
30 )
31 from vllm.parser.engine.registered_adapters import (
32 Gemma4Parser,
33 + NemotronV3Parser,
34 Qwen3Parser,
35 )
36
@@ -486,11 +487,22 @@ def _build_gemma4(scenario: Scenario, validate: bool = True) -> Sample:
487 return sample
488
489
490 +def _build_nemotron_v3(scenario: Scenario, validate: bool = True) -> Sample:
491 + return _build_qwen3(
492 + scenario,
493 + name="nemotron_v3",
494 + parser_cls=NemotronV3Parser,
495 + strip_trailing_ws=True,
496 + validate=validate,
497 + )
498 +
499 +
500 # ── Registry and public API ──────────────────────────────────────────
501
502 _BUILDERS: dict[str, Any] = {
503 "qwen3": _build_qwen3,
504 "gemma4": _build_gemma4,
505 + "nemotron_v3": _build_nemotron_v3,
506 }
507
508
tests/reasoning/test_nemotron_v3_reasoning_parser.py
+6 -2
@@ -9,8 +9,8 @@ import regex as re
9 from tests.reasoning.utils import run_reasoning_extraction
10 from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
11 from vllm.parser.abstract_parser import DelegatingParser
12 +from vllm.parser.engine.registered_adapters import NemotronV3ParserReasoningAdapter
13 from vllm.reasoning import ReasoningParser, ReasoningParserManager
13 -from vllm.reasoning.nemotron_v3_reasoning_parser import NemotronV3ReasoningParser
14
15 parser_name = "nemotron_v3"
16
@@ -27,6 +27,7 @@ class FakeNemotronTokenizer:
27 "<think>": 1,
28 "</think>": 2,
29 }
30 + self._inv_vocab = {v: k for k, v in self._vocab.items()}
31 self._pattern = re.compile(r"(<think>|</think>)")
32
33 def get_vocab(self) -> dict[str, int]:
@@ -42,6 +43,9 @@ class FakeNemotronTokenizer:
43 def convert_tokens_to_string(self, tokens: list[str]) -> str:
44 return "".join(tokens)
45
46 + def decode(self, token_ids: list[int]) -> str:
47 + return "".join(self._inv_vocab.get(tid, f"<unk:{tid}>") for tid in token_ids)
48 +
49
50 @pytest.fixture
51 def tokenizer():
@@ -210,7 +214,7 @@ def _token_id(token: str) -> int:
214
215 def _make_reasoning_parser(tokenizer):
216 class _NemotronParser(DelegatingParser):
213 - reasoning_parser_cls = NemotronV3ReasoningParser
217 + reasoning_parser_cls = NemotronV3ParserReasoningAdapter
218 tool_parser_cls = None
219
220 return _NemotronParser(tokenizer)
vllm/parser/engine/adapters.py
+7
@@ -110,6 +110,13 @@ class ParserEngineReasoningAdapter(ReasoningParser):
110 def finish_streaming(self) -> DeltaMessage | None:
111 return self._parser_engine.finish_streaming()
112
113 + def get_streaming_fallback_content(
114 + self,
115 + text: str,
116 + request: ChatCompletionRequest | ResponsesRequest,
117 + ) -> str | None:
118 + return self._parser_engine.get_streaming_fallback_content(text, request)
119 +
120 def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
121 return self._parser_engine.count_reasoning_tokens(token_ids)
122
vllm/parser/engine/parser_engine.py
+7
@@ -519,6 +519,13 @@ class ParserEngine(Parser):
519 return input_ids[i + 1 :]
520 return input_ids
521
522 + def get_streaming_fallback_content(
523 + self,
524 + text: str,
525 + request: ChatCompletionRequest | ResponsesRequest,
526 + ) -> str | None:
527 + return None
528 +
529 def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
530 start_id = self._reasoning_start_token_id
531 end_id = self._reasoning_end_token_id
vllm/parser/engine/registered_adapters.py
+6
@@ -9,6 +9,7 @@ names so that :class:`ReasoningParserManager` and
9
10 from vllm.parser.engine.adapters import make_adapters
11 from vllm.parser.gemma4 import Gemma4Parser
12 +from vllm.parser.nemotron_v3 import NemotronV3Parser
13 from vllm.parser.qwen3 import Qwen3Parser
14
15 (
@@ -16,6 +17,11 @@ from vllm.parser.qwen3 import Qwen3Parser
17 Gemma4ParserToolAdapter,
18 ) = make_adapters(Gemma4Parser)
19
20 +(
21 + NemotronV3ParserReasoningAdapter,
22 + NemotronV3ParserToolAdapter,
23 +) = make_adapters(NemotronV3Parser)
24 +
25 (
26 Qwen3ParserReasoningAdapter,
27 Qwen3ParserToolAdapter,
vllm/parser/nemotron_v3.py new
+113
@@ -0,0 +1,113 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Nemotron V3 parser.
4 +
5 +The Nemotron 3 Super model uses the same tool call and reasoning
6 +format as Qwen3 (``<think>``/``</think>`` + ``<tool_call>`` XML).
7 +This config reuses :func:`qwen3_config` with a distinct name.
8 +
9 +When ``enable_thinking=False`` or ``force_nonempty_content=True`` and
10 +content is empty, reasoning and content are swapped.
11 +"""
12 +
13 +from __future__ import annotations
14 +
15 +import dataclasses
16 +import functools
17 +from typing import TYPE_CHECKING
18 +
19 +from vllm.parser.qwen3 import Qwen3Parser, qwen3_config
20 +
21 +if TYPE_CHECKING:
22 + from vllm.entrypoints.openai.chat_completion.protocol import (
23 + ChatCompletionRequest,
24 + )
25 + from vllm.entrypoints.openai.engine.protocol import DeltaMessage
26 + from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
27 + from vllm.parser.engine.parser_engine import SemanticEvent
28 + from vllm.parser.engine.parser_engine_config import ParserEngineConfig
29 + from vllm.tokenizers import TokenizerLike
30 + from vllm.tool_parsers.abstract_tool_parser import Tool
31 +
32 +
33 +@functools.cache
34 +def nemotron_v3_config(thinking: bool = True) -> ParserEngineConfig:
35 + return dataclasses.replace(
36 + qwen3_config(thinking=thinking),
37 + name="nemotron_v3",
38 + strip_trailing_reasoning_whitespace=True,
39 + )
40 +
41 +
42 +class NemotronV3Parser(Qwen3Parser):
43 + """Nemotron V3 parser: same format as Qwen3, with Nemotron-specific
44 + behavior: when ``enable_thinking=False`` or
45 + ``force_nonempty_content=True`` and content is empty, swaps
46 + reasoning and content.
47 + """
48 +
49 + def __init__(
50 + self,
51 + tokenizer: TokenizerLike,
52 + tools: list[Tool] | None = None,
53 + **kwargs,
54 + ) -> None:
55 + chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {}
56 + thinking = chat_kwargs.get("enable_thinking", True)
57 + super().__init__(
58 + tokenizer,
59 + tools,
60 + parser_engine_config=nemotron_v3_config(thinking=thinking),
61 + **kwargs,
62 + )
63 + self._streamed_reasoning: list[str] = []
64 +
65 + def _reset(self, initial_state=None) -> None:
66 + super()._reset(initial_state=initial_state)
67 + self._streamed_reasoning = []
68 +
69 + def _events_to_delta(
70 + self,
71 + events: list[SemanticEvent],
72 + finished: bool = False,
73 + ) -> DeltaMessage | None:
74 + delta = super()._events_to_delta(events, finished=finished)
75 + if delta is not None and delta.reasoning is not None:
76 + self._streamed_reasoning.append(delta.reasoning)
77 + return delta
78 +
79 + @staticmethod
80 + def _should_force_content(
81 + request: ChatCompletionRequest | ResponsesRequest,
82 + ) -> bool:
83 + chat_template_kwargs = getattr(request, "chat_template_kwargs", None)
84 + return bool(
85 + chat_template_kwargs
86 + and (
87 + chat_template_kwargs.get("enable_thinking") is False
88 + or chat_template_kwargs.get("force_nonempty_content") is True
89 + )
90 + )
91 +
92 + def get_streaming_fallback_content(
93 + self,
94 + text: str,
95 + request: ChatCompletionRequest | ResponsesRequest,
96 + ) -> str | None:
97 + if not self._should_force_content(request):
98 + return None
99 + return "".join(self._streamed_reasoning) or None
100 +
101 + def extract_reasoning(
102 + self,
103 + model_output: str,
104 + request: ChatCompletionRequest | ResponsesRequest,
105 + ) -> tuple[str | None, str | None]:
106 + reasoning, content = super().extract_reasoning(model_output, request)
107 +
108 + if self._should_force_content(request) and (
109 + content is None or not content.strip()
110 + ):
111 + reasoning, content = content, reasoning
112 +
113 + return reasoning, content
vllm/reasoning/__init__.py
+2 -2
@@ -101,8 +101,8 @@ _REASONING_PARSERS_TO_REGISTER = {
101 "MistralReasoningParser",
102 ),
103 "nemotron_v3": (
104 - "nemotron_v3_reasoning_parser",
105 - "NemotronV3ReasoningParser",
104 + "nemotron_v3_engine_reasoning_parser",
105 + "NemotronV3ParserReasoningAdapter",
106 ),
107 "olmo3": (
108 "olmo3_reasoning_parser",
vllm/reasoning/nemotron_v3_engine_reasoning_parser.py new
+8
@@ -0,0 +1,8 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from vllm.parser.engine.registered_adapters import (
5 + NemotronV3ParserReasoningAdapter,
6 +)
7 +
8 +__all__ = ["NemotronV3ParserReasoningAdapter"]
vllm/reasoning/nemotron_v3_reasoning_parser.py deleted
-48
@@ -1,48 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -from vllm.entrypoints.openai.chat_completion.protocol import (
4 - ChatCompletionRequest,
5 -)
6 -from vllm.entrypoints.openai.responses.protocol import (
7 - ResponsesRequest,
8 -)
9 -from vllm.reasoning.deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser
10 -
11 -
12 -class NemotronV3ReasoningParser(DeepSeekR1ReasoningParser):
13 - """
14 - Reasoning parser for Nemotron V3 models.
15 - """
16 -
17 - def _should_force_content(
18 - self, request: ChatCompletionRequest | ResponsesRequest
19 - ) -> bool:
20 - chat_template_kwargs = getattr(request, "chat_template_kwargs", None)
21 - return bool(
22 - chat_template_kwargs
23 - and (
24 - chat_template_kwargs.get("enable_thinking") is False
25 - or chat_template_kwargs.get("force_nonempty_content") is True
26 - )
27 - )
28 -
29 - def extract_reasoning(
30 - self, model_output: str, request: ChatCompletionRequest | ResponsesRequest
31 - ) -> tuple[str | None, str | None]:
32 - reasoning, final_content = super().extract_reasoning(model_output, request)
33 -
34 - if self._should_force_content(request) and (
35 - final_content is None or not final_content.strip()
36 - ):
37 - reasoning, final_content = final_content, reasoning
38 -
39 - return reasoning, final_content
40 -
41 - def get_streaming_fallback_content(
42 - self, text: str, request: ChatCompletionRequest | ResponsesRequest
43 - ) -> str | None:
44 - """Reasoning to duplicate into content on the terminal streaming delta."""
45 - if not self._should_force_content(request):
46 - return None
47 - reasoning, _ = super().extract_reasoning(text, request)
48 - return reasoning