A high-throughput and memory-efficient inference and serving engine for LLMs
[Bugfix][Reasoning] Nemotron V3: surface reasoning as content when thinking is unterminated (#39091)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com>
Andrii Skliar committed
Jun 10, 2026 at 14:58 UTC
4882fd763282799e3570ca45baeca5c14c659e39
3 files changed
+163
-10
tests/reasoning/test_nemotron_v3_reasoning_parser.py
+118
-2
@@ -8,7 +8,9 @@ import regex as re
8
9
from tests.reasoning.utils import run_reasoning_extraction
10
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
11
+from vllm.parser.abstract_parser import DelegatingParser
12
from vllm.reasoning import ReasoningParser, ReasoningParserManager
13
+from vllm.reasoning.nemotron_v3_reasoning_parser import NemotronV3ReasoningParser
14
15
parser_name = "nemotron_v3"
16
@@ -106,7 +108,7 @@ def test_nemotron_v3_reasoning(
108
assert content == param_dict["content"]
109
110
109
-def test_nemotron_v3_without_thinking_returns_content(
111
+def test_nemotron_v3_without_thinking_moves_into_content(
112
tokenizer: FakeNemotronTokenizer,
113
):
114
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
@@ -124,11 +126,13 @@ def test_nemotron_v3_without_thinking_returns_content(
126
streaming=False,
127
)
128
129
+ # No real content followed the reasoning, so the trace is moved into
130
+ # content (reasoning left empty) — matching main's behavior.
131
assert reasoning is None
132
assert content == "This is plain content"
133
134
131
-def test_nemotron_v3_force_nonempty_content_returns_content(
135
+def test_nemotron_v3_force_nonempty_content_moves_into_content(
136
tokenizer: FakeNemotronTokenizer,
137
):
138
parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
@@ -150,6 +154,30 @@ def test_nemotron_v3_force_nonempty_content_returns_content(
154
assert content == "This is plain content"
155
156
157
+def test_nemotron_v3_force_nonempty_keeps_real_content(
158
+ tokenizer: FakeNemotronTokenizer,
159
+):
160
+ # When real content follows the closing tag nothing is promoted: the
161
+ # content after </think> is returned as-is and reasoning stays separate.
162
+ parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name)
163
+ parser = parser_cls(tokenizer)
164
+ request = ChatCompletionRequest(
165
+ model="test-model",
166
+ messages=[],
167
+ chat_template_kwargs={"force_nonempty_content": True},
168
+ )
169
+
170
+ reasoning, content = run_reasoning_extraction(
171
+ parser,
172
+ ["<think>reasoning here</think>real answer"],
173
+ request=request,
174
+ streaming=False,
175
+ )
176
+
177
+ assert reasoning == "reasoning here"
178
+ assert content == "real answer"
179
+
180
+
181
def test_nemotron_v3_with_thinking_keeps_truncated_reasoning(
182
tokenizer: FakeNemotronTokenizer,
183
):
@@ -170,3 +198,91 @@ def test_nemotron_v3_with_thinking_keeps_truncated_reasoning(
198
199
assert reasoning == "This is truncated reasoning"
200
assert content is None
201
+
202
+
203
+_SPECIAL_TOKEN_IDS = {"<think>": 1, "</think>": 2}
204
+
205
+
206
+def _token_id(token: str) -> int:
207
+ # Only the think markers need stable ids; everything else is non-special.
208
+ return _SPECIAL_TOKEN_IDS.get(token, 0)
209
+
210
+
211
+def _make_reasoning_parser(tokenizer):
212
+ class _NemotronParser(DelegatingParser):
213
+ reasoning_parser_cls = NemotronV3ReasoningParser
214
+ tool_parser_cls = None
215
+
216
+ return _NemotronParser(tokenizer)
217
+
218
+
219
+def _run_parse_delta(parser, tokenizer, text, request):
220
+ tokens = tokenizer.tokenize(text)
221
+ reasoning_parts: list[str] = []
222
+ content_parts: list[str] = []
223
+ for i, token in enumerate(tokens):
224
+ delta = parser.parse_delta(
225
+ delta_text=token,
226
+ delta_token_ids=[_token_id(token)],
227
+ request=request,
228
+ prompt_token_ids=[] if i == 0 else None,
229
+ finished=(i == len(tokens) - 1),
230
+ )
231
+ if delta is None:
232
+ continue
233
+ if delta.reasoning:
234
+ reasoning_parts.append(delta.reasoning)
235
+ if delta.content:
236
+ content_parts.append(delta.content)
237
+ return "".join(reasoning_parts), "".join(content_parts)
238
+
239
+
240
+def test_nemotron_v3_streaming_promotes_reasoning_to_content(
241
+ tokenizer: FakeNemotronTokenizer,
242
+):
243
+ # Model never closes <think>: reasoning streams normally AND is duplicated
244
+ # into content on the terminal delta.
245
+ request = ChatCompletionRequest(
246
+ model="test-model",
247
+ messages=[],
248
+ chat_template_kwargs={"force_nonempty_content": True},
249
+ )
250
+ parser = _make_reasoning_parser(tokenizer)
251
+
252
+ reasoning, content = _run_parse_delta(parser, tokenizer, "<think>4", request)
253
+
254
+ assert reasoning == "4"
255
+ assert content == "4"
256
+
257
+
258
+def test_nemotron_v3_streaming_no_promotion_with_real_content(
259
+ tokenizer: FakeNemotronTokenizer,
260
+):
261
+ request = ChatCompletionRequest(
262
+ model="test-model",
263
+ messages=[],
264
+ chat_template_kwargs={"force_nonempty_content": True},
265
+ )
266
+ parser = _make_reasoning_parser(tokenizer)
267
+
268
+ reasoning, content = _run_parse_delta(
269
+ parser, tokenizer, "<think>reason</think>real answer", request
270
+ )
271
+
272
+ # Real content followed </think>, so nothing is duplicated.
273
+ assert reasoning == "reason"
274
+ assert content == "real answer"
275
+
276
+
277
+def test_nemotron_v3_streaming_no_promotion_without_opt_in(
278
+ tokenizer: FakeNemotronTokenizer,
279
+):
280
+ # Without enable_thinking=False / force_nonempty_content the fallback must
281
+ # stay disabled: the response stays reasoning-only, content empty.
282
+ request = ChatCompletionRequest(model="test-model", messages=[])
283
+ parser = _make_reasoning_parser(tokenizer)
284
+
285
+ reasoning, content = _run_parse_delta(parser, tokenizer, "<think>4", request)
286
+
287
+ assert reasoning == "4"
288
+ assert content == ""
vllm/parser/abstract_parser.py
+23
-1
@@ -776,6 +776,28 @@ class DelegatingParser(Parser):
776
last_tc.function.arguments or ""
777
) + self._tool_parser.get_remaining_unstreamed_args()
778
779
+ def finalize_generation(
780
+ self,
781
+ delta_message: DeltaMessage | None,
782
+ request: ChatCompletionRequest | ResponsesRequest,
783
+ state: StreamState,
784
+ ) -> DeltaMessage | None:
785
+ """Finalize generation for cases where generation was incomplete.
786
+ For example, if streaming terminated before reasoning ended
787
+ """
788
+ fallback_fn = getattr(
789
+ self._reasoning_parser, "get_streaming_fallback_content", None
790
+ )
791
+ if fallback_fn is not None and not state.reasoning_ended:
792
+ promoted = fallback_fn(state.previous_text, request)
793
+ if promoted:
794
+ if delta_message is None:
795
+ delta_message = DeltaMessage()
796
+ delta_message.content = (delta_message.content or "") + promoted
797
+
798
+ self._append_unstreamed_tool_args(delta_message)
799
+ return delta_message
800
+
801
def parse(
802
self,
803
model_output: str,
@@ -883,6 +905,6 @@ class DelegatingParser(Parser):
905
state.previous_token_ids = current_token_ids
906
907
if finished:
886
- self._append_unstreamed_tool_args(delta_message)
908
+ delta_message = self.finalize_generation(delta_message, request, state)
909
910
return delta_message
vllm/reasoning/nemotron_v3_reasoning_parser.py
+22
-7
@@ -14,20 +14,35 @@ class NemotronV3ReasoningParser(DeepSeekR1ReasoningParser):
14
Reasoning parser for Nemotron V3 models.
15
"""
16
17
- def extract_reasoning(
18
- self, model_output: str, request: ChatCompletionRequest | ResponsesRequest
19
- ) -> tuple[str | None, str | None]:
20
- reasoning, final_content = super().extract_reasoning(model_output, request)
17
+ def _should_force_content(
18
+ self, request: ChatCompletionRequest | ResponsesRequest
19
+ ) -> bool:
20
chat_template_kwargs = getattr(request, "chat_template_kwargs", None)
22
-
23
- if (
21
+ return bool(
22
chat_template_kwargs
23
and (
24
chat_template_kwargs.get("enable_thinking") is False
25
or chat_template_kwargs.get("force_nonempty_content") is True
26
)
29
- and (final_content is None or not final_content.strip())
27
+ )
28
+
29
+ def extract_reasoning(
30
+ self, model_output: str, request: ChatCompletionRequest | ResponsesRequest
31
+ ) -> tuple[str | None, str | None]:
32
+ reasoning, final_content = super().extract_reasoning(model_output, request)
33
+
34
+ if self._should_force_content(request) and (
35
+ final_content is None or not final_content.strip()
36
):
37
reasoning, final_content = final_content, reasoning
38
39
return reasoning, final_content
40
+
41
+ def get_streaming_fallback_content(
42
+ self, text: str, request: ChatCompletionRequest | ResponsesRequest
43
+ ) -> str | None:
44
+ """Reasoning to duplicate into content on the terminal streaming delta."""
45
+ if not self._should_force_content(request):
46
+ return None
47
+ reasoning, _ = super().extract_reasoning(text, request)
48
+ return reasoning