@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1711b929b6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 17:15 UTC

[Model] Add Reasoning Parser for Granite Models (#14202)

Signed-off-by: Alex-Brooks <Alex.brooks@ibm.com> Co-authored-by: Joe Runde <joe@joerun.de>

Alex Brooks committed Mar 26, 2025 at 08:28 UTC 1711b929b6fadd02a7d66d936ec6ffd24e4c3b54
8 files changed +730 -3
docs/source/features/reasoning_outputs.md
+6 -1
@@ -4,7 +4,7 @@
4
5 vLLM offers support for reasoning models like [DeepSeek R1](https://huggingface.co/deepseek-ai/DeepSeek-R1), which are designed to generate outputs containing both reasoning steps and final conclusions.
6
7 -Reasoning models return a additional `reasoning_content` field in their outputs, which contains the reasoning steps that led to the final conclusion. This field is not present in the outputs of other models.
7 +Reasoning models return an additional `reasoning_content` field in their outputs, which contains the reasoning steps that led to the final conclusion. This field is not present in the outputs of other models.
8
9 ## Supported Models
10
@@ -14,6 +14,9 @@ vLLM currently supports the following reasoning models:
14 |--------------|-------------|------------------|-------------|
15 | [DeepSeek R1 series](https://huggingface.co/collections/deepseek-ai/deepseek-r1-678e1e131c0169c0bc89728d) | `deepseek_r1` | `guided_json`, `guided_regex` | ❌ |
16 | [QwQ-32B](https://huggingface.co/Qwen/QwQ-32B) | `deepseek_r1` | `guided_json`, `guided_regex` | ✅ |
17 +| [IBM Granite 3.2 language models](https://huggingface.co/collections/ibm-granite/granite-32-language-models-67b3bc8c13508f6d064cff9a) | `granite` | ❌ | ❌ |
18 +
19 +- IBM Granite 3.2 reasoning is disabled by default; to enable it, you must also pass `thinking=True` in your `chat_template_kwargs`.
20
21 ## Quickstart
22
@@ -43,6 +46,7 @@ model = models.data[0].id
46
47 # Round 1
48 messages = [{"role": "user", "content": "9.11 and 9.8, which is greater?"}]
49 +# For granite, add: `extra_body={"chat_template_kwargs": {"thinking": True}}`
50 response = client.chat.completions.create(model=model, messages=messages)
51
52 reasoning_content = response.choices[0].message.reasoning_content
@@ -97,6 +101,7 @@ models = client.models.list()
101 model = models.data[0].id
102
103 messages = [{"role": "user", "content": "9.11 and 9.8, which is greater?"}]
104 +# For granite, add: `extra_body={"chat_template_kwargs": {"thinking": True}}`
105 stream = client.chat.completions.create(model=model,
106 messages=messages,
107 stream=True)
examples/online_serving/openai_chat_completion_with_reasoning.py
+1
@@ -31,6 +31,7 @@ model = models.data[0].id
31
32 # Round 1
33 messages = [{"role": "user", "content": "9.11 and 9.8, which is greater?"}]
34 +# For granite, add: `extra_body={"chat_template_kwargs": {"thinking": True}}`
35 response = client.chat.completions.create(model=model, messages=messages)
36
37 reasoning_content = response.choices[0].message.reasoning_content
examples/online_serving/openai_chat_completion_with_reasoning_streaming.py
+1
@@ -38,6 +38,7 @@ models = client.models.list()
38 model = models.data[0].id
39
40 messages = [{"role": "user", "content": "9.11 and 9.8, which is greater?"}]
41 +# For granite, add: `extra_body={"chat_template_kwargs": {"thinking": True}}`
42 stream = client.chat.completions.create(model=model,
43 messages=messages,
44 stream=True)
tests/entrypoints/openai/reasoning_parsers/test_granite_reasoning_parser.py new
+349
@@ -0,0 +1,349 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +import pytest
3 +from transformers import AutoTokenizer
4 +
5 +from tests.entrypoints.openai.reasoning_parsers.utils import (
6 + DeltaMessage, run_reasoning_extraction)
7 +from vllm.entrypoints.openai.reasoning_parsers import (ReasoningParser,
8 + ReasoningParserManager)
9 +
10 +parser_name = "granite"
11 +START_REASONING = "Here is my thought process:"
12 +START_RESPONSE = "Here is my response:"
13 +
14 +SIMPLE_REASONING = {
15 + "output":
16 + f"{START_REASONING}This is a reasoning section{START_RESPONSE}This is the rest", #noqa: E501
17 + "reasoning_content": "This is a reasoning section",
18 + "content": "This is the rest",
19 +}
20 +COMPLETE_REASONING = {
21 + "output": f"{START_REASONING}This is a reasoning section{START_RESPONSE}",
22 + "reasoning_content": "This is a reasoning section",
23 + "content": None,
24 +}
25 +NO_REASONING = {
26 + "output": "This is content",
27 + "reasoning_content": None,
28 + "content": "This is content",
29 +}
30 +MULTIPLE_LINES = {
31 + "output":
32 + f"{START_REASONING}This\nThat{START_RESPONSE}This is the rest\nThat",
33 + "reasoning_content": "This\nThat",
34 + "content": "This is the rest\nThat",
35 +}
36 +REASONING_WITH_THINK = {
37 + "output":
38 + f"{START_REASONING}This is a reasoning section{START_RESPONSE}This is the rest", #noqa: E501
39 + "reasoning_content": "This is a reasoning section",
40 + "content": "This is the rest",
41 +}
42 +COMPLETE_REASONING_WITH_THINK = {
43 + "output": f"{START_REASONING}This is a reasoning section{START_RESPONSE}",
44 + "reasoning_content": "This is a reasoning section",
45 + "content": None,
46 +}
47 +MULTIPLE_LINES_WITH_THINK = {
48 + "output":
49 + f"{START_REASONING}This\nThat{START_RESPONSE}This is the rest\nThat",
50 + "reasoning_content": "This\nThat",
51 + "content": "This is the rest\nThat",
52 +}
53 +
54 +TEST_CASES = [
55 + pytest.param(
56 + False,
57 + SIMPLE_REASONING,
58 + id="simple_reasoning",
59 + ),
60 + pytest.param(
61 + False,
62 + COMPLETE_REASONING,
63 + id="complete_reasoning",
64 + ),
65 + pytest.param(
66 + False,
67 + NO_REASONING,
68 + id="no_reasoning",
69 + ),
70 + pytest.param(
71 + False,
72 + MULTIPLE_LINES,
73 + id="multiple_lines",
74 + ),
75 + pytest.param(
76 + False,
77 + REASONING_WITH_THINK,
78 + id="reasoning_with_think",
79 + ),
80 + pytest.param(
81 + False,
82 + COMPLETE_REASONING_WITH_THINK,
83 + id="complete_reasoning_with_think",
84 + ),
85 + pytest.param(
86 + False,
87 + MULTIPLE_LINES_WITH_THINK,
88 + id="multiple_lines_with_think",
89 + ),
90 + pytest.param(
91 + True,
92 + SIMPLE_REASONING,
93 + id="simple_reasoning_streaming",
94 + ),
95 + pytest.param(
96 + True,
97 + COMPLETE_REASONING,
98 + id="complete_reasoning_streaming",
99 + ),
100 + pytest.param(
101 + True,
102 + NO_REASONING,
103 + id="no_reasoning_streaming",
104 + ),
105 + pytest.param(
106 + True,
107 + MULTIPLE_LINES,
108 + id="multiple_lines_streaming",
109 + ),
110 + pytest.param(
111 + True,
112 + REASONING_WITH_THINK,
113 + id="reasoning_with_think_streaming",
114 + ),
115 + pytest.param(
116 + True,
117 + COMPLETE_REASONING_WITH_THINK,
118 + id="complete_reasoning_with_think_streaming",
119 + ),
120 + pytest.param(
121 + True,
122 + MULTIPLE_LINES_WITH_THINK,
123 + id="multiple_lines_with_think_streaming",
124 + ),
125 +]
126 +
127 +# Global tokenizer initialization to avoid repeated loading
128 +tokenizer = AutoTokenizer.from_pretrained("facebook/opt-125m")
129 +
130 +
131 +@pytest.mark.parametrize("streaming, param_dict", TEST_CASES)
132 +def test_reasoning(
133 + streaming: bool,
134 + param_dict: dict,
135 +):
136 + output = tokenizer.tokenize(param_dict["output"])
137 + # decode everything to tokens
138 + output_tokens: list[str] = [
139 + tokenizer.convert_tokens_to_string([token]) for token in output
140 + ]
141 + parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(
142 + parser_name)(tokenizer)
143 +
144 + reasoning, content = run_reasoning_extraction(parser,
145 + output_tokens,
146 + streaming=streaming)
147 +
148 + assert reasoning == param_dict["reasoning_content"]
149 + assert content == param_dict["content"]
150 +
151 +
152 +# Additional tests for verifying the correctness of granite streaming; this
153 +# is complicated because granite uses multiple tokens to indicate when thinking
154 +# is starting / when it's starting its response, so skipping special tokens
155 +# is awkward.
156 +
157 +### Handling the start of reasoning
158 +STREAMING_1 = {
159 + "previous_text": None,
160 + "current_text": "Here",
161 + "delta_text": "Here",
162 + "reasoning_content": None,
163 + "content": None,
164 +}
165 +# When we fail, we should give what was previously being silenced first
166 +STREAMING_2 = {
167 + "previous_text": "Here is my thought",
168 + "current_text": "Here is my thought failure",
169 + "delta_text": " failure",
170 + "reasoning_content": None,
171 + "content": "Here is my thought failure",
172 +}
173 +# But then after the first one, we should only add the delta text to content
174 +STREAMING_3 = {
175 + "previous_text": "Here wrong",
176 + "current_text": " words",
177 + "delta_text": " Here wrong words",
178 + "reasoning_content": None,
179 + "content": " words",
180 +}
181 +# But then after the first one, we should only add the delta text to content
182 +STREAMING_4 = {
183 + "previous_text": "Here is my thought",
184 + "current_text": "Here is my thought process:",
185 + "delta_text": " process:",
186 + "reasoning_content": None,
187 + "content": None,
188 +}
189 +# Reasoning started successfully; parse reasoning content
190 +STREAMING_5 = {
191 + "previous_text": "Here is my thought process:",
192 + "current_text": "Here is my thought process: foo",
193 + "delta_text": " foo",
194 + "reasoning_content": " foo",
195 + "content": None,
196 +}
197 +# Response special sequence has started, but not finished.
198 +STREAMING_6 = {
199 + "previous_text": "Here is my thought process: foo",
200 + "current_text": "Here is my thought process: foo Here is",
201 + "delta_text": " Here is",
202 + "reasoning_content": " ",
203 + "content": None,
204 +}
205 +# Response special sequence started, but was broken; the reasoning
206 +# content should be the content that was previously unused.
207 +STREAMING_7 = {
208 + "previous_text": "Here is my thought process: foo Here is",
209 + "current_text": "Here is my thought process: foo Here is Here",
210 + "delta_text": " Here",
211 + "reasoning_content": "Here is ",
212 + "content": None,
213 +}
214 +# Response special sequence is ongoing
215 +STREAMING_8 = {
216 + "previous_text": "Here is my thought process: foo Here is my response:",
217 + "current_text": "Here is my thought process: foo Here is my response: bar",
218 + "delta_text": " bar",
219 + "reasoning_content": None,
220 + "content": " bar",
221 +}
222 +# The delta text has everything; we should be able to correctly parse both
223 +STREAMING_9 = {
224 + "previous_text": None,
225 + "current_text": "Here is my thought process: foo Here is my response: bar",
226 + "delta_text": "Here is my thought process: foo Here is my response: bar",
227 + "reasoning_content": " foo ",
228 + "content": " bar",
229 +}
230 +## The Response is ongoing, and the delta mixes reasoning content / content
231 +STREAMING_10 = {
232 + "previous_text": "Here is my thought process: foo",
233 + "current_text":
234 + "Here is my thought process: foo bar Here is my response: baz",
235 + "delta_text": " bar Here is my response: baz",
236 + "reasoning_content": " bar ",
237 + "content": " baz",
238 +}
239 +# The delta text starts a new substring that might be a response special seq
240 +STREAMING_11 = {
241 + "previous_text":
242 + "Here is my thought process: This is a reasoning section ",
243 + "current_text":
244 + "Here is my thought process: This is a reasoning section Here",
245 + "delta_text": "Here",
246 + "reasoning_content": None,
247 + "content": None,
248 +}
249 +# The delta text is finishing the response special seq
250 +STREAMING_12 = {
251 + "previous_text": "Here is my thought process: foo Here is my response",
252 + "current_text": "Here is my thought process: foo Here is my response:",
253 + "delta_text": ":",
254 + "reasoning_content": None,
255 + "content": None,
256 +}
257 +STREAMING_13 = {
258 + "previous_text": "Here is my thought process: foo Here",
259 + "current_text": "Here is my thought process: foo Here was",
260 + "delta_text": " was",
261 + "reasoning_content": "Here was",
262 + "content": None,
263 +}
264 +
265 +STREAMING_SUBCASES = [
266 + pytest.param(
267 + STREAMING_1,
268 + id="Starting reasoning special sequence",
269 + ),
270 + pytest.param(
271 + STREAMING_2,
272 + id="Unexpected start reasoning sequence",
273 + ),
274 + pytest.param(
275 + STREAMING_3,
276 + id="Continuing unexpected start reasoning sequence",
277 + ),
278 + pytest.param(
279 + STREAMING_4,
280 + id="Only start reasoning sequence and nothing else",
281 + ),
282 + pytest.param(
283 + STREAMING_5,
284 + id="Reasoning content has started",
285 + ),
286 + pytest.param(
287 + STREAMING_6,
288 + id="Response special sequence has started",
289 + ),
290 + pytest.param(
291 + STREAMING_7,
292 + id="Response special sequence reset",
293 + ),
294 + pytest.param(
295 + STREAMING_8,
296 + id="Response text has started",
297 + ),
298 + pytest.param(
299 + STREAMING_9,
300 + id="Delta contains everything",
301 + ),
302 + pytest.param(
303 + STREAMING_10,
304 + id="Delta contains some reasoning and response",
305 + ),
306 + pytest.param(
307 + STREAMING_11,
308 + id="Delta starts response sequence",
309 + ),
310 + pytest.param(
311 + STREAMING_12,
312 + id="Delta finishes response sequence",
313 + ),
314 + pytest.param(
315 + STREAMING_13,
316 + id="Delta breaks potential responise sequence",
317 + ),
318 +]
319 +
320 +
321 +@pytest.mark.parametrize("param_dict", STREAMING_SUBCASES)
322 +def test_streaming_subcases(param_dict):
323 + # Get all of the token IDs
324 + previous_token_ids = tokenizer.encode(
325 + param_dict["previous_text"]
326 + ) if param_dict["previous_text"] is not None else []
327 + current_token_ids = tokenizer.encode(param_dict["current_text"])
328 + delta_token_ids = tokenizer.encode(param_dict["delta_text"])
329 +
330 + parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(
331 + parser_name)(tokenizer)
332 +
333 + response = parser.extract_reasoning_content_streaming(
334 + previous_text=param_dict["previous_text"],
335 + current_text=param_dict["current_text"],
336 + delta_text=param_dict["delta_text"],
337 + previous_token_ids=previous_token_ids,
338 + current_token_ids=current_token_ids,
339 + delta_token_ids=delta_token_ids,
340 + )
341 + # Streaming currently expects at least one of reasoning content / content,
342 + # so the response should return None in that case.
343 + if param_dict["reasoning_content"] is None and param_dict[
344 + "content"] is None:
345 + assert response is None
346 + else:
347 + assert isinstance(response, DeltaMessage)
348 + assert param_dict["reasoning_content"] == response.reasoning_content
349 + assert param_dict["content"] == response.content
vllm/engine/arg_utils.py
+1 -1
@@ -1099,7 +1099,7 @@ class EngineArgs:
1099 parser.add_argument(
1100 "--reasoning-parser",
1101 type=str,
1102 - choices=["deepseek_r1"],
1102 + choices=["deepseek_r1", "granite"],
1103 default=None,
1104 help=
1105 "Select the reasoning parser depending on the model that you're "
vllm/entrypoints/openai/reasoning_parsers/__init__.py
+5 -1
@@ -2,7 +2,11 @@
2
3 from .abs_reasoning_parsers import ReasoningParser, ReasoningParserManager
4 from .deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser
5 +from .granite_reasoning_parser import GraniteReasoningParser
6
7 __all__ = [
7 - "ReasoningParser", "ReasoningParserManager", "DeepSeekR1ReasoningParser"
8 + "ReasoningParser",
9 + "ReasoningParserManager",
10 + "DeepSeekR1ReasoningParser",
11 + "GraniteReasoningParser",
12 ]
vllm/entrypoints/openai/reasoning_parsers/granite_reasoning_parser.py new
+363
@@ -0,0 +1,363 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +import re
4 +from collections.abc import Sequence
5 +from typing import Optional, Union
6 +
7 +from transformers import PreTrainedTokenizerBase
8 +
9 +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
10 + DeltaMessage)
11 +from vllm.entrypoints.openai.reasoning_parsers.abs_reasoning_parsers import (
12 + ReasoningParser, ReasoningParserManager)
13 +from vllm.logger import init_logger
14 +
15 +logger = init_logger(__name__)
16 +
17 +
18 +@ReasoningParserManager.register_module("granite")
19 +class GraniteReasoningParser(ReasoningParser):
20 + """
21 + Reasoning parser for IBM Granite.
22 +
23 + IBM granite models currently use "Here is my thought process:"
24 + and "Here is my response:" to separate its thinking / response outputs.
25 + """
26 +
27 + def __init__(self, tokenizer: PreTrainedTokenizerBase):
28 + super().__init__(tokenizer)
29 +
30 + # NOTE: There have been some observed occurrences of quantized
31 + # instances of the current models using "Here's" instead of "Here is",
32 + # so to be safe, we match on both.
33 + self.think_start_expr = r"(?:Here's|Here is) my thought process:"
34 + self.response_start_expr = r"(?:Here's|Here is) my response:"
35 +
36 + self.reasoning_regex = re.compile(
37 + rf"{self.think_start_expr}(.*?){self.response_start_expr}(.*)",
38 + re.DOTALL)
39 +
40 + self.valid_think_starts = [
41 + "Here's my thought process:", "Here is my thought process:"
42 + ]
43 + self.valid_response_starts = [
44 + "Here's my response:", "Here is my response:"
45 + ]
46 +
47 + # Substrings to match for sequence boundaries on raw text
48 + self.seq_boundary_end = ":"
49 + self.seq_boundary_start = "Here"
50 +
51 + # The longest any thinking / start of response message can be
52 + self.longest_think_start = max(
53 + len(think_start) for think_start in self.valid_think_starts)
54 +
55 + def extract_reasoning_content(
56 + self, model_output: str, request: ChatCompletionRequest
57 + ) -> tuple[Optional[str], Optional[str]]:
58 + """Extract the reasoning content & content sections, respectively.
59 + If the sequence doesn't match what we expect, i.e., the model generates
60 + something else, all content is considered non-reasoning content.
61 +
62 + Args:
63 + model_output (str): Output of the model to be parsed.
64 + request (ChatCompletionReqest): Request being processed.
65 +
66 + Returns:
67 + tuple[Optional[str], Optional[str]]: Tuple pair containing the
68 + reasoning content and non-reasoning content.
69 + """
70 + re_match = self.reasoning_regex.findall(model_output)
71 + if not re_match:
72 + return None, model_output
73 + reasoning_content, response_content = re_match[0]
74 + if not response_content:
75 + return reasoning_content, None
76 + return reasoning_content, response_content
77 +
78 + def extract_reasoning_content_streaming(
79 + self,
80 + previous_text: str,
81 + current_text: str,
82 + delta_text: str,
83 + previous_token_ids: Sequence[int],
84 + current_token_ids: Sequence[int],
85 + delta_token_ids: Sequence[int],
86 + ) -> Union[DeltaMessage, None]:
87 + """Extract the reasoning content / content emitted by granite models;
88 + If the sequence doesn't match what we expect, i.e., the model generates
89 + something else, all content is considered non-reasoning content.
90 +
91 + NOTE: Granite models do not use a special token to start their reasoning
92 + and response sections; instead they have token sequences, e.g.,
93 +
94 + Here is my thought process: Foo Here is my response: Bar
95 +
96 + This increases the complexity of correctly handling streams, since we
97 + need to watch for specific sequences and correctly parse them without
98 + dropping content that is potentially overlapping & spanning multiple
99 + delta messages.
100 +
101 + Args:
102 + previous_text (str): Previous text outside of this delta message.
103 + current_text (str): Previous text + delta text.
104 + delta_text (str): Text to consider and parse content from.
105 + previous_token_ids (Sequence[int]): Token IDs of previous_text.
106 + current_token_ids (Sequence[int]): Token IDs of current_text.
107 + delta_token_ids (Sequence[int]): Token IDs of delta_text.
108 +
109 + Returns:
110 + Union[DeltaMessage, None]
111 + DeltaMessage with either reasoning content or content, or None.
112 + """
113 + reasoning_content, resp_seq_len, content = self._get_content_sections(
114 + current_text)
115 + # Either we haven't finished the start of the reasoning sequence,
116 + # or the model is generating something unexpected.
117 + if not reasoning_content:
118 + delta_message = self._get_delta_message_with_no_reasoning_bounds(
119 + current_text, delta_text)
120 + # We have a start of reasoning message, but have not yet finished
121 + # the start of response sequence.
122 + elif not content:
123 + delta_message = self._get_delta_message_with_no_response_bounds(
124 + current_text, reasoning_content, delta_text)
125 + # We've finished both the start of reasoning and start of response seq.
126 + else:
127 + # This should never happen since we matched on the response
128 + assert resp_seq_len is not None
129 + delta_message = self._get_delta_message_with_both_bounds(
130 + delta_text, reasoning_content, content, current_text,
131 + resp_seq_len)
132 + if not delta_message.content and not delta_message.reasoning_content:
133 + return None
134 + return delta_message
135 +
136 + #### Implementation details of stream parsing for granite models
137 + def _is_reasoning_start_substr(self, text: str) -> bool:
138 + """Check if a text matches one of the possible start reasoning seqs.
139 +
140 + Args:
141 + text (str): Text to check for leading substr.
142 +
143 + Returns:
144 + bool: True if any of the possible reasoning start seqs match.
145 + """
146 + return any(
147 + think_start.startswith(text)
148 + for think_start in self.valid_think_starts)
149 +
150 + def _is_response_start_substr(self, text: str) -> bool:
151 + """Check if a text matches one of the possible start response seqs.
152 +
153 + Args:
154 + text (str): Text to check for leading substr.
155 +
156 + Returns:
157 + bool: True if any of the possible response start seqs match.
158 + """
159 + return any(
160 + response_start.startswith(text)
161 + for response_start in self.valid_response_starts)
162 +
163 + def _get_delta_message_with_no_reasoning_bounds(
164 + self,
165 + current_text: str,
166 + delta_text: str,
167 + ) -> DeltaMessage:
168 + """Parse the delta message when the current text has not yet completed
169 + its start of reasoning sequence.
170 +
171 + Args:
172 + current_text (str): The full previous + delta text.
173 + delta_text (str): Text to consider and parse content from.
174 +
175 + Returns:
176 + DeltaMessage: Message containing the parsed content.
177 + """
178 + prev_longest_length = len(current_text) - len(delta_text)
179 + is_substr = self._is_reasoning_start_substr(current_text)
180 + was_substr = self._is_reasoning_start_substr(
181 + current_text[:prev_longest_length])
182 +
183 + # Check if we just generated something NOT in the special token seq;
184 + # if so, add everything that we previously skipped with this delta
185 + # message and append everything to content in the future.
186 + if was_substr and not is_substr:
187 + return DeltaMessage(
188 + reasoning_content=None,
189 + content=current_text,
190 + )
191 + if is_substr:
192 + # Might still be in the special token sequence; return nothing
193 + return DeltaMessage(reasoning_content=None, content=None)
194 + # Otherwise the sequence has already been broken and we already
195 + # corrected; just return the delta text as normal content.
196 + return DeltaMessage(reasoning_content=None, content=delta_text)
197 +
198 + def _get_delta_message_with_no_response_bounds(
199 + self,
200 + current_text: str,
201 + reasoning_content: str,
202 + delta_text: str,
203 + ) -> DeltaMessage:
204 + """Parse the delta message when the current text has both reasoning
205 + content with no (response) content. NOTE that we may have overlapping
206 + tokens with the start of reasoning / start of response sequences on
207 + either side of the delta text.
208 +
209 + Args:
210 + current_text (str): The full previous + delta text.
211 + reasoning_content (str): reasoning content from current_text.
212 + delta_text (str): Text to consider and parse content from.
213 +
214 + Returns:
215 + DeltaMessage: Message containing the parsed content.
216 + """
217 + # If we have no reasoning content or explicitly end with the start of
218 + # response sequence, we are in transition to the response; need to be
219 + # careful here, since the final token (:) will match the reasoning
220 + # content and fully parse it out; we should not pass the : back.
221 + ends_with_start_response_seq = any(
222 + current_text.endswith(response_start)
223 + for response_start in self.valid_response_starts)
224 + if reasoning_content is None or ends_with_start_response_seq:
225 + return DeltaMessage(reasoning_content=None, content=None)
226 +
227 + # Consider previous / current text only within context of the reasoning
228 + previous_text = reasoning_content[:-len(delta_text)]
229 + current_text = reasoning_content
230 +
231 + # We need to be careful about adding unfinished response sequences;
232 + # Find the place at which we MIGHT be starting a response sequence
233 + prev_idx = previous_text.rfind(self.seq_boundary_start)
234 + delta_idx = delta_text.rfind(self.seq_boundary_start)
235 +
236 + # Check the state of potential start of response substring matches.
237 + prev_was_substr = self._is_response_start_substr(
238 + previous_text[prev_idx:]) if prev_idx >= 0 else False
239 + delta_continues_substr = self._is_response_start_substr(
240 + current_text[prev_idx:]) if prev_idx >= 0 else False
241 + delta_new_substr = self._is_response_start_substr(
242 + delta_text[delta_idx:]) if delta_idx >= 0 else False
243 +
244 + # Delta only contains potential continued response sequence text.
245 + if delta_continues_substr:
246 + return DeltaMessage(reasoning_content=None, content=None)
247 +
248 + if not prev_was_substr:
249 + # Delta may be starting a new response seq but has other text too.
250 + if delta_new_substr:
251 + return DeltaMessage(reasoning_content=delta_text[:delta_idx],
252 + content=None)
253 + # Normal case for most reasoning text (no potential special seqs).
254 + return DeltaMessage(reasoning_content=delta_text, content=None)
255 + # The substring that previously seemed to be a potential response
256 + # seq wasn't one; we need to add the content to the delta message,
257 + # and also slice off the potential response sequence
258 + elif delta_new_substr:
259 + reasoning_content = previous_text[
260 + prev_idx:] + delta_text[:delta_idx]
261 + return DeltaMessage(reasoning_content=reasoning_content,
262 + content=None)
263 + # No new substring yet, and we broke our old one; take the whole delta
264 + return DeltaMessage(
265 + reasoning_content=previous_text[prev_idx:] + delta_text,
266 + content=None,
267 + )
268 +
269 + def _get_delta_message_with_both_bounds(
270 + self,
271 + delta_text: str,
272 + reasoning_content: str,
273 + response_content: str,
274 + current_text: str,
275 + response_seq_len: int,
276 + ) -> DeltaMessage:
277 + """Parse the delta message when the current text has both reasoning
278 + content and normal (response) content.
279 +
280 + Args:
281 + delta_text (str): Text to consider and parse content from.
282 + reasoning_content (str): reasoning content from current_text.
283 + response_content (str): response content from current_text.
284 + current_text (str): The full previous + delta text.
285 + response_seq_len(str): Len of the complete response sequence used.
286 +
287 + Returns:
288 + DeltaMessage: Message containing the parsed content.
289 + """
290 + # Always have content; take length to the end
291 + delta_content = delta_text[-len(response_content):]
292 + reasoning_end_idx = len(delta_text) - (len(response_content) +
293 + response_seq_len)
294 +
295 + if reasoning_end_idx < 0:
296 + delta_reasoning_content = None
297 + else:
298 + # Get the starting offset
299 + start_reasoning_content_idx = len(
300 + reasoning_content) + response_seq_len + len(
301 + response_content) - 1
302 + delta_offset = len(current_text) - len(delta_text)
303 + start_offset = start_reasoning_content_idx - delta_offset
304 + if start_offset < 0:
305 + start_offset = 0
306 + delta_reasoning_content = delta_text[
307 + start_offset:reasoning_end_idx]
308 +
309 + return DeltaMessage(
310 + reasoning_content=delta_reasoning_content,
311 + content=delta_content,
312 + )
313 +
314 + def _get_content_sections(
315 + self, current_text: str
316 + ) -> tuple[Optional[str], Optional[int], Optional[str]]:
317 + """Parse the text to extract the reasoning content / content
318 + if we have them.
319 +
320 + Args:
321 + current_text (str): The full previous + delta text.
322 +
323 + Returns:
324 + tuple[Optional[str], Optional[int], Optional[str]]: Tuple of len 3
325 + containing the reasoning content, the length of the response seq
326 + (if there is one) and the non-reasoning content.
327 + """
328 + current_chunk_start = 0
329 + start_reasoning_content = None
330 + parsed_content = False
331 + delimiter_idxs = [
332 + idx for idx, char in enumerate(current_text)
333 + if char == self.seq_boundary_end
334 + ]
335 +
336 + for current_chunk_end in delimiter_idxs:
337 + current_chunk = current_text[current_chunk_start:current_chunk_end]
338 + # Check to see if the start of reasoning seq if complete
339 + if start_reasoning_content is None:
340 + for think_start in self.valid_think_starts:
341 + if current_chunk == think_start[:-1]:
342 + start_reasoning_content = current_chunk_end + 1
343 + current_chunk_start = current_chunk_end + 1
344 + break
345 +
346 + # Check to see if the start of response seq if complete
347 + elif not parsed_content:
348 + for response_start in self.valid_response_starts:
349 + if current_chunk[-len(response_start) +
350 + 1:] == response_start[:-1]:
351 + # Mark end of reasoning and start response content
352 + # after the start of response sequence.
353 + end_reasoning_content = current_chunk_end - len(
354 + response_start)
355 + reasoning_content = current_text[
356 + start_reasoning_content:end_reasoning_content]
357 + response_content = current_text[current_chunk_end + 1:]
358 + return reasoning_content, len(
359 + response_start), response_content
360 +
361 + if start_reasoning_content and not parsed_content:
362 + return current_text[start_reasoning_content:], None, None
363 + return None, None, None
vllm/model_executor/guided_decoding/reasoner/__init__.py
+4
@@ -19,6 +19,10 @@ def get_reasoner(tokenizer: PreTrainedTokenizer,
19 return None
20 elif reasoning_backend == "deepseek_r1":
21 return DeepSeekReasoner.from_tokenizer(tokenizer)
22 + elif reasoning_backend == "granite":
23 + logger.warning(
24 + "Granite reasoner not yet implemented for structured outputs")
25 + return None
26 else:
27 # Raise a warning for unknown reasoning backend and return None
28 # We cannot raise an error here because some reasoning models