| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from collections.abc import Sequence |
| 5 | from typing import TYPE_CHECKING |
| 6 | |
| 7 | import regex as re |
| 8 | from transformers import PreTrainedTokenizerBase |
| 9 | |
| 10 | from vllm.entrypoints.generate.base.protocol import DeltaMessage |
| 11 | from vllm.reasoning import ReasoningParser |
| 12 | |
| 13 | if TYPE_CHECKING: |
| 14 | from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest |
| 15 | from vllm.entrypoints.openai.responses.protocol import ResponsesRequest |
| 16 | |
| 17 | |
| 18 | class GraniteReasoningParser(ReasoningParser): |
| 19 | """Reasoning parser for IBM Granite. |
| 20 | |
| 21 | IBM granite models currently use "Here is my thought process:" |
| 22 | and "Here is my response:" to separate its thinking / response outputs. |
| 23 | """ |
| 24 | |
| 25 | def __init__(self, tokenizer: PreTrainedTokenizerBase, *args, **kwargs): |
| 26 | super().__init__(tokenizer, *args, **kwargs) |
| 27 | |
| 28 | # NOTE: There have been some observed occurrences of quantized |
| 29 | # instances of the current models using "Here's" instead of "Here is", |
| 30 | # so to be safe, we match on both. |
| 31 | self.think_start_expr = r"(?:Here's|Here is) my thought process:" |
| 32 | self.response_start_expr = r"(?:Here's|Here is) my response:" |
| 33 | |
| 34 | self.reasoning_regex = re.compile( |
| 35 | rf"{self.think_start_expr}(.*?){self.response_start_expr}(.*)", re.DOTALL |
| 36 | ) |
| 37 | |
| 38 | self.valid_think_starts = [ |
| 39 | "Here's my thought process:", |
| 40 | "Here is my thought process:", |
| 41 | ] |
| 42 | self.valid_response_starts = ["Here's my response:", "Here is my response:"] |
| 43 | |
| 44 | # Substrings to match for sequence boundaries on raw text |
| 45 | self.seq_boundary_end = ":" |
| 46 | self.seq_boundary_start = "Here" |
| 47 | |
| 48 | # The longest any thinking / start of response message can be |
| 49 | self.longest_think_start = max( |
| 50 | len(think_start) for think_start in self.valid_think_starts |
| 51 | ) |
| 52 | |
| 53 | def extract_reasoning( |
| 54 | self, model_output: str, request: "ChatCompletionRequest | ResponsesRequest" |
| 55 | ) -> tuple[str | None, str | None]: |
| 56 | """Extract the reasoning content & content sections, respectively. |
| 57 | If the sequence doesn't match what we expect, i.e., the model generates |
| 58 | something else, all content is considered non-reasoning content. |
| 59 | |
| 60 | Args: |
| 61 | model_output (str): Output of the model to be parsed. |
| 62 | request (ChatCompletionRequest): Request being processed. |
| 63 | |
| 64 | Returns: |
| 65 | tuple[Optional[str], Optional[str]]: Tuple pair containing the |
| 66 | reasoning content and non-reasoning content. |
| 67 | |
| 68 | """ |
| 69 | re_match = self.reasoning_regex.findall(model_output) |
| 70 | if not re_match: |
| 71 | return None, model_output |
| 72 | reasoning, response_content = re_match[0] |
| 73 | if not response_content: |
| 74 | return reasoning, None |
| 75 | return reasoning, response_content |
| 76 | |
| 77 | def extract_reasoning_streaming( |
| 78 | self, |
| 79 | previous_text: str, |
| 80 | current_text: str, |
| 81 | delta_text: str, |
| 82 | previous_token_ids: Sequence[int], |
| 83 | current_token_ids: Sequence[int], |
| 84 | delta_token_ids: Sequence[int], |
| 85 | ) -> DeltaMessage | None: |
| 86 | """Extract the reasoning content / content emitted by granite models; |
| 87 | If the sequence doesn't match what we expect, i.e., the model generates |
| 88 | something else, all content is considered non-reasoning content. |
| 89 | |
| 90 | NOTE: Granite models do not use a special token to start their reasoning |
| 91 | and response sections; instead they have token sequences, e.g., |
| 92 | |
| 93 | Here is my thought process: Foo Here is my response: Bar |
| 94 | |
| 95 | This increases the complexity of correctly handling streams, since we |
| 96 | need to watch for specific sequences and correctly parse them without |
| 97 | dropping content that is potentially overlapping & spanning multiple |
| 98 | delta messages. |
| 99 | |
| 100 | Args: |
| 101 | previous_text (str): Previous text outside of this delta message. |
| 102 | current_text (str): Previous text + delta text. |
| 103 | delta_text (str): Text to consider and parse content from. |
| 104 | previous_token_ids (Sequence[int]): Token IDs of previous_text. |
| 105 | current_token_ids (Sequence[int]): Token IDs of current_text. |
| 106 | delta_token_ids (Sequence[int]): Token IDs of delta_text. |
| 107 | |
| 108 | Returns: |
| 109 | Union[DeltaMessage, None] |
| 110 | DeltaMessage with either reasoning content or content, or None. |
| 111 | |
| 112 | """ |
| 113 | reasoning, resp_seq_len, content = self._get_content_sections(current_text) |
| 114 | # Either we haven't finished the start of the reasoning sequence, |
| 115 | # or the model is generating something unexpected. |
| 116 | if not reasoning: |
| 117 | delta_message = self._get_delta_message_with_no_reasoning_bounds( |
| 118 | current_text, delta_text |
| 119 | ) |
| 120 | # We have a start of reasoning message, but have not yet finished |
| 121 | # the start of response sequence. |
| 122 | elif not content: |
| 123 | delta_message = self._get_delta_message_with_no_response_bounds( |
| 124 | current_text, reasoning, delta_text |
| 125 | ) |
| 126 | # We've finished both the start of reasoning and start of response seq. |
| 127 | else: |
| 128 | # This should never happen since we matched on the response |
| 129 | assert resp_seq_len is not None |
| 130 | delta_message = self._get_delta_message_with_both_bounds( |
| 131 | delta_text, reasoning, content, current_text, resp_seq_len |
| 132 | ) |
| 133 | if not delta_message.content and not delta_message.reasoning: |
| 134 | return None |
| 135 | return delta_message |
| 136 | |
| 137 | #### Implementation details of stream parsing for granite models |
| 138 | def _is_reasoning_start_substr(self, text: str) -> bool: |
| 139 | """Check if a text matches one of the possible start reasoning seqs. |
| 140 | |
| 141 | Args: |
| 142 | text (str): Text to check for leading substr. |
| 143 | |
| 144 | Returns: |
| 145 | bool: True if any of the possible reasoning start seqs match. |
| 146 | |
| 147 | """ |
| 148 | return any( |
| 149 | think_start.startswith(text) for think_start in self.valid_think_starts |
| 150 | ) |
| 151 | |
| 152 | def _is_response_start_substr(self, text: str) -> bool: |
| 153 | """Check if a text matches one of the possible start response seqs. |
| 154 | |
| 155 | Args: |
| 156 | text (str): Text to check for leading substr. |
| 157 | |
| 158 | Returns: |
| 159 | bool: True if any of the possible response start seqs match. |
| 160 | |
| 161 | """ |
| 162 | return any( |
| 163 | response_start.startswith(text) |
| 164 | for response_start in self.valid_response_starts |
| 165 | ) |
| 166 | |
| 167 | def _get_delta_message_with_no_reasoning_bounds( |
| 168 | self, |
| 169 | current_text: str, |
| 170 | delta_text: str, |
| 171 | ) -> DeltaMessage: |
| 172 | """Parse the delta message when the current text has not yet completed |
| 173 | its start of reasoning sequence. |
| 174 | |
| 175 | Args: |
| 176 | current_text (str): The full previous + delta text. |
| 177 | delta_text (str): Text to consider and parse content from. |
| 178 | |
| 179 | Returns: |
| 180 | DeltaMessage: Message containing the parsed content. |
| 181 | |
| 182 | """ |
| 183 | prev_longest_length = len(current_text) - len(delta_text) |
| 184 | is_substr = self._is_reasoning_start_substr(current_text) |
| 185 | was_substr = self._is_reasoning_start_substr(current_text[:prev_longest_length]) |
| 186 | |
| 187 | # Check if we just generated something NOT in the special token seq; |
| 188 | # if so, add everything that we previously skipped with this delta |
| 189 | # message and append everything to content in the future. |
| 190 | if was_substr and not is_substr: |
| 191 | return DeltaMessage( |
| 192 | reasoning=None, |
| 193 | content=current_text, |
| 194 | ) |
| 195 | if is_substr: |
| 196 | # Might still be in the special token sequence; return nothing |
| 197 | return DeltaMessage(reasoning=None, content=None) |
| 198 | # Otherwise the sequence has already been broken and we already |
| 199 | # corrected; just return the delta text as normal content. |
| 200 | return DeltaMessage(reasoning=None, content=delta_text) |
| 201 | |
| 202 | def _get_delta_message_with_no_response_bounds( |
| 203 | self, |
| 204 | current_text: str, |
| 205 | reasoning: str, |
| 206 | delta_text: str, |
| 207 | ) -> DeltaMessage: |
| 208 | """Parse the delta message when the current text has both reasoning |
| 209 | content with no (response) content. NOTE that we may have overlapping |
| 210 | tokens with the start of reasoning / start of response sequences on |
| 211 | either side of the delta text. |
| 212 | |
| 213 | Args: |
| 214 | current_text (str): The full previous + delta text. |
| 215 | reasoning (str): reasoning content from current_text. |
| 216 | delta_text (str): Text to consider and parse content from. |
| 217 | |
| 218 | Returns: |
| 219 | DeltaMessage: Message containing the parsed content. |
| 220 | |
| 221 | """ |
| 222 | # If we have no reasoning content or explicitly end with the start of |
| 223 | # response sequence, we are in transition to the response; need to be |
| 224 | # careful here, since the final token (:) will match the reasoning |
| 225 | # content and fully parse it out; we should not pass the : back. |
| 226 | ends_with_start_response_seq = any( |
| 227 | current_text.endswith(response_start) |
| 228 | for response_start in self.valid_response_starts |
| 229 | ) |
| 230 | if reasoning is None or ends_with_start_response_seq: |
| 231 | return DeltaMessage(reasoning=None, content=None) |
| 232 | |
| 233 | # Consider previous / current text only within context of the reasoning |
| 234 | previous_text = reasoning[: -len(delta_text)] |
| 235 | current_text = reasoning |
| 236 | |
| 237 | # We need to be careful about adding unfinished response sequences; |
| 238 | # Find the place at which we MIGHT be starting a response sequence |
| 239 | prev_idx = previous_text.rfind(self.seq_boundary_start) |
| 240 | delta_idx = delta_text.rfind(self.seq_boundary_start) |
| 241 | |
| 242 | # Check the state of potential start of response substring matches. |
| 243 | prev_was_substr = ( |
| 244 | self._is_response_start_substr(previous_text[prev_idx:]) |
| 245 | if prev_idx >= 0 |
| 246 | else False |
| 247 | ) |
| 248 | delta_continues_substr = ( |
| 249 | self._is_response_start_substr(current_text[prev_idx:]) |
| 250 | if prev_idx >= 0 |
| 251 | else False |
| 252 | ) |
| 253 | delta_new_substr = ( |
| 254 | self._is_response_start_substr(delta_text[delta_idx:]) |
| 255 | if delta_idx >= 0 |
| 256 | else False |
| 257 | ) |
| 258 | |
| 259 | # Delta only contains potential continued response sequence text. |
| 260 | if delta_continues_substr: |
| 261 | return DeltaMessage(reasoning=None, content=None) |
| 262 | |
| 263 | if not prev_was_substr: |
| 264 | # Delta may be starting a new response seq but has other text too. |
| 265 | if delta_new_substr: |
| 266 | return DeltaMessage(reasoning=delta_text[:delta_idx], content=None) |
| 267 | # Normal case for most reasoning text (no potential special seqs). |
| 268 | return DeltaMessage(reasoning=delta_text, content=None) |
| 269 | # The substring that previously seemed to be a potential response |
| 270 | # seq wasn't one; we need to add the content to the delta message, |
| 271 | # and also slice off the potential response sequence |
| 272 | elif delta_new_substr: |
| 273 | reasoning = previous_text[prev_idx:] + delta_text[:delta_idx] |
| 274 | return DeltaMessage(reasoning=reasoning, content=None) |
| 275 | # No new substring yet, and we broke our old one; take the whole delta |
| 276 | return DeltaMessage( |
| 277 | reasoning=previous_text[prev_idx:] + delta_text, |
| 278 | content=None, |
| 279 | ) |
| 280 | |
| 281 | def _get_delta_message_with_both_bounds( |
| 282 | self, |
| 283 | delta_text: str, |
| 284 | reasoning: str, |
| 285 | response_content: str, |
| 286 | current_text: str, |
| 287 | response_seq_len: int, |
| 288 | ) -> DeltaMessage: |
| 289 | """Parse the delta message when the current text has both reasoning |
| 290 | content and normal (response) content. |
| 291 | |
| 292 | Args: |
| 293 | delta_text: Text to consider and parse content from. |
| 294 | reasoning: reasoning content from current_text. |
| 295 | response_content: response content from current_text. |
| 296 | current_text: The full previous + delta text. |
| 297 | response_seq_len: Len of the complete response sequence used. |
| 298 | |
| 299 | Returns: |
| 300 | DeltaMessage: Message containing the parsed content. |
| 301 | |
| 302 | """ |
| 303 | # Always have content; take length to the end |
| 304 | delta_content = delta_text[-len(response_content) :] |
| 305 | reasoning_end_idx = len(delta_text) - (len(response_content) + response_seq_len) |
| 306 | |
| 307 | if reasoning_end_idx < 0: |
| 308 | delta_reasoning = None |
| 309 | else: |
| 310 | # Get the starting offset |
| 311 | start_reasoning_idx = ( |
| 312 | len(reasoning) + response_seq_len + len(response_content) - 1 |
| 313 | ) |
| 314 | delta_offset = len(current_text) - len(delta_text) |
| 315 | start_offset = start_reasoning_idx - delta_offset |
| 316 | if start_offset < 0: |
| 317 | start_offset = 0 |
| 318 | delta_reasoning = delta_text[start_offset:reasoning_end_idx] |
| 319 | |
| 320 | return DeltaMessage( |
| 321 | reasoning=delta_reasoning, |
| 322 | content=delta_content, |
| 323 | ) |
| 324 | |
| 325 | def _get_content_sections( |
| 326 | self, current_text: str |
| 327 | ) -> tuple[str | None, int | None, str | None]: |
| 328 | """Parse the text to extract the reasoning content / content |
| 329 | if we have them. |
| 330 | |
| 331 | Args: |
| 332 | current_text (str): The full previous + delta text. |
| 333 | |
| 334 | Returns: |
| 335 | tuple[Optional[str], Optional[int], Optional[str]]: Tuple of len 3 |
| 336 | containing the reasoning content, the length of the response seq |
| 337 | (if there is one) and the non-reasoning content. |
| 338 | |
| 339 | """ |
| 340 | current_chunk_start = 0 |
| 341 | start_reasoning = None |
| 342 | parsed_content = False |
| 343 | delimiter_idxs = [ |
| 344 | idx |
| 345 | for idx, char in enumerate(current_text) |
| 346 | if char == self.seq_boundary_end |
| 347 | ] |
| 348 | |
| 349 | for current_chunk_end in delimiter_idxs: |
| 350 | current_chunk = current_text[current_chunk_start:current_chunk_end] |
| 351 | # Check to see if the start of reasoning seq if complete |
| 352 | if start_reasoning is None: |
| 353 | for think_start in self.valid_think_starts: |
| 354 | if current_chunk == think_start[:-1]: |
| 355 | start_reasoning = current_chunk_end + 1 |
| 356 | current_chunk_start = current_chunk_end + 1 |
| 357 | break |
| 358 | |
| 359 | # Check to see if the start of response seq if complete |
| 360 | elif not parsed_content: |
| 361 | for response_start in self.valid_response_starts: |
| 362 | if current_chunk[-len(response_start) + 1 :] == response_start[:-1]: |
| 363 | # Mark end of reasoning and start response content |
| 364 | # after the start of response sequence. |
| 365 | end_reasoning = current_chunk_end - len(response_start) |
| 366 | reasoning = current_text[start_reasoning:end_reasoning] |
| 367 | response_content = current_text[current_chunk_end + 1 :] |
| 368 | return reasoning, len(response_start), response_content |
| 369 | |
| 370 | if start_reasoning and not parsed_content: |
| 371 | return current_text[start_reasoning:], None, None |
| 372 | return None, None, None |