| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from __future__ import annotations |
| 5 | |
| 6 | from collections.abc import Iterable, Sequence |
| 7 | from typing import TYPE_CHECKING, TypedDict |
| 8 | |
| 9 | import regex as re |
| 10 | |
| 11 | from vllm.entrypoints.generate.base.protocol import DeltaMessage |
| 12 | from vllm.logger import init_logger |
| 13 | from vllm.reasoning import ReasoningParser |
| 14 | from vllm.tokenizers import TokenizerLike |
| 15 | |
| 16 | if TYPE_CHECKING: |
| 17 | from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest |
| 18 | from vllm.entrypoints.openai.responses.protocol import ResponsesRequest |
| 19 | |
| 20 | logger = init_logger(__name__) |
| 21 | |
| 22 | # Must match the chat template's own default for ``reasoning_effort``. |
| 23 | DEFAULT_REASONING_EFFORT = "high" |
| 24 | |
| 25 | |
| 26 | class ReasoningDelta(TypedDict): |
| 27 | """One streaming reasoning delta. Either/both fields may be None.""" |
| 28 | |
| 29 | reasoning: str | None |
| 30 | content: str | None |
| 31 | |
| 32 | |
| 33 | # NOTE: mirrored in ``vllm.tool_parsers.hy_v4_tool_parser`` (same pattern as |
| 34 | # ``gemma4_utils``) so neither package depends on the other. |
| 35 | def detect_token_suffix(tokenizer: TokenizerLike) -> str: |
| 36 | """Detect the per-checkpoint suffix used by Hunyuan structural tokens. |
| 37 | |
| 38 | Args: |
| 39 | tokenizer: Tokenizer of the served checkpoint. |
| 40 | |
| 41 | Returns: |
| 42 | The suffix including its leading colon (e.g. ``":6124c78e"``), or ``""`` |
| 43 | when the checkpoint uses unsuffixed tokens. |
| 44 | |
| 45 | Raises: |
| 46 | RuntimeError: The tokenizer declares the structural tokens through |
| 47 | ``model_specific_special_tokens``, which transformers 5 no longer |
| 48 | round-trips. |
| 49 | |
| 50 | """ |
| 51 | init_kwargs = getattr(tokenizer, "init_kwargs", None) or {} |
| 52 | think_begin_as_special = init_kwargs.get("model_specific_special_tokens", {}).get( |
| 53 | "think_begin_token", "" |
| 54 | ) |
| 55 | if think_begin_as_special: |
| 56 | raise RuntimeError( |
| 57 | "This checkpoint declares HYV4 structural tokens (think_begin_token" |
| 58 | "/toolcalls_begin_token/argkey_begin_token) in " |
| 59 | "tokenizer_config.json, which transformers 5 no longer supports. " |
| 60 | "Remove those fields and keep the tokens in the tokenizer's own " |
| 61 | "token definitions so the suffix can be read from the vocab." |
| 62 | ) |
| 63 | |
| 64 | structural_token_re = re.compile( |
| 65 | r"<(?:think|tool_calls|tool_call|arg_key|arg_value)(:[^\s>]+)?>" |
| 66 | ) |
| 67 | for token in tokenizer.get_vocab(): |
| 68 | match = structural_token_re.fullmatch(token) |
| 69 | if match: |
| 70 | return match.group(1) or "" |
| 71 | |
| 72 | return "" |
| 73 | |
| 74 | |
| 75 | class HYV4ReasoningExtractor: |
| 76 | """Reasoning extraction for HYV4, on plain data. |
| 77 | |
| 78 | ``thinking`` selects the mode: |
| 79 | - ``True``: parse ``<think>...</think>``; the start token is injected at the |
| 80 | end of the prompt and is therefore usually absent from the output. |
| 81 | - ``False``: ``no_think`` -- the whole output is content, no reasoning. |
| 82 | """ |
| 83 | |
| 84 | def __init__(self, vocab: dict[str, int], token_suffix: str, thinking: bool = True): |
| 85 | self.thinking = thinking |
| 86 | self.start_token: str = f"<think{token_suffix}>" |
| 87 | self.end_token: str = f"</think{token_suffix}>" |
| 88 | |
| 89 | start_token_id = vocab.get(self.start_token) |
| 90 | end_token_id = vocab.get(self.end_token) |
| 91 | if thinking and (start_token_id is None or end_token_id is None): |
| 92 | raise RuntimeError( |
| 93 | "HYV4 reasoning extractor could not locate think " |
| 94 | "start/end tokens in the tokenizer!" |
| 95 | ) |
| 96 | # In no_think mode the ids are unused (kept for symmetry / debugging). |
| 97 | self.start_token_id: int | None = start_token_id |
| 98 | self.end_token_id: int | None = end_token_id |
| 99 | |
| 100 | def has_reasoning_ended(self, token_ids: Sequence[int]) -> bool: |
| 101 | """Idempotent STATE query: has ``</think>`` been emitted yet? |
| 102 | |
| 103 | Monotonic -- once the end token is present this stays True. Use for |
| 104 | "has reasoning finished, given everything so far". (no_think is always |
| 105 | ended.) Contrast with ``reasoning_ended_in_delta`` (edge detector). |
| 106 | """ |
| 107 | if not self.thinking: |
| 108 | return True |
| 109 | for i in range(len(token_ids) - 1, -1, -1): |
| 110 | if token_ids[i] == self.start_token_id: |
| 111 | return False |
| 112 | if token_ids[i] == self.end_token_id: |
| 113 | return True |
| 114 | return False |
| 115 | |
| 116 | def reasoning_ended_in_delta( |
| 117 | self, token_ids: Sequence[int], delta_ids: Iterable[int] |
| 118 | ) -> bool: |
| 119 | """Edge detector (NOT idempotent): did reasoning end *in this* step? |
| 120 | |
| 121 | True only on the step whose ``delta_ids`` contains ``</think>``. Callers |
| 122 | use it once behind their own "already ended" latch, so it need not be |
| 123 | monotonic; it only checks the new delta (cheaper than scanning the whole |
| 124 | sequence). Contrast with ``has_reasoning_ended`` (cumulative state). |
| 125 | """ |
| 126 | if not self.thinking: |
| 127 | return True |
| 128 | end_token_id = self.end_token_id |
| 129 | if end_token_id is None: |
| 130 | return False |
| 131 | return end_token_id in delta_ids |
| 132 | |
| 133 | def extract_content_ids(self, input_ids: list[int]) -> list[int]: |
| 134 | """Return the token ids after ``</think>`` (all of them in no_think).""" |
| 135 | if not self.thinking: |
| 136 | return input_ids |
| 137 | head = input_ids[: max(0, len(input_ids) - 1)] |
| 138 | if self.end_token_id not in head: |
| 139 | return [] |
| 140 | return input_ids[input_ids.index(self.end_token_id) + 1 :] |
| 141 | |
| 142 | def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: |
| 143 | """Count reasoning tokens (0 in no_think). |
| 144 | |
| 145 | The ``<think>`` start token is injected at the END of the prompt, so it |
| 146 | is normally absent from ``token_ids``. Only skip a leading start token |
| 147 | when it actually appears first (legacy / no prompt-injection); otherwise |
| 148 | count every token up to ``</think>``. |
| 149 | """ |
| 150 | if not self.thinking: |
| 151 | return 0 |
| 152 | start = 1 if (token_ids and token_ids[0] == self.start_token_id) else 0 |
| 153 | try: |
| 154 | return token_ids.index(self.end_token_id, start) - start |
| 155 | except ValueError: |
| 156 | # No end token -- reasoning was truncated; everything counts. |
| 157 | return len(token_ids) - start |
| 158 | |
| 159 | def extract_reasoning(self, model_output: str) -> tuple[str | None, str | None]: |
| 160 | """Split a complete output into ``(reasoning, content)``. |
| 161 | |
| 162 | In no_think mode reasoning is None and the whole output is content. The |
| 163 | start token may be absent (injected in the prompt), so reasoning is |
| 164 | assumed to begin at the start of the output. |
| 165 | """ |
| 166 | if not self.thinking: |
| 167 | return None, model_output |
| 168 | |
| 169 | parts = model_output.partition(self.start_token) |
| 170 | model_output = parts[2] if parts[1] else parts[0] |
| 171 | |
| 172 | if self.end_token not in model_output: |
| 173 | return model_output, None |
| 174 | reasoning, _, content = model_output.partition(self.end_token) |
| 175 | return reasoning, content or None |
| 176 | |
| 177 | def extract_reasoning_streaming( |
| 178 | self, |
| 179 | previous_text: str, |
| 180 | current_text: str, |
| 181 | delta_text: str, |
| 182 | previous_token_ids: Sequence[int], |
| 183 | current_token_ids: Sequence[int], |
| 184 | delta_token_ids: Sequence[int], |
| 185 | ) -> ReasoningDelta | None: |
| 186 | """Classify a streaming delta as reasoning and/or content. |
| 187 | |
| 188 | In no_think mode every delta is content. Otherwise: same logic as vLLM's |
| 189 | generic thinking parser, except the final branch (no start token seen |
| 190 | anywhere) treats output as reasoning-then-content, because HYV4 injects |
| 191 | ``<think>`` in the prompt (absent from the output). |
| 192 | """ |
| 193 | if not self.thinking: |
| 194 | if not delta_text: |
| 195 | return None |
| 196 | return ReasoningDelta(reasoning=None, content=delta_text) |
| 197 | |
| 198 | # Skip a delta that is a single lone start/end special token. |
| 199 | if len(delta_token_ids) == 1 and delta_token_ids[0] in ( |
| 200 | self.start_token_id, |
| 201 | self.end_token_id, |
| 202 | ): |
| 203 | return None |
| 204 | |
| 205 | if self.start_token_id in previous_token_ids: |
| 206 | if self.end_token_id in delta_token_ids: |
| 207 | end_index = delta_text.find(self.end_token) |
| 208 | reasoning = delta_text[:end_index] |
| 209 | content = delta_text[end_index + len(self.end_token) :] |
| 210 | return ReasoningDelta(reasoning=reasoning, content=content or None) |
| 211 | elif self.end_token_id in previous_token_ids: |
| 212 | return ReasoningDelta(reasoning=None, content=delta_text) |
| 213 | else: |
| 214 | return ReasoningDelta(reasoning=delta_text, content=None) |
| 215 | elif self.start_token_id in delta_token_ids: |
| 216 | if self.end_token_id in delta_token_ids: |
| 217 | start_index = delta_text.find(self.start_token) |
| 218 | end_index = delta_text.find(self.end_token) |
| 219 | reasoning = delta_text[start_index + len(self.start_token) : end_index] |
| 220 | content = delta_text[end_index + len(self.end_token) :] |
| 221 | return ReasoningDelta(reasoning=reasoning, content=content or None) |
| 222 | else: |
| 223 | return ReasoningDelta(reasoning=delta_text, content=None) |
| 224 | else: |
| 225 | # HYV4: <think> was injected at the end of the prompt, so it never |
| 226 | # appears in the output. Treat the stream as reasoning until |
| 227 | # </think>, then content. |
| 228 | if self.end_token_id in delta_token_ids: |
| 229 | end_index = delta_text.find(self.end_token) |
| 230 | reasoning = delta_text[:end_index] |
| 231 | content = delta_text[end_index + len(self.end_token) :] |
| 232 | return ReasoningDelta(reasoning=reasoning, content=content or None) |
| 233 | elif self.end_token_id in previous_token_ids: |
| 234 | return ReasoningDelta(reasoning=None, content=delta_text) |
| 235 | else: |
| 236 | return ReasoningDelta(reasoning=delta_text, content=None) |
| 237 | |
| 238 | |
| 239 | def build_reasoning_extractor( |
| 240 | tokenizer: TokenizerLike, *, thinking: bool = True |
| 241 | ) -> HYV4ReasoningExtractor: |
| 242 | return HYV4ReasoningExtractor( |
| 243 | tokenizer.get_vocab(), detect_token_suffix(tokenizer), thinking |
| 244 | ) |
| 245 | |
| 246 | |
| 247 | class HYV4ReasoningParser(ReasoningParser): |
| 248 | """vLLM adapter around :class:`HYV4ReasoningExtractor`. |
| 249 | |
| 250 | Decides ``thinking`` from the request's ``reasoning_effort`` and converts |
| 251 | the extractor's ``ReasoningDelta`` to ``DeltaMessage``. |
| 252 | """ |
| 253 | |
| 254 | def __init__(self, tokenizer: TokenizerLike, *args, **kwargs): |
| 255 | super().__init__(tokenizer, *args, **kwargs) |
| 256 | |
| 257 | # Decide thinking vs no_think from the request; the extractor handles |
| 258 | # both modes internally. Prefer chat_template_kwargs.reasoning_effort, |
| 259 | # then the top-level reasoning_effort. The default must match the chat |
| 260 | # template, which sets reasoning_effort='high' when the request omits |
| 261 | # it -- defaulting to no_think here would return the whole |
| 262 | # chain-of-thought (and a stray ``</think:SUF>``) as content. |
| 263 | chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {} |
| 264 | reasoning_effort = chat_kwargs.get("reasoning_effort", None) |
| 265 | if not reasoning_effort: |
| 266 | reasoning_effort = ( |
| 267 | kwargs.get("reasoning_effort") or DEFAULT_REASONING_EFFORT |
| 268 | ) |
| 269 | logger.debug("reasoning_effort for choosing parser: %s", reasoning_effort) |
| 270 | thinking = reasoning_effort != "no_think" |
| 271 | |
| 272 | self._extractor = build_reasoning_extractor(tokenizer, thinking=thinking) |
| 273 | |
| 274 | # Reported regardless of the per-request thinking mode: ``ReasoningConfig`` |
| 275 | # is engine-wide and needs the checkpoint's delimiters to derive the token |
| 276 | # ids used for thinking budget enforcement. |
| 277 | @property |
| 278 | def reasoning_start_str(self) -> str: |
| 279 | return self._extractor.start_token |
| 280 | |
| 281 | @property |
| 282 | def reasoning_end_str(self) -> str: |
| 283 | return self._extractor.end_token |
| 284 | |
| 285 | def is_reasoning_end(self, input_ids) -> bool: |
| 286 | return self._extractor.has_reasoning_ended(input_ids) |
| 287 | |
| 288 | def is_reasoning_end_streaming(self, input_ids, delta_ids) -> bool: |
| 289 | return self._extractor.reasoning_ended_in_delta(input_ids, delta_ids) |
| 290 | |
| 291 | def extract_content_ids(self, input_ids) -> list[int]: |
| 292 | return self._extractor.extract_content_ids(input_ids) |
| 293 | |
| 294 | def count_reasoning_tokens(self, token_ids) -> int: |
| 295 | return self._extractor.count_reasoning_tokens(token_ids) |
| 296 | |
| 297 | def extract_reasoning( |
| 298 | self, |
| 299 | model_output: str, |
| 300 | request: ChatCompletionRequest | ResponsesRequest, |
| 301 | ) -> tuple[str | None, str | None]: |
| 302 | return self._extractor.extract_reasoning(model_output) |
| 303 | |
| 304 | def extract_reasoning_streaming( |
| 305 | self, |
| 306 | previous_text: str, |
| 307 | current_text: str, |
| 308 | delta_text: str, |
| 309 | previous_token_ids, |
| 310 | current_token_ids, |
| 311 | delta_token_ids, |
| 312 | ) -> DeltaMessage | None: |
| 313 | delta = self._extractor.extract_reasoning_streaming( |
| 314 | previous_text, |
| 315 | current_text, |
| 316 | delta_text, |
| 317 | previous_token_ids, |
| 318 | current_token_ids, |
| 319 | delta_token_ids, |
| 320 | ) |
| 321 | if delta is None: |
| 322 | return None |
| 323 | return DeltaMessage(reasoning=delta["reasoning"], content=delta["content"]) |