| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Inkling parser: typed content blocks parsed by a single state machine. |
| 4 | |
| 5 | Inkling output format (every marker is a dedicated special token):: |
| 6 | |
| 7 | <|message_model|><|content_thinking|>...reasoning...<|end_message|> |
| 8 | <|message_model|><|content_text|>...visible text...<|end_message|> |
| 9 | <|message_model|><|content_invoke_tool_json|> |
| 10 | {"name":"get_weather","args":{"city":"SF"}}<|end_message|> |
| 11 | |
| 12 | Blocks are self-describing and may repeat in any order; sampling may |
| 13 | also end a block with the standalone ``<|content_model_end_sampling|>`` |
| 14 | token. The tool-call payload is a single JSON object whose ``name`` is |
| 15 | extracted by the engine's name-from-args path and whose ``args`` object |
| 16 | is carved out of the wrapper by :func:`_inkling_arg_converter`. |
| 17 | |
| 18 | Note the terminal *labels*: ``THINK_START``/``THINK_END`` are what the |
| 19 | engine keys its reasoning plumbing on (``is_reasoning_end``, |
| 20 | ``count_reasoning_tokens``, initial-state seeding), so ``<|end_message|>`` |
| 21 | is labelled ``THINK_END`` here even though it ends every block kind — |
| 22 | the transition table, not the label, carries the semantics. |
| 23 | """ |
| 24 | |
| 25 | from __future__ import annotations |
| 26 | |
| 27 | import functools |
| 28 | import json |
| 29 | from collections.abc import Sequence |
| 30 | from typing import TYPE_CHECKING |
| 31 | |
| 32 | from vllm.entrypoints.generate.base.protocol import ExtractedToolCallInformation |
| 33 | from vllm.parser.engine.events import EventType |
| 34 | from vllm.parser.engine.parser_engine import ParserEngine |
| 35 | from vllm.parser.engine.parser_engine_config import ( |
| 36 | ParserEngineConfig, |
| 37 | ParserState, |
| 38 | Transition, |
| 39 | ) |
| 40 | from vllm.parser.utils import extract_json_member_object |
| 41 | |
| 42 | if TYPE_CHECKING: |
| 43 | from vllm.tokenizers import TokenizerLike |
| 44 | from vllm.tool_parsers.abstract_tool_parser import Tool |
| 45 | |
| 46 | MESSAGE_MODEL = "<|message_model|>" |
| 47 | CONTENT_TEXT = "<|content_text|>" |
| 48 | CONTENT_THINKING = "<|content_thinking|>" |
| 49 | CONTENT_INVOKE_TOOL_JSON = "<|content_invoke_tool_json|>" |
| 50 | CONTENT_INVOKE_TOOL_TEXT = "<|content_invoke_tool_text|>" |
| 51 | CONTENT_TOOL_ERROR = "<|content_tool_error|>" |
| 52 | CONTENT_MODEL_END_SAMPLING = "<|content_model_end_sampling|>" |
| 53 | END_MESSAGE = "<|end_message|>" |
| 54 | |
| 55 | INKLING_SPECIAL_TOKENS = ( |
| 56 | MESSAGE_MODEL, |
| 57 | CONTENT_TEXT, |
| 58 | CONTENT_THINKING, |
| 59 | CONTENT_INVOKE_TOOL_JSON, |
| 60 | CONTENT_INVOKE_TOOL_TEXT, |
| 61 | CONTENT_TOOL_ERROR, |
| 62 | CONTENT_MODEL_END_SAMPLING, |
| 63 | END_MESSAGE, |
| 64 | ) |
| 65 | |
| 66 | |
| 67 | def _inkling_arg_converter(raw_args: str, partial: bool) -> str: |
| 68 | """Carve the ``args`` object out of the tool-call JSON wrapper. |
| 69 | |
| 70 | Why a converter at all: the engine's ``tool_args_json`` machinery |
| 71 | treats the *entire* TOOL_ARGS text as the tool arguments, but Inkling's |
| 72 | payload is the ``{"name":...,"args":{...}}`` wrapper — without a |
| 73 | converter, ``_compute_arg_delta`` streams the wrapper verbatim into |
| 74 | the OpenAI ``arguments`` field (``converter is None -> raw delta``, |
| 75 | unconditionally; ``stream_arg_deltas=False`` does not stop it). |
| 76 | :func:`extract_json_member_object` returns a prefix-stable verbatim |
| 77 | substring, which the engine's argument-delta diffing requires. |
| 78 | """ |
| 79 | span = extract_json_member_object(raw_args, "args") |
| 80 | if span is None: |
| 81 | # No args value yet (streaming) or none at all (treat as empty). |
| 82 | return "" if partial else "{}" |
| 83 | return span |
| 84 | |
| 85 | |
| 86 | @functools.cache |
| 87 | def inkling_config() -> ParserEngineConfig: |
| 88 | terminals = { |
| 89 | "MSG_MODEL": MESSAGE_MODEL, |
| 90 | "TEXT_START": CONTENT_TEXT, |
| 91 | "THINK_START": CONTENT_THINKING, |
| 92 | "THINK_END": END_MESSAGE, |
| 93 | "END_SAMPLING": CONTENT_MODEL_END_SAMPLING, |
| 94 | "TOOL_START": CONTENT_INVOKE_TOOL_JSON, |
| 95 | "TOOL_TEXT": CONTENT_INVOKE_TOOL_TEXT, |
| 96 | "TOOL_ERROR": CONTENT_TOOL_ERROR, |
| 97 | } |
| 98 | transitions: dict[tuple[ParserState, str], Transition] = { |
| 99 | # ── Between blocks / inside a text block ────────────────────── |
| 100 | (ParserState.CONTENT, "MSG_MODEL"): Transition( |
| 101 | ParserState.MESSAGE_HEADER, |
| 102 | (), |
| 103 | ), |
| 104 | # Opening a block that renders as visible content proves no reasoning |
| 105 | # is still open. Confirming that here is what lets DelegatingParser |
| 106 | # hand the following blocks to the tool pass: the reasoning pass only |
| 107 | # leaves its reasoning phase on an explicit reasoning-end event, and a |
| 108 | # response with no thinking block never emits one otherwise. |
| 109 | (ParserState.CONTENT, "TEXT_START"): Transition( |
| 110 | ParserState.CONTENT, |
| 111 | (EventType.REASONING_END,), |
| 112 | ), |
| 113 | (ParserState.CONTENT, "THINK_START"): Transition( |
| 114 | ParserState.REASONING, |
| 115 | (EventType.REASONING_START,), |
| 116 | ), |
| 117 | # A tool block confirms the same boundary, and is the one opener that |
| 118 | # can start a turn with no visible block ahead of it. |
| 119 | (ParserState.CONTENT, "TOOL_START"): Transition( |
| 120 | ParserState.TOOL_ARGS, |
| 121 | (EventType.REASONING_END, EventType.TOOL_CALL_START), |
| 122 | ), |
| 123 | # Raw / error tool blocks render as visible text. |
| 124 | (ParserState.CONTENT, "TOOL_TEXT"): Transition( |
| 125 | ParserState.CONTENT, |
| 126 | (EventType.REASONING_END,), |
| 127 | ), |
| 128 | (ParserState.CONTENT, "TOOL_ERROR"): Transition( |
| 129 | ParserState.CONTENT, |
| 130 | (EventType.REASONING_END,), |
| 131 | ), |
| 132 | # The optional function name between the model-role and content-kind |
| 133 | # markers is metadata, not visible assistant content. |
| 134 | (ParserState.MESSAGE_HEADER, "MSG_MODEL"): Transition( |
| 135 | ParserState.MESSAGE_HEADER, |
| 136 | (), |
| 137 | ), |
| 138 | (ParserState.MESSAGE_HEADER, "TEXT_START"): Transition( |
| 139 | ParserState.CONTENT, |
| 140 | (EventType.REASONING_END,), |
| 141 | ), |
| 142 | (ParserState.MESSAGE_HEADER, "THINK_START"): Transition( |
| 143 | ParserState.REASONING, |
| 144 | (EventType.REASONING_START,), |
| 145 | ), |
| 146 | (ParserState.MESSAGE_HEADER, "TOOL_START"): Transition( |
| 147 | ParserState.TOOL_ARGS, |
| 148 | (EventType.REASONING_END, EventType.TOOL_CALL_START), |
| 149 | ), |
| 150 | (ParserState.MESSAGE_HEADER, "TOOL_TEXT"): Transition( |
| 151 | ParserState.CONTENT, |
| 152 | (EventType.REASONING_END,), |
| 153 | ), |
| 154 | (ParserState.MESSAGE_HEADER, "TOOL_ERROR"): Transition( |
| 155 | ParserState.CONTENT, |
| 156 | (EventType.REASONING_END,), |
| 157 | ), |
| 158 | # ── Inside a thinking block ─────────────────────────────────── |
| 159 | (ParserState.REASONING, "THINK_START"): Transition( |
| 160 | ParserState.REASONING, |
| 161 | (), |
| 162 | ), |
| 163 | # Defensive: tool call opening while a thinking block is still |
| 164 | # unclosed. |
| 165 | (ParserState.REASONING, "TOOL_START"): Transition( |
| 166 | ParserState.TOOL_ARGS, |
| 167 | (EventType.REASONING_END, EventType.TOOL_CALL_START), |
| 168 | ), |
| 169 | } |
| 170 | # A closed tool block returns to CONTENT (Inkling has no section wrapper; |
| 171 | # blocks of any kind may follow), which also keeps the block-kind |
| 172 | # and role tokens out of the engine's tool-terminal set so the |
| 173 | # skip_tool_parsing reasoning pass still classifies reasoning. |
| 174 | # In a reasoning block, THINK_END defers REASONING_END to the next block |
| 175 | # opener so the function-name header is not emitted as content. |
| 176 | for end in ("THINK_END", "END_SAMPLING"): |
| 177 | transitions[(ParserState.CONTENT, end)] = Transition( |
| 178 | ParserState.CONTENT, |
| 179 | (), |
| 180 | ) |
| 181 | transitions[(ParserState.REASONING, end)] = Transition( |
| 182 | ParserState.CONTENT, |
| 183 | () if end == "THINK_END" else (EventType.REASONING_END,), |
| 184 | ) |
| 185 | transitions[(ParserState.TOOL_ARGS, end)] = Transition( |
| 186 | ParserState.CONTENT, |
| 187 | (EventType.TOOL_CALL_END,), |
| 188 | ) |
| 189 | transitions[(ParserState.MESSAGE_HEADER, end)] = Transition( |
| 190 | ParserState.CONTENT, |
| 191 | (EventType.TEXT_CHUNK,), |
| 192 | ) |
| 193 | |
| 194 | return ParserEngineConfig( |
| 195 | name="inkling", |
| 196 | # Normal generation continues after a prompt-prefilled |
| 197 | # `<|message_model|>`. Non-streaming parsing receives only the generated |
| 198 | # suffix, so begin in the corresponding message-header state as well. |
| 199 | initial_state=ParserState.MESSAGE_HEADER, |
| 200 | wait_for_reasoning=True, |
| 201 | terminals=terminals, |
| 202 | # Inkling content-kind markers are the grammar. When the engine is |
| 203 | # used through DelegatingParser, the reasoning pass can hand the tool |
| 204 | # pass reconstructed text whose token-id slice no longer contains the |
| 205 | # content-kind marker that starts the tool block, so keep Inkling on |
| 206 | # the text grammar instead of token-id-only terminal matching. |
| 207 | token_id_terminals={}, |
| 208 | transitions=transitions, |
| 209 | arg_converter=_inkling_arg_converter, |
| 210 | stream_arg_deltas=True, |
| 211 | tool_args_json=True, |
| 212 | strip_trailing_reasoning_whitespace=True, |
| 213 | drop_whitespace_only_content_before_tools=True, |
| 214 | strip_content_whitespace_with_tools=False, |
| 215 | validate_tool_names=False, |
| 216 | ) |
| 217 | |
| 218 | |
| 219 | class InklingParser(ParserEngine): |
| 220 | CONFIG_NAME = "inkling" |
| 221 | |
| 222 | def __init__( |
| 223 | self, |
| 224 | tokenizer: TokenizerLike, |
| 225 | tools: list[Tool] | None = None, |
| 226 | **kwargs, |
| 227 | ) -> None: |
| 228 | kwargs.setdefault("parser_engine_config", inkling_config()) |
| 229 | super().__init__(tokenizer, tools, **kwargs) |
| 230 | |
| 231 | def adjust_initial_state_from_prompt(self, prompt_token_ids: Sequence[int]) -> None: |
| 232 | """Seed the initial parsing state from the prompt tail. |
| 233 | |
| 234 | Mirrors the Rust parser's ``initialize()``: scanning the prompt |
| 235 | backwards, the last relevant special token decides whether generation |
| 236 | continues inside a thinking block, a text block, a model message |
| 237 | header, or between blocks. |
| 238 | """ |
| 239 | vocab = self.vocab |
| 240 | thinking_id = vocab.get(CONTENT_THINKING) |
| 241 | text_id = vocab.get(CONTENT_TEXT) |
| 242 | model_id = vocab.get(MESSAGE_MODEL) |
| 243 | special_ids = {vocab[text] for text in INKLING_SPECIAL_TOKENS if text in vocab} |
| 244 | for token_id in reversed(prompt_token_ids): |
| 245 | if token_id == thinking_id: |
| 246 | self._engine.reset(initial_state=ParserState.REASONING) |
| 247 | self._streaming_initialized = True |
| 248 | return |
| 249 | if token_id == text_id: |
| 250 | self._engine.reset(initial_state=ParserState.CONTENT) |
| 251 | self._streaming_initialized = True |
| 252 | return |
| 253 | if token_id == model_id: |
| 254 | self._engine.reset(initial_state=ParserState.MESSAGE_HEADER) |
| 255 | self._streaming_initialized = True |
| 256 | return |
| 257 | if token_id in special_ids: |
| 258 | break |
| 259 | |
| 260 | def is_reasoning_end(self, input_ids: list[int]) -> bool: |
| 261 | vocab = self.vocab |
| 262 | thinking_id = vocab.get(CONTENT_THINKING) |
| 263 | text_id = vocab.get(CONTENT_TEXT) |
| 264 | model_id = vocab.get(MESSAGE_MODEL) |
| 265 | end_sampling_id = vocab.get(CONTENT_MODEL_END_SAMPLING) |
| 266 | for token_id in reversed(input_ids): |
| 267 | if token_id in (thinking_id, model_id): |
| 268 | return False |
| 269 | if token_id in (text_id, end_sampling_id): |
| 270 | return True |
| 271 | return False |
| 272 | |
| 273 | def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: |
| 274 | vocab = self.vocab |
| 275 | thinking_id = vocab.get(CONTENT_THINKING) |
| 276 | end_ids = { |
| 277 | token_id |
| 278 | for token_id in ( |
| 279 | vocab.get(END_MESSAGE), |
| 280 | vocab.get(CONTENT_MODEL_END_SAMPLING), |
| 281 | ) |
| 282 | if token_id is not None |
| 283 | } |
| 284 | in_reasoning = False |
| 285 | count = 0 |
| 286 | for token_id in token_ids: |
| 287 | if token_id == thinking_id: |
| 288 | in_reasoning = True |
| 289 | continue |
| 290 | if token_id in end_ids: |
| 291 | in_reasoning = False |
| 292 | continue |
| 293 | if in_reasoning: |
| 294 | count += 1 |
| 295 | return count |
| 296 | |
| 297 | def _single_pass_parse( |
| 298 | self, |
| 299 | text: str, |
| 300 | token_ids: Sequence[int], |
| 301 | initial_state: ParserState | None = None, |
| 302 | ) -> tuple[str | None, str | None, ExtractedToolCallInformation]: |
| 303 | reasoning, content, tool_call_info = super()._single_pass_parse( |
| 304 | text, token_ids, initial_state=initial_state |
| 305 | ) |
| 306 | # The engine defers content that follows tool-call events within a |
| 307 | # single pass; Inkling allows text blocks after tool-call blocks, so |
| 308 | # flush the trailing text (matching the Rust unified parser). |
| 309 | if self._deferred_content: |
| 310 | trailing = self._deferred_content |
| 311 | self._deferred_content = "" |
| 312 | content = self._strip_content_whitespace( |
| 313 | (content or "") + trailing, |
| 314 | tool_call_info.tools_called, |
| 315 | ) |
| 316 | tool_call_info = ExtractedToolCallInformation( |
| 317 | tools_called=tool_call_info.tools_called, |
| 318 | tool_calls=tool_call_info.tool_calls, |
| 319 | content=content, |
| 320 | ) |
| 321 | return reasoning, content, tool_call_info |
| 322 | |
| 323 | @staticmethod |
| 324 | def _extract_args_value(parsed: dict) -> str | None: |
| 325 | # Inkling wraps arguments under "args" rather than "arguments". |
| 326 | for key in ("args", "arguments", "parameters"): |
| 327 | if key in parsed: |
| 328 | val = parsed[key] |
| 329 | if isinstance(val, str): |
| 330 | return val |
| 331 | return json.dumps(val, ensure_ascii=False) |
| 332 | return None |