| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Inkling chat-encoding core. |
| 4 | |
| 5 | Pure implementation of Inkling chat rendering, kept deliberately free of |
| 6 | vLLM imports: it depends only on the :class:`InklingTextTokenizer` protocol |
| 7 | and speaks OpenAI-style message dicts. If a standalone Inkling |
| 8 | input-processing library becomes available, this module is the unit to |
| 9 | swap out — the swap point is marked in ``vllm/renderers/inkling.py``. |
| 10 | |
| 11 | Multimodal parts follow the contract of vLLM's Inkling multimodal processor |
| 12 | (``InklingMultiModalProcessor`` anchors prompt updates on the bare |
| 13 | content-kind marker and inserts the per-patch placeholder run itself): |
| 14 | |
| 15 | * image parts emit only ``<|content_image|>`` — no seed placeholder id; |
| 16 | * audio parts emit ``<|content_audio_input|><|audio_end|>`` — no seed |
| 17 | placeholder id between them. |
| 18 | |
| 19 | ``reasoning_effort`` (a float in [0, 0.99], sourced from |
| 20 | ``chat_template_kwargs`` only) renders the ``Thinking effort level:`` |
| 21 | system control block after the tool declarations and initial system messages. |
| 22 | """ |
| 23 | |
| 24 | from __future__ import annotations |
| 25 | |
| 26 | import json |
| 27 | from collections.abc import Iterator, Mapping, Sequence |
| 28 | from typing import Any, Protocol |
| 29 | |
| 30 | END_OF_TEXT = "<|endoftext|>" |
| 31 | MESSAGE_USER = "<|message_user|>" |
| 32 | MESSAGE_MODEL = "<|message_model|>" |
| 33 | MESSAGE_SYSTEM = "<|message_system|>" |
| 34 | MESSAGE_TOOL = "<|message_tool|>" |
| 35 | CONTENT_TEXT = "<|content_text|>" |
| 36 | CONTENT_IMAGE = "<|content_image|>" |
| 37 | CONTENT_MODEL_END_SAMPLING = "<|content_model_end_sampling|>" |
| 38 | CONTENT_THINKING = "<|content_thinking|>" |
| 39 | CONTENT_AUDIO_INPUT = "<|content_audio_input|>" |
| 40 | CONTENT_TOOL_ERROR = "<|content_tool_error|>" |
| 41 | CONTENT_XML = "<|content_xml|>" |
| 42 | CONTENT_INVOKE_TOOL_JSON = "<|content_invoke_tool_json|>" |
| 43 | CONTENT_INVOKE_TOOL_TEXT = "<|content_invoke_tool_text|>" |
| 44 | END_MESSAGE = "<|end_message|>" |
| 45 | AUDIO_END = "<|audio_end|>" |
| 46 | |
| 47 | ROLE_MESSAGE_TOKENS: dict[str, str] = { |
| 48 | "user": MESSAGE_USER, |
| 49 | "assistant": MESSAGE_MODEL, |
| 50 | "system": MESSAGE_SYSTEM, |
| 51 | # The developer role folds into the system role on the Inkling wire. |
| 52 | "developer": MESSAGE_SYSTEM, |
| 53 | "tool": MESSAGE_TOOL, |
| 54 | } |
| 55 | |
| 56 | # Alternate vocab spellings per semantic token. The Inkling HF tokenizer |
| 57 | # exposes some semantic slots as ``<|unused_NNNNNN|>`` tokens (notably |
| 58 | # CONTENT_XML = <|unused_200024|>), so ID resolution must try these |
| 59 | # spellings in order. |
| 60 | SPECIAL_TOKEN_SPELLINGS: dict[str, tuple[str, ...]] = { |
| 61 | MESSAGE_USER: (MESSAGE_USER,), |
| 62 | MESSAGE_MODEL: (MESSAGE_MODEL,), |
| 63 | MESSAGE_SYSTEM: (MESSAGE_SYSTEM,), |
| 64 | MESSAGE_TOOL: (MESSAGE_TOOL,), |
| 65 | CONTENT_TEXT: (CONTENT_TEXT,), |
| 66 | CONTENT_IMAGE: (CONTENT_IMAGE,), |
| 67 | CONTENT_MODEL_END_SAMPLING: (CONTENT_MODEL_END_SAMPLING,), |
| 68 | CONTENT_THINKING: (CONTENT_THINKING,), |
| 69 | CONTENT_AUDIO_INPUT: (CONTENT_AUDIO_INPUT,), |
| 70 | CONTENT_XML: (CONTENT_XML, "<|unused_200024|>"), |
| 71 | CONTENT_INVOKE_TOOL_JSON: (CONTENT_INVOKE_TOOL_JSON,), |
| 72 | END_MESSAGE: (END_MESSAGE,), |
| 73 | AUDIO_END: (AUDIO_END,), |
| 74 | } |
| 75 | |
| 76 | |
| 77 | class InklingTextTokenizer(Protocol): |
| 78 | """Structural tokenizer contract required by the renderer.""" |
| 79 | |
| 80 | def encode_text(self, text: str) -> list[int]: ... |
| 81 | |
| 82 | def encode_special(self, token: str) -> int: ... |
| 83 | |
| 84 | |
| 85 | # OpenAI content-part type spellings that mean image / audio (rendering only |
| 86 | # needs the kind, not the bytes — the bytes are handled by the MM processor). |
| 87 | _IMAGE_PART_TYPES = frozenset({"image", "input_image", "image_url"}) |
| 88 | _AUDIO_PART_TYPES = frozenset({"audio", "input_audio", "audio_url"}) |
| 89 | |
| 90 | _MAX_REASONING_EFFORT = 0.99 |
| 91 | |
| 92 | |
| 93 | def render_inkling_messages( |
| 94 | messages: Sequence[Mapping[str, Any]], |
| 95 | tokenizer: InklingTextTokenizer, |
| 96 | *, |
| 97 | add_generation_prompt: bool = True, |
| 98 | tools: Sequence[Mapping[str, Any]] | None = None, |
| 99 | reasoning_effort: float | None = None, |
| 100 | ) -> list[int]: |
| 101 | """Render chat messages to Inkling input ids. |
| 102 | |
| 103 | PURE renderer: emits Inkling framing plus bare media markers; media |
| 104 | encoding and placeholder expansion happen later in the MM processor. |
| 105 | ``add_generation_prompt`` appends the assistant turn opener so the |
| 106 | model continues into the response. |
| 107 | """ |
| 108 | input_ids: list[int] = [] |
| 109 | tool_call_id_to_name: dict[str, str] = {} |
| 110 | |
| 111 | # Request-level tools plus per-developer-message tools (Rust renderer |
| 112 | # semantics) are declared in a single leading system block. |
| 113 | all_tools = list(tools or []) |
| 114 | for message in messages: |
| 115 | if message.get("role") == "developer": |
| 116 | all_tools.extend(message.get("tools") or []) |
| 117 | |
| 118 | if all_tools: |
| 119 | _append_message( |
| 120 | input_ids, |
| 121 | tokenizer, |
| 122 | "system", |
| 123 | "xml", |
| 124 | _tool_declare_json(all_tools), |
| 125 | author_name="tool_declare", |
| 126 | ) |
| 127 | |
| 128 | for message in messages: |
| 129 | role = _expect_role(message) |
| 130 | if role not in {"system", "developer"} and reasoning_effort is not None: |
| 131 | _append_reasoning_effort(input_ids, tokenizer, reasoning_effort) |
| 132 | reasoning_effort = None |
| 133 | |
| 134 | if role == "tool": |
| 135 | tool_name = message.get("name") or tool_call_id_to_name.get( |
| 136 | str(message.get("tool_call_id") or ""), "" |
| 137 | ) |
| 138 | _append_message( |
| 139 | input_ids, |
| 140 | tokenizer, |
| 141 | "tool", |
| 142 | "text", |
| 143 | _flatten_text_content(message.get("content")), |
| 144 | author_name=str(tool_name), |
| 145 | ) |
| 146 | continue |
| 147 | |
| 148 | if role == "assistant": |
| 149 | reasoning_content = message.get("reasoning") |
| 150 | if reasoning_content is None: |
| 151 | reasoning_content = message.get("reasoning_content") |
| 152 | if reasoning_content: |
| 153 | if not isinstance(reasoning_content, str): |
| 154 | raise TypeError( |
| 155 | "assistant reasoning_content must be a string for " |
| 156 | "Inkling rendering" |
| 157 | ) |
| 158 | _append_message( |
| 159 | input_ids, |
| 160 | tokenizer, |
| 161 | "assistant", |
| 162 | "thinking", |
| 163 | reasoning_content, |
| 164 | ) |
| 165 | |
| 166 | for kind, text in _iter_render_parts(message.get("content")): |
| 167 | _append_message(input_ids, tokenizer, role, kind, text) |
| 168 | |
| 169 | if role == "assistant": |
| 170 | for tool_call in message.get("tool_calls") or []: |
| 171 | name, args = _tool_call_name_and_args(tool_call) |
| 172 | tool_call_id = _as_mapping(tool_call).get("id") |
| 173 | if tool_call_id: |
| 174 | tool_call_id_to_name[str(tool_call_id)] = name |
| 175 | _append_message( |
| 176 | input_ids, |
| 177 | tokenizer, |
| 178 | "assistant", |
| 179 | "invoke_tool_json", |
| 180 | _tool_call_json(name, args), |
| 181 | author_name=name, |
| 182 | ) |
| 183 | |
| 184 | input_ids.append(tokenizer.encode_special(CONTENT_MODEL_END_SAMPLING)) |
| 185 | |
| 186 | if reasoning_effort is not None: |
| 187 | _append_reasoning_effort(input_ids, tokenizer, reasoning_effort) |
| 188 | |
| 189 | if add_generation_prompt: |
| 190 | input_ids.append(tokenizer.encode_special(MESSAGE_MODEL)) |
| 191 | return input_ids |
| 192 | |
| 193 | |
| 194 | def _append_reasoning_effort( |
| 195 | input_ids: list[int], |
| 196 | tokenizer: InklingTextTokenizer, |
| 197 | reasoning_effort: float, |
| 198 | ) -> None: |
| 199 | _append_message( |
| 200 | input_ids, |
| 201 | tokenizer, |
| 202 | "system", |
| 203 | "text", |
| 204 | _thinking_effort_text(reasoning_effort), |
| 205 | ) |
| 206 | |
| 207 | |
| 208 | def _thinking_effort_text(reasoning_effort: Any) -> str: |
| 209 | if isinstance(reasoning_effort, bool) or not isinstance( |
| 210 | reasoning_effort, (int, float) |
| 211 | ): |
| 212 | raise TypeError( |
| 213 | "Inkling reasoning_effort must be a number in [0.0, 0.99], " |
| 214 | f"got {type(reasoning_effort).__name__}" |
| 215 | ) |
| 216 | value = float(reasoning_effort) |
| 217 | if not 0.0 <= value <= _MAX_REASONING_EFFORT: |
| 218 | raise ValueError( |
| 219 | f"Inkling reasoning_effort must be in [0.0, 0.99], got {value}" |
| 220 | ) |
| 221 | |
| 222 | effort_text = f"{value:.2f}".rstrip("0").rstrip(".") |
| 223 | if effort_text in {"0", "-0"}: |
| 224 | effort_text = "0.0" |
| 225 | return f"Thinking effort level: {effort_text}" |
| 226 | |
| 227 | |
| 228 | def _append_message( |
| 229 | input_ids: list[int], |
| 230 | tokenizer: InklingTextTokenizer, |
| 231 | role: str, |
| 232 | kind: str, |
| 233 | text: str, |
| 234 | *, |
| 235 | author_name: str | None = None, |
| 236 | ) -> None: |
| 237 | input_ids.append(tokenizer.encode_special(ROLE_MESSAGE_TOKENS[role])) |
| 238 | if author_name: |
| 239 | input_ids.extend(tokenizer.encode_text(author_name)) |
| 240 | |
| 241 | if kind == "text": |
| 242 | input_ids.append(tokenizer.encode_special(CONTENT_TEXT)) |
| 243 | input_ids.extend(tokenizer.encode_text(text)) |
| 244 | elif kind == "image": |
| 245 | # Bare marker only: the MM processor replaces the marker with |
| 246 | # `marker + placeholder * num_patches` (see mm_preprocess.py). |
| 247 | input_ids.append(tokenizer.encode_special(CONTENT_IMAGE)) |
| 248 | elif kind == "audio": |
| 249 | input_ids.append(tokenizer.encode_special(CONTENT_AUDIO_INPUT)) |
| 250 | input_ids.append(tokenizer.encode_special(AUDIO_END)) |
| 251 | elif kind == "thinking": |
| 252 | input_ids.append(tokenizer.encode_special(CONTENT_THINKING)) |
| 253 | input_ids.extend(tokenizer.encode_text(text)) |
| 254 | elif kind == "xml": |
| 255 | input_ids.append(tokenizer.encode_special(CONTENT_XML)) |
| 256 | input_ids.extend(tokenizer.encode_text(text)) |
| 257 | elif kind == "invoke_tool_json": |
| 258 | input_ids.append(tokenizer.encode_special(CONTENT_INVOKE_TOOL_JSON)) |
| 259 | input_ids.extend(tokenizer.encode_text(text)) |
| 260 | else: |
| 261 | raise ValueError(f"unsupported Inkling render part kind: {kind!r}") |
| 262 | |
| 263 | input_ids.append(tokenizer.encode_special(END_MESSAGE)) |
| 264 | |
| 265 | |
| 266 | def _iter_render_parts(content: Any) -> Iterator[tuple[str, str]]: |
| 267 | """Yield (kind, text) per content part: kind in {text, image, audio}.""" |
| 268 | if content is None: |
| 269 | return |
| 270 | if isinstance(content, str): |
| 271 | if content: |
| 272 | yield ("text", content) |
| 273 | return |
| 274 | if not isinstance(content, Sequence) or isinstance(content, (bytes, bytearray)): |
| 275 | raise TypeError("message content must be a string or a sequence of parts") |
| 276 | for part in content: |
| 277 | if isinstance(part, str): |
| 278 | yield ("text", part) |
| 279 | continue |
| 280 | if not isinstance(part, Mapping): |
| 281 | raise TypeError(f"content part must be mapping, got {type(part).__name__}") |
| 282 | ptype = part.get("type") |
| 283 | if ptype in (None, "text", "input_text"): |
| 284 | text = part.get("text", "") |
| 285 | yield ("text", text if isinstance(text, str) else "") |
| 286 | elif ptype in _IMAGE_PART_TYPES: |
| 287 | yield ("image", "") |
| 288 | elif ptype in _AUDIO_PART_TYPES: |
| 289 | yield ("audio", "") |
| 290 | else: |
| 291 | raise ValueError(f"unsupported content part type: {ptype!r}") |
| 292 | |
| 293 | |
| 294 | def _flatten_text_content(content: Any) -> str: |
| 295 | """Flatten a tool-response content (string or text parts) to text.""" |
| 296 | if content is None: |
| 297 | return "" |
| 298 | if isinstance(content, str): |
| 299 | return content |
| 300 | parts: list[str] = [] |
| 301 | for kind, text in _iter_render_parts(content): |
| 302 | if kind != "text": |
| 303 | raise ValueError( |
| 304 | "Inkling tool response content must be text, " |
| 305 | f"got a part of kind {kind!r}" |
| 306 | ) |
| 307 | parts.append(text) |
| 308 | return "".join(parts) |
| 309 | |
| 310 | |
| 311 | def _expect_role(message: Mapping[str, Any]) -> str: |
| 312 | role = message.get("role") |
| 313 | if role not in ROLE_MESSAGE_TOKENS: |
| 314 | raise ValueError( |
| 315 | f"unsupported Inkling message role {role!r}; " |
| 316 | f"expected one of {sorted(ROLE_MESSAGE_TOKENS)}" |
| 317 | ) |
| 318 | return str(role) |
| 319 | |
| 320 | |
| 321 | def _as_mapping(value: Any) -> Mapping[str, Any]: |
| 322 | if isinstance(value, Mapping): |
| 323 | return value |
| 324 | if hasattr(value, "model_dump"): |
| 325 | dumped = value.model_dump() |
| 326 | if isinstance(dumped, Mapping): |
| 327 | return dumped |
| 328 | raise TypeError(f"expected mapping, got {type(value).__name__}") |
| 329 | |
| 330 | |
| 331 | def _canonical_json(value: Any) -> str: |
| 332 | return json.dumps( |
| 333 | _sort_json(value), |
| 334 | ensure_ascii=False, |
| 335 | allow_nan=False, |
| 336 | separators=(",", ":"), |
| 337 | ) |
| 338 | |
| 339 | |
| 340 | def _sort_json(value: Any) -> Any: |
| 341 | if isinstance(value, Mapping): |
| 342 | return {str(key): _sort_json(value[key]) for key in sorted(value)} |
| 343 | if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): |
| 344 | return [_sort_json(item) for item in value] |
| 345 | return value |
| 346 | |
| 347 | |
| 348 | def _tool_declare_json(tools: Sequence[Mapping[str, Any]]) -> str: |
| 349 | tool_specs = [] |
| 350 | for tool_value in tools: |
| 351 | tool = _as_mapping(tool_value) |
| 352 | function = _as_mapping(tool.get("function", {})) |
| 353 | tool_specs.append( |
| 354 | { |
| 355 | "description": function.get("description") or "", |
| 356 | "name": function["name"], |
| 357 | "parameters": function.get("parameters") or {}, |
| 358 | "type": tool.get("type", "function"), |
| 359 | } |
| 360 | ) |
| 361 | return _canonical_json(tool_specs) |
| 362 | |
| 363 | |
| 364 | def _tool_call_name_and_args(tool_call_value: Any) -> tuple[str, Mapping[str, Any]]: |
| 365 | tool_call = _as_mapping(tool_call_value) |
| 366 | function = _as_mapping(tool_call.get("function", {})) |
| 367 | name = function.get("name") |
| 368 | if not isinstance(name, str): |
| 369 | raise TypeError("tool call function name must be a string") |
| 370 | |
| 371 | raw_args = function.get("arguments") or {} |
| 372 | if isinstance(raw_args, str): |
| 373 | args = json.loads(raw_args) if raw_args.strip() else {} |
| 374 | else: |
| 375 | args = raw_args |
| 376 | if not isinstance(args, Mapping): |
| 377 | raise TypeError("tool call function arguments must decode to an object") |
| 378 | return name, args |
| 379 | |
| 380 | |
| 381 | def _tool_call_json(name: str, args: Mapping[str, Any]) -> str: |
| 382 | name_json = json.dumps(name, ensure_ascii=False, allow_nan=False) |
| 383 | return f'{{"name":{name_json},"args":{_canonical_json(args)}}}' |