@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
py 178 lines 6 KB
Raw
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 """Native Inkling chat renderer for the Python frontend.
4
5 Mirrors the Rust frontend's native Inkling renderer
6 (``rust/src/chat/src/renderer/inkling/mod.rs``): chat messages are rendered
7 directly to token ids — Inkling has no Jinja chat template and no faithful
8 text form.
9
10 The encoding logic lives in ``inkling_encoding.py`` behind a narrow
11 "OpenAI messages + tools -> token ids" call; see the swap-point comment
12 in :meth:`InklingRenderer._render` for adopting a standalone Inkling
13 input-processing library (mistral-common style) later.
14 """
15
16 from vllm.config import VllmConfig
17 from vllm.entrypoints.chat_utils import (
18 ChatCompletionMessageParam,
19 ConversationMessage,
20 parse_chat_messages,
21 parse_chat_messages_async,
22 )
23 from vllm.logger import init_logger
24 from vllm.tokenizers.hf import HfTokenizer
25 from vllm.utils.async_utils import make_async
26
27 from .base import BaseRenderer
28 from .inkling_encoding import SPECIAL_TOKEN_SPELLINGS, render_inkling_messages
29 from .inputs import DictPrompt
30 from .inputs.preprocess import parse_dec_only_prompt
31 from .params import ChatParams
32
33 logger = init_logger(__name__)
34
35 _NAMED_REASONING_EFFORT = {
36 "none": 0.0,
37 "minimal": 0.1,
38 "low": 0.2,
39 "medium": 0.7,
40 "high": 0.9,
41 "xhigh": 0.99,
42 "max": 0.99,
43 }
44
45 _DEFAULT_REASONING_EFFORT = 0.9
46
47
48 def _resolve_reasoning_effort(value: object) -> float | int | None:
49 if value is None:
50 return _DEFAULT_REASONING_EFFORT
51 if isinstance(value, str):
52 return _NAMED_REASONING_EFFORT.get(value)
53 if isinstance(value, bool) or not isinstance(value, (int, float)):
54 return None
55 return value
56
57
58 class _HfBackedTmlTokenizer:
59 """Adapts an HF tokenizer to the encoding core's tokenizer protocol.
60
61 Special-token ids are resolved from the tokenizer vocab at
62 construction (never hardcoded), trying each known spelling — the Inkling
63 HF vocab exposes some semantic slots as ``<|unused_NNNNNN|>`` tokens.
64 """
65
66 def __init__(self, tokenizer: HfTokenizer) -> None:
67 self._tokenizer = tokenizer
68
69 vocab = tokenizer.get_vocab()
70 special_ids: dict[str, int] = {}
71 missing: list[str] = []
72 for token, spellings in SPECIAL_TOKEN_SPELLINGS.items():
73 for spelling in spellings:
74 token_id = vocab.get(spelling)
75 if token_id is not None:
76 special_ids[token] = token_id
77 break
78 else:
79 missing.append(token)
80 if missing:
81 raise ValueError(f"Inkling tokenizer is missing special tokens: {missing}")
82 self._special_ids = special_ids
83
84 def encode_text(self, text: str) -> list[int]:
85 return self._tokenizer.encode(text, add_special_tokens=False)
86
87 def encode_special(self, token: str) -> int:
88 return self._special_ids[token]
89
90
91 class InklingRenderer(BaseRenderer[HfTokenizer]):
92 def __init__(
93 self,
94 config: VllmConfig,
95 tokenizer: HfTokenizer | None,
96 ) -> None:
97 super().__init__(config, tokenizer)
98
99 self._inkling_tokenizer = _HfBackedTmlTokenizer(self.get_tokenizer())
100 self._render_async = make_async(self._render, executor=self._executor)
101
102 def _render(
103 self,
104 messages: list[ChatCompletionMessageParam],
105 params: ChatParams,
106 ) -> list[int]:
107 kwargs = params.chat_template_kwargs or {}
108
109 if kwargs.get("continue_final_message"):
110 raise ValueError("Inkling renderer does not support continue_final_message")
111
112 reasoning_effort = _resolve_reasoning_effort(kwargs.get("reasoning_effort"))
113
114 try:
115 # Swap point: to adopt a standalone Inkling input-processing
116 # library, replace this call (and the _HfBackedTmlTokenizer
117 # adapter above) with the library's renderer.
118 return render_inkling_messages(
119 messages,
120 self._inkling_tokenizer,
121 add_generation_prompt=kwargs.get("add_generation_prompt", True),
122 tools=kwargs.get("tools"),
123 reasoning_effort=reasoning_effort,
124 )
125 except ValueError:
126 raise
127 except (TypeError, KeyError) as e:
128 # Malformed request content; surface as a request error.
129 raise ValueError(str(e)) from e
130 except Exception as e:
131 logger.exception("Error while rendering Inkling chat messages")
132 raise ValueError(str(e)) from e
133
134 def render_messages(
135 self,
136 messages: list[ChatCompletionMessageParam],
137 params: ChatParams,
138 ) -> tuple[list[ConversationMessage], DictPrompt]:
139 conversation, mm_data, mm_uuids = parse_chat_messages(
140 messages,
141 self.model_config,
142 content_format="string",
143 media_io_kwargs=params.media_io_kwargs,
144 mm_processor_kwargs=params.mm_processor_kwargs,
145 )
146
147 token_ids = self._render(messages, params)
148
149 prompt = parse_dec_only_prompt(token_ids)
150 if mm_data is not None:
151 prompt["multi_modal_data"] = mm_data
152 if mm_uuids is not None:
153 prompt["multi_modal_uuids"] = mm_uuids
154
155 return conversation, prompt
156
157 async def render_messages_async(
158 self,
159 messages: list[ChatCompletionMessageParam],
160 params: ChatParams,
161 ) -> tuple[list[ConversationMessage], DictPrompt]:
162 conversation, mm_data, mm_uuids = await parse_chat_messages_async(
163 messages,
164 self.model_config,
165 content_format="string",
166 media_io_kwargs=params.media_io_kwargs,
167 mm_processor_kwargs=params.mm_processor_kwargs,
168 )
169
170 token_ids = await self._render_async(messages, params)
171
172 prompt = parse_dec_only_prompt(token_ids)
173 if mm_data is not None:
174 prompt["multi_modal_data"] = mm_data
175 if mm_uuids is not None:
176 prompt["multi_modal_uuids"] = mm_uuids
177
178 return conversation, prompt