@tduncan / Nvidia-Optimization-Stack-VLLM / commits / f5a7cce9b6

[Model] Add Kimi K3 support: Python frontend [2/2] (#50093)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>

Bugen Zhao committed Jul 29, 2026 at 16:06 UTC f5a7cce9b6a61f4d995629a7418c7ea822e34a64
20 files changed +3056 -5
tests/reasoning/test_kimi_k3_reasoning_parser.py new
+250
@@ -0,0 +1,250 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import pytest
5 +
6 +from vllm.entrypoints.openai.chat_completion.protocol import (
7 + ChatCompletionRequest,
8 +)
9 +from vllm.entrypoints.openai.engine.protocol import DeltaMessage
10 +from vllm.parser.kimi_k3 import KimiK3Parser
11 +from vllm.parser.parser_manager import ParserManager
12 +from vllm.reasoning.kimi_k3_reasoning_parser import KimiK3ReasoningParser
13 +
14 +pytestmark = pytest.mark.skip_global_cleanup
15 +
16 +OPEN = "<|open|>"
17 +CLOSE = "<|close|>"
18 +SEP = "<|sep|>"
19 +THINK_OPEN = f"{OPEN}think{SEP}"
20 +THINK_CLOSE = f"{CLOSE}think{SEP}"
21 +RESPONSE_OPEN = f"{OPEN}response{SEP}"
22 +
23 +
24 +class DummyTokenizer:
25 + def get_vocab(self) -> dict[str, int]:
26 + return {}
27 +
28 + def encode(self, text: str, add_special_tokens: bool = False) -> list[int]:
29 + if text == THINK_OPEN:
30 + return [1, 2, 3]
31 + if text == THINK_CLOSE:
32 + return [4, 2, 3]
33 + return [ord(ch) for ch in text]
34 +
35 +
36 +class ReasoningOnlyParser(KimiK3Parser):
37 + reasoning_parser_cls = KimiK3ReasoningParser
38 +
39 +
40 +def test_parser_manager_selects_kimi_k3_parser_for_reasoning_only():
41 + parser_cls = ParserManager.get_parser(reasoning_parser_name="kimi_k3")
42 +
43 + assert parser_cls is not None
44 + assert issubclass(parser_cls, KimiK3Parser)
45 + assert parser_cls.reasoning_parser_cls is KimiK3ReasoningParser
46 + assert parser_cls.tool_parser_cls is None
47 +
48 +
49 +def test_parser_selection_thinking_disabled():
50 + parser = KimiK3ReasoningParser(
51 + DummyTokenizer(), chat_template_kwargs={"thinking": False}
52 + )
53 +
54 + assert parser._thinking_enabled is False
55 +
56 +
57 +def test_extract_reasoning_with_xtml_tags():
58 + parser = KimiK3ReasoningParser(DummyTokenizer())
59 + request = ChatCompletionRequest(model="test-model", messages=[])
60 +
61 + reasoning, content = parser.extract_reasoning_content(
62 + f"{THINK_OPEN}step{THINK_CLOSE}{RESPONSE_OPEN}answer",
63 + request,
64 + )
65 +
66 + assert reasoning == "step"
67 + assert content == "answer"
68 +
69 +
70 +def test_extract_reasoning_with_generation_prefix_consumed():
71 + parser = KimiK3ReasoningParser(DummyTokenizer())
72 + request = ChatCompletionRequest(model="test-model", messages=[])
73 +
74 + reasoning, content = parser.extract_reasoning_content(
75 + f"step{THINK_CLOSE}{RESPONSE_OPEN}answer",
76 + request,
77 + )
78 +
79 + assert reasoning == "step"
80 + assert content == "answer"
81 +
82 +
83 +def test_delegating_parser_strips_response_wrapper_without_tool_parser():
84 + parser = ReasoningOnlyParser(DummyTokenizer())
85 + request = ChatCompletionRequest(model="test-model", messages=[])
86 +
87 + reasoning, content, tool_calls = parser.parse(
88 + f"{THINK_OPEN}step{THINK_CLOSE}{RESPONSE_OPEN}answer",
89 + request,
90 + )
91 +
92 + assert reasoning == "step"
93 + assert content == "answer"
94 + assert tool_calls == []
95 +
96 +
97 +def test_is_reasoning_end_uses_full_input_ids():
98 + parser = KimiK3ReasoningParser(DummyTokenizer())
99 +
100 + assert not parser.is_reasoning_end([4, 2])
101 + assert parser.is_reasoning_end([4, 2, 3])
102 +
103 +
104 +def test_is_reasoning_end_ignores_stale_close_from_prior_turn():
105 + # DummyTokenizer: THINK_OPEN -> [1, 2, 3], THINK_CLOSE -> [4, 2, 3].
106 + # Multi-turn / agent continuation: a prior turn's think channel (its close
107 + # marker) is kept in the prompt, then the current turn opens a new think
108 + # block that has not closed yet. Reasoning must read as NOT ended, otherwise
109 + # the structured-output gate constrains the current turn's reasoning.
110 + parser = KimiK3ReasoningParser(DummyTokenizer())
111 +
112 + stale_close = [4, 2, 3]
113 + new_open = [1, 2, 3]
114 + # prior close, then current-turn open still unclosed -> not ended
115 + assert not parser.is_reasoning_end([*stale_close, *new_open])
116 + # ...then the current turn emits its own close -> ended
117 + assert parser.is_reasoning_end([*stale_close, *new_open, *stale_close])
118 + # open with no close yet -> not ended
119 + assert not parser.is_reasoning_end([*new_open])
120 +
121 +
122 +def test_streaming_split_open_marker_is_held_back():
123 + parser = KimiK3ReasoningParser(DummyTokenizer())
124 +
125 + first = parser.extract_reasoning_content_streaming(
126 + previous_text="",
127 + current_text=OPEN,
128 + delta_text=OPEN,
129 + previous_token_ids=[],
130 + current_token_ids=[1],
131 + delta_token_ids=[1],
132 + )
133 + second = parser.extract_reasoning_content_streaming(
134 + previous_text=OPEN,
135 + current_text=f"{OPEN}think",
136 + delta_text="think",
137 + previous_token_ids=[1],
138 + current_token_ids=[1, 2],
139 + delta_token_ids=[2],
140 + )
141 + third = parser.extract_reasoning_content_streaming(
142 + previous_text=f"{OPEN}think",
143 + current_text=THINK_OPEN + "step",
144 + delta_text=f"{SEP}step",
145 + previous_token_ids=[1, 2],
146 + current_token_ids=[1, 2, 3, 9],
147 + delta_token_ids=[3, 9],
148 + )
149 +
150 + assert first is None
151 + assert second is None
152 + assert isinstance(third, DeltaMessage)
153 + assert third.reasoning == "step"
154 +
155 +
156 +def test_streaming_split_close_marker_hands_content_downstream():
157 + parser = KimiK3ReasoningParser(DummyTokenizer())
158 +
159 + previous_text = f"{THINK_OPEN}step"
160 + partial_close = parser.extract_reasoning_content_streaming(
161 + previous_text=previous_text,
162 + current_text=previous_text + CLOSE,
163 + delta_text=CLOSE,
164 + previous_token_ids=[1, 2, 3, 9],
165 + current_token_ids=[1, 2, 3, 9, 4],
166 + delta_token_ids=[4],
167 + )
168 + closed = parser.extract_reasoning_content_streaming(
169 + previous_text=previous_text + CLOSE,
170 + current_text=previous_text + f"{THINK_CLOSE}{RESPONSE_OPEN}answer",
171 + delta_text=f"think{SEP}{RESPONSE_OPEN}answer",
172 + previous_token_ids=[1, 2, 3, 9, 4],
173 + current_token_ids=[1, 2, 3, 9, 4, 2, 3, 10],
174 + delta_token_ids=[2, 3, 10],
175 + )
176 +
177 + assert partial_close is None
178 + assert isinstance(closed, DeltaMessage)
179 + assert closed.reasoning is None
180 + assert closed.content == f"{RESPONSE_OPEN}answer"
181 + assert parser.extract_content_ids([2, 3, 10]) == [10]
182 +
183 +
184 +def test_thinking_disabled_streams_content():
185 + parser = KimiK3ReasoningParser(
186 + DummyTokenizer(), chat_template_kwargs={"enable_thinking": False}
187 + )
188 +
189 + delta = parser.extract_reasoning_content_streaming(
190 + previous_text="",
191 + current_text=f"{RESPONSE_OPEN}answer",
192 + delta_text=f"{RESPONSE_OPEN}answer",
193 + previous_token_ids=[],
194 + current_token_ids=[1],
195 + delta_token_ids=[1],
196 + )
197 +
198 + assert isinstance(delta, DeltaMessage)
199 + assert delta.content == f"{RESPONSE_OPEN}answer"
200 + assert delta.reasoning is None
201 +
202 +
203 +def test_delegating_parser_thinking_false_streams_response_content():
204 + parser = ReasoningOnlyParser(
205 + DummyTokenizer(), chat_template_kwargs={"thinking": False}
206 + )
207 + request = ChatCompletionRequest(
208 + model="test-model",
209 + messages=[],
210 + chat_template_kwargs={"thinking": False},
211 + )
212 +
213 + first = parser.parse_delta(
214 + delta_text="OK",
215 + delta_token_ids=[10],
216 + request=request,
217 + prompt_token_ids=[1],
218 + finished=False,
219 + )
220 + partial_close = parser.parse_delta(
221 + delta_text=CLOSE,
222 + delta_token_ids=[2],
223 + request=request,
224 + prompt_token_ids=[1],
225 + finished=False,
226 + )
227 + closed = parser.parse_delta(
228 + delta_text=f"response{SEP}",
229 + delta_token_ids=[3, 4],
230 + request=request,
231 + prompt_token_ids=[1],
232 + finished=False,
233 + )
234 +
235 + assert first is not None
236 + assert first.content == "OK"
237 + assert first.reasoning is None
238 + assert partial_close is None
239 + assert closed is None
240 +
241 +
242 +def test_adjust_request_keeps_xtml_markers_contiguous():
243 + parser = KimiK3ReasoningParser(DummyTokenizer())
244 + request = ChatCompletionRequest(model="test-model", messages=[])
245 +
246 + adjusted = parser.adjust_request(request)
247 +
248 + assert adjusted.skip_special_tokens is False
249 + if hasattr(adjusted, "spaces_between_special_tokens"):
250 + assert adjusted.spaces_between_special_tokens is False
tests/renderers/test_kimi_k3.py new
+353
@@ -0,0 +1,353 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +from dataclasses import dataclass, field
4 +from typing import Any
5 +
6 +import pytest
7 +
8 +from vllm.exceptions import VLLMValidationError
9 +from vllm.renderers import ChatParams
10 +from vllm.renderers.kimi_k3 import KimiK3Renderer, _merge_k3_media_io_kwargs
11 +from vllm.renderers.registry import RENDERER_REGISTRY
12 +from vllm.tokenizers.registry import TokenizerRegistry
13 +
14 +
15 +class StubTokenizer:
16 + """Stands in for the model's TikTokenTokenizer.
17 +
18 + Records the kwargs it is called with and returns fixed token ids, so tests
19 + can assert how the renderer drives ``apply_chat_template`` without the real
20 + (git-LFS) tiktoken vocabulary.
21 + """
22 +
23 + def __init__(self, token_ids: list[int]) -> None:
24 + self.token_ids = token_ids
25 + self.calls: list[dict[str, Any]] = []
26 + self.conversations: list[list[dict[str, Any]]] = []
27 +
28 + def apply_chat_template(self, conversation, **kwargs) -> list[int]:
29 + self.conversations.append(conversation)
30 + self.calls.append(kwargs)
31 + return list(self.token_ids)
32 +
33 +
34 +@dataclass
35 +class MockHFConfig:
36 + model_type: str = "kimi_k3"
37 +
38 +
39 +@dataclass
40 +class MockModelConfig:
41 + runner_type: str = "generate"
42 + is_multimodal_model: bool = False
43 + multimodal_config: Any = None
44 + hf_config: MockHFConfig = field(default_factory=MockHFConfig)
45 + allowed_local_media_path: str = ""
46 + allowed_media_domains: Any = None
47 + enable_prompt_embeds: bool = False
48 + renderer_num_workers: int = 1
49 +
50 +
51 +@dataclass
52 +class MockParallelConfig:
53 + _api_process_rank: int = 0
54 +
55 +
56 +@dataclass
57 +class MockVllmConfig:
58 + model_config: MockModelConfig
59 + parallel_config: MockParallelConfig
60 +
61 +
62 +def _make_renderer(tokenizer: StubTokenizer) -> KimiK3Renderer:
63 + config = MockVllmConfig(MockModelConfig(), MockParallelConfig())
64 + return KimiK3Renderer(config, tokenizer)
65 +
66 +
67 +def test_kimi_k3_registered():
68 + assert RENDERER_REGISTRY.load_renderer_cls("kimi_k3").__name__ == "KimiK3Renderer"
69 + assert (
70 + TokenizerRegistry.load_tokenizer_cls("kimi_k3").__name__ == "CachedHfTokenizer"
71 + )
72 +
73 +
74 +def test_k3_media_io_defaults_preserve_original_mode():
75 + # Default: K3 keeps the original image mode (no background flattening).
76 + assert _merge_k3_media_io_kwargs(None) == {"image": {"image_mode": None}}
77 +
78 + # Server-/request-level values take precedence over the K3 default.
79 + assert _merge_k3_media_io_kwargs({"image": {"image_mode": "RGB"}}) == {
80 + "image": {"image_mode": "RGB"}
81 + }
82 +
83 + # Unrelated image kwargs are merged with the default.
84 + assert _merge_k3_media_io_kwargs(
85 + {"image": {"rgba_background_color": (0, 0, 0)}}
86 + ) == {"image": {"image_mode": None, "rgba_background_color": (0, 0, 0)}}
87 +
88 +
89 +def test_apply_chat_template_forces_tokenize_and_pins_return_dict():
90 + tokenizer = StubTokenizer([7, 8, 9])
91 + renderer = _make_renderer(tokenizer)
92 + tools = [{"type": "function", "function": {"name": "search"}}]
93 + params = ChatParams(
94 + chat_template_kwargs={"tools": tools, "tokenize": False, "thinking": True}
95 + )
96 +
97 + token_ids = renderer._apply_chat_template(
98 + [{"role": "user", "content": "hi"}], params
99 + )
100 +
101 + assert token_ids == [7, 8, 9]
102 + kwargs = tokenizer.calls[-1]
103 + # tokenize is forced on even though the request asked for False, so K3 keeps
104 + # the special-vs-ordinary token distinction instead of re-tokenizing a string.
105 + assert kwargs["tokenize"] is True
106 + # return_dict is pinned False so we always get a flat list of ids.
107 + assert kwargs["return_dict"] is False
108 + assert kwargs["tools"] == tools
109 + assert kwargs["thinking"] is True
110 +
111 +
112 +def test_apply_chat_template_translates_standard_thinking_kwargs():
113 + # Standard enable_thinking/reasoning_effort kwargs must be translated
114 + # to K3's native thinking/thinking_effort.
115 + tokenizer = StubTokenizer([7, 8, 9])
116 + renderer = _make_renderer(tokenizer)
117 + params = ChatParams(
118 + chat_template_kwargs={"enable_thinking": False, "reasoning_effort": "none"}
119 + )
120 +
121 + renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
122 +
123 + kwargs = tokenizer.calls[-1]
124 + assert kwargs["thinking"] is False
125 + assert "thinking_effort" not in kwargs
126 + assert "enable_thinking" not in kwargs
127 + assert "reasoning_effort" not in kwargs
128 +
129 +
130 +@pytest.mark.parametrize("reasoning_effort", ["low", "high", "max"])
131 +def test_apply_chat_template_translates_supported_reasoning_effort(
132 + reasoning_effort: str,
133 +):
134 + tokenizer = StubTokenizer([7, 8, 9])
135 + renderer = _make_renderer(tokenizer)
136 + params = ChatParams(chat_template_kwargs={"reasoning_effort": reasoning_effort})
137 +
138 + renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
139 +
140 + kwargs = tokenizer.calls[-1]
141 + assert kwargs["thinking_effort"] == reasoning_effort
142 + assert "reasoning_effort" not in kwargs
143 +
144 +
145 +@pytest.mark.parametrize("reasoning_effort", ["minimal", "medium", "xhigh"])
146 +def test_apply_chat_template_rejects_unsupported_reasoning_effort(
147 + reasoning_effort: str,
148 +):
149 + tokenizer = StubTokenizer([7, 8, 9])
150 + renderer = _make_renderer(tokenizer)
151 + params = ChatParams(chat_template_kwargs={"reasoning_effort": reasoning_effort})
152 +
153 + with pytest.raises(VLLMValidationError, match="thinking_effort") as exc_info:
154 + renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
155 +
156 + assert exc_info.value.parameter == "thinking_effort"
157 + assert exc_info.value.value == reasoning_effort
158 + assert tokenizer.calls == []
159 +
160 +
161 +def test_apply_chat_template_validates_canonical_native_thinking_effort():
162 + tokenizer = StubTokenizer([7, 8, 9])
163 + renderer = _make_renderer(tokenizer)
164 + params = ChatParams(
165 + chat_template_kwargs={
166 + "thinking_effort": "low",
167 + "reasoning_effort": "medium",
168 + }
169 + )
170 +
171 + renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
172 +
173 + assert tokenizer.calls[-1]["thinking_effort"] == "low"
174 +
175 +
176 +@pytest.mark.parametrize("thinking_effort", ["none", "minimal", "medium", "xhigh"])
177 +def test_apply_chat_template_rejects_unsupported_native_thinking_effort(
178 + thinking_effort: str,
179 +):
180 + tokenizer = StubTokenizer([7, 8, 9])
181 + renderer = _make_renderer(tokenizer)
182 + params = ChatParams(chat_template_kwargs={"thinking_effort": thinking_effort})
183 +
184 + with pytest.raises(VLLMValidationError, match="thinking_effort") as exc_info:
185 + renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
186 +
187 + assert exc_info.value.parameter == "thinking_effort"
188 + assert exc_info.value.value == thinking_effort
189 + assert tokenizer.calls == []
190 +
191 +
192 +def test_apply_chat_template_native_k3_kwargs_take_precedence():
193 + tokenizer = StubTokenizer([7, 8, 9])
194 + renderer = _make_renderer(tokenizer)
195 + params = ChatParams(
196 + chat_template_kwargs={
197 + "thinking": True,
198 + "enable_thinking": False,
199 + "thinking_effort": "low",
200 + "reasoning_effort": "high",
201 + }
202 + )
203 +
204 + renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
205 +
206 + kwargs = tokenizer.calls[-1]
207 + assert kwargs["thinking"] is True
208 + assert kwargs["thinking_effort"] == "low"
209 +
210 +
211 +def test_apply_chat_template_adds_k3_api_metadata():
212 + tokenizer = StubTokenizer([7, 8, 9])
213 + renderer = _make_renderer(tokenizer)
214 + response_format = {"type": "json_object"}
215 + params = ChatParams(
216 + tool_choice="required",
217 + response_format=response_format,
218 + )
219 +
220 + renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
221 +
222 + kwargs = tokenizer.calls[-1]
223 + assert kwargs["tool_choice"] == "required"
224 + assert kwargs["response_format"] == response_format
225 +
226 +
227 +def test_apply_chat_template_auto_tool_choice_keeps_template_kwarg():
228 + tokenizer = StubTokenizer([7, 8, 9])
229 + renderer = _make_renderer(tokenizer)
230 + params = ChatParams(
231 + chat_template_kwargs={"tool_choice": "required"},
232 + tool_choice="auto",
233 + )
234 +
235 + renderer._apply_chat_template([{"role": "user", "content": "hi"}], params)
236 +
237 + assert tokenizer.calls[-1]["tool_choice"] == "required"
238 +
239 +
240 +def test_apply_chat_template_omits_tool_choice_without_tools():
241 + tokenizer = StubTokenizer([7, 8, 9])
242 + renderer = _make_renderer(tokenizer)
243 +
244 + renderer._apply_chat_template(
245 + [{"role": "user", "content": "hi"}], ChatParams(tool_choice=None)
246 + )
247 +
248 + assert "tool_choice" not in tokenizer.calls[-1]
249 +
250 +
251 +def test_render_messages_returns_token_prompt():
252 + renderer = _make_renderer(StubTokenizer([1, 2, 3]))
253 +
254 + conversation, prompt = renderer.render_messages(
255 + [{"role": "user", "content": "hi"}], ChatParams()
256 + )
257 +
258 + assert prompt == {"prompt_token_ids": [1, 2, 3]}
259 + assert "multi_modal_data" not in prompt
260 + assert conversation[0]["role"] == "user"
261 +
262 +
263 +def test_render_messages_derives_private_xtml_tool_attrs():
264 + tokenizer = StubTokenizer([1, 2, 3])
265 + renderer = _make_renderer(tokenizer)
266 +
267 + conversation, _ = renderer.render_messages(
268 + [
269 + {
270 + "role": "assistant",
271 + "content": "",
272 + "tool_calls": [
273 + {
274 + "id": "lookup:0",
275 + "type": "function",
276 + "function": {"name": "lookup", "arguments": "{}"},
277 + },
278 + {
279 + "id": "lookup:1",
280 + "type": "function",
281 + "function": {"name": "lookup", "arguments": "{}"},
282 + },
283 + ],
284 + },
285 + {
286 + "role": "tool",
287 + "tool_call_id": "lookup:1",
288 + "tool": "client-supplied-name",
289 + "index": 99,
290 + "content": "second",
291 + },
292 + {
293 + "role": "tool",
294 + "tool_call_id": "lookup:0",
295 + "content": "first",
296 + },
297 + ],
298 + ChatParams(),
299 + )
300 +
301 + assert [message["content"] for message in conversation[1:]] == [
302 + "first",
303 + "second",
304 + ]
305 + assert conversation[1]["tool"] == "lookup"
306 + assert conversation[1]["index"] == 1
307 + assert conversation[2]["tool"] == "lookup"
308 + assert conversation[2]["index"] == 2
309 + assert tokenizer.conversations[-1] == conversation
310 +
311 +
312 +def test_render_messages_ignores_client_supplied_xtml_tool_attrs():
313 + tokenizer = StubTokenizer([1, 2, 3])
314 + renderer = _make_renderer(tokenizer)
315 +
316 + conversation, _ = renderer.render_messages(
317 + [
318 + {
319 + "role": "assistant",
320 + "content": "",
321 + "tool_calls": [
322 + {
323 + "id": "lookup:0",
324 + "type": "function",
325 + "function": {"name": "lookup", "arguments": "{}"},
326 + }
327 + ],
328 + },
329 + {
330 + "role": "tool",
331 + "tool_call_id": "unknown",
332 + "tool": "lookup",
333 + "index": 1,
334 + "content": "result",
335 + },
336 + ],
337 + ChatParams(),
338 + )
339 +
340 + assert "tool" not in conversation[1]
341 + assert "index" not in conversation[1]
342 +
343 +
344 +@pytest.mark.asyncio
345 +async def test_render_messages_async_returns_token_prompt():
346 + renderer = _make_renderer(StubTokenizer([4, 5]))
347 +
348 + conversation, prompt = await renderer.render_messages_async(
349 + [{"role": "user", "content": "hi"}], ChatParams()
350 + )
351 +
352 + assert prompt == {"prompt_token_ids": [4, 5]}
353 + assert conversation[0]["role"] == "user"
tests/tool_parsers/test_kimi_k3_named_tool_choice.py new
+62
@@ -0,0 +1,62 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Named tool choice for Kimi K3: allowed when the XTML structural tag is
4 +attached (strict tool calling), rejected otherwise."""
5 +
6 +import pytest
7 +
8 +from vllm.entrypoints.openai.chat_completion.protocol import (
9 + ChatCompletionRequest,
10 +)
11 +from vllm.exceptions import VLLMValidationError
12 +from vllm.sampling_params import StructuredOutputsParams
13 +
14 +
15 +class _DummyTokenizer:
16 + def get_vocab(self):
17 + return {}
18 +
19 + def encode(self, text, add_special_tokens=False):
20 + return [ord(c) for c in text]
21 +
22 +
23 +def _request(with_tag: bool) -> ChatCompletionRequest:
24 + req = ChatCompletionRequest(
25 + model="k3",
26 + messages=[{"role": "user", "content": "hi"}],
27 + tools=[
28 + {
29 + "type": "function",
30 + "function": {
31 + "name": "get_weather",
32 + "parameters": {
33 + "type": "object",
34 + "properties": {"city": {"type": "string"}},
35 + "required": ["city"],
36 + },
37 + },
38 + }
39 + ],
40 + tool_choice={"type": "function", "function": {"name": "get_weather"}},
41 + )
42 + if with_tag:
43 + req.structured_outputs = StructuredOutputsParams(
44 + structural_tag='{"type": "structural_tag", "format": {}}'
45 + )
46 + return req
47 +
48 +
49 +def _parser():
50 + from vllm.tool_parsers.kimi_k3_tool_parser import KimiK3ToolParser
51 +
52 + return KimiK3ToolParser(_DummyTokenizer())
53 +
54 +
55 +def test_named_choice_allowed_with_structural_tag():
56 + req = _parser().adjust_request(_request(with_tag=True))
57 + assert req.skip_special_tokens is False
58 +
59 +
60 +def test_named_choice_rejected_without_structural_tag():
61 + with pytest.raises(VLLMValidationError):
62 + _parser().adjust_request(_request(with_tag=False))
tests/tool_parsers/test_structural_tag_registry.py
+341 -1
@@ -5,7 +5,8 @@ from types import SimpleNamespace
5 from unittest.mock import MagicMock
6
7 import pytest
8 -from xgrammar import StructuralTag
8 +from xgrammar import Grammar, StructuralTag
9 +from xgrammar.testing import _is_grammar_accept_string
10
11 from vllm.entrypoints.openai.chat_completion.protocol import (
12 ChatCompletionNamedFunction,
@@ -24,6 +25,7 @@ from vllm.tool_parsers.deepseekv32_engine_tool_parser import (
25 from vllm.tool_parsers.glm47_moe_tool_parser import Glm47MoeModelToolParser
26 from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
27 from vllm.tool_parsers.kimi_k2_tool_parser import KimiK2ToolParser
28 +from vllm.tool_parsers.kimi_k3_tool_parser import KimiK3ToolParser
29 from vllm.tool_parsers.llama_tool_parser import Llama3JsonToolParser
30 from vllm.tool_parsers.minimax_m2_tool_parser import MinimaxM2ToolParser
31 from vllm.tool_parsers.qwen3_engine_tool_parser import Qwen3EngineToolParser
@@ -165,6 +167,196 @@ def test_hermes_required_tool_calls_use_empty_separator():
167 assert tag.format.separator == ""
168
169
170 +# ---------------------------------------------------------------------------
171 +# Kimi K3 (XTML channel format) structural tag
172 +# ---------------------------------------------------------------------------
173 +_K3_RESPONSE_OPEN = "<|open|>response<|sep|>"
174 +_K3_RESPONSE_CLOSE = "<|close|>response<|sep|>"
175 +_K3_TOOLS_OPEN = "<|open|>tools<|sep|>"
176 +_K3_TOOLS_CLOSE = "<|close|>tools<|sep|>"
177 +_K3_CALL_CLOSE = "<|close|>call<|sep|>"
178 +_K3_ARG_CLOSE = "<|close|>argument<|sep|>"
179 +_K3_MESSAGE_CLOSE = "<|close|>message<|sep|>"
180 +
181 +
182 +def _k3_tools_by_name() -> list[ChatCompletionToolsParam]:
183 + return [
184 + ChatCompletionToolsParam(
185 + type="function",
186 + function={
187 + "name": "get_weather",
188 + "parameters": {
189 + "type": "object",
190 + "properties": {
191 + "city": {"type": "string"},
192 + "days": {"type": "integer"},
193 + },
194 + "required": ["city"],
195 + },
196 + },
197 + ),
198 + ChatCompletionToolsParam(
199 + type="function",
200 + function={
201 + "name": "run_command",
202 + "parameters": {
203 + "type": "object",
204 + "properties": {"command": {"type": "string"}},
205 + "required": ["command"],
206 + },
207 + },
208 + ),
209 + ]
210 +
211 +
212 +def _k3_arg(key: str, typ: str, val: str) -> str:
213 + return f'<|open|>argument key="{key}" type="{typ}"<|sep|>{val}{_K3_ARG_CLOSE}'
214 +
215 +
216 +def _k3_call(name: str, args: str, idx: int = 1) -> str:
217 + return f'<|open|>call tool="{name}" index="{idx}"<|sep|>{args}{_K3_CALL_CLOSE}'
218 +
219 +
220 +def _k3_response(content: str = "") -> str:
221 + return f"{_K3_RESPONSE_OPEN}{content}{_K3_RESPONSE_CLOSE}"
222 +
223 +
224 +def _k3_tools(*calls: str) -> str:
225 + return f"{_K3_TOOLS_OPEN}{''.join(calls)}{_K3_TOOLS_CLOSE}"
226 +
227 +
228 +def _k3_grammar(tool_choice, tools=None):
229 + tag = get_model_structural_tag(
230 + model="kimi_k3",
231 + tools=tools if tools is not None else _k3_tools_by_name(),
232 + tool_choice=tool_choice,
233 + reasoning=False,
234 + )
235 + assert isinstance(tag, StructuralTag)
236 + return Grammar.from_structural_tag(tag)
237 +
238 +
239 +def test_kimi_k3_registered_as_vllm_builtin():
240 + assert "kimi_k3" in VLLM_BUILTIN_STRUCTURAL_TAG_MODELS
241 + assert KimiK3ToolParser.structural_tag_model == "kimi_k3"
242 +
243 +
244 +def test_kimi_k3_auto_without_strict_is_unconstrained():
245 + # auto + no strict tool => no structural tag (matches the strict gate).
246 + tag = get_model_structural_tag(
247 + model="kimi_k3",
248 + tools=_k3_tools_by_name(),
249 + tool_choice="auto",
250 + reasoning=False,
251 + )
252 + assert tag is None
253 +
254 +
255 +@pytest.mark.parametrize(
256 + "body",
257 + [
258 + # single required arg
259 + _k3_response()
260 + + _k3_tools(_k3_call("get_weather", _k3_arg("city", "string", "Paris"))),
261 + # response content + two args (string + number)
262 + _k3_response("Checking.")
263 + + _k3_tools(
264 + _k3_call(
265 + "get_weather",
266 + _k3_arg("city", "string", "Paris") + _k3_arg("days", "number", "3"),
267 + )
268 + ),
269 + # args in reverse order (parser is order-agnostic)
270 + _k3_response()
271 + + _k3_tools(
272 + _k3_call(
273 + "get_weather",
274 + _k3_arg("days", "number", "3") + _k3_arg("city", "string", "Paris"),
275 + )
276 + ),
277 + # two calls, second tool
278 + _k3_response()
279 + + _k3_tools(
280 + _k3_call("get_weather", _k3_arg("city", "string", "Paris"), 1),
281 + _k3_call("run_command", _k3_arg("command", "string", "ls -la"), 2),
282 + ),
283 + # string value with regex metacharacters / spaces
284 + _k3_response()
285 + + _k3_tools(
286 + _k3_call(
287 + "run_command", _k3_arg("command", "string", "grep -E 'a|b{2,}' x.py")
288 + )
289 + ),
290 + # trailing message-close marker (model's natural turn terminator)
291 + _k3_response()
292 + + _k3_tools(_k3_call("get_weather", _k3_arg("city", "string", "Paris")))
293 + + _K3_MESSAGE_CLOSE,
294 + # non-thinking mode: response-open is the prompt prefix, so it is absent
295 + _K3_RESPONSE_CLOSE
296 + + _k3_tools(_k3_call("get_weather", _k3_arg("city", "string", "Paris"))),
297 + ],
298 +)
299 +def test_kimi_k3_required_accepts_valid_tool_calls(body: str):
300 + assert _is_grammar_accept_string(_k3_grammar("required"), body)
301 +
302 +
303 +@pytest.mark.parametrize(
304 + "body",
305 + [
306 + # unknown tool name
307 + _k3_response()
308 + + _k3_tools(_k3_call("get_temperature", _k3_arg("city", "string", "x"))),
309 + # number arg given a non-numeric JSON value
310 + _k3_response()
311 + + _k3_tools(_k3_call("get_weather", _k3_arg("days", "number", "abc"))),
312 + # undeclared argument key
313 + _k3_response()
314 + + _k3_tools(
315 + _k3_call(
316 + "get_weather",
317 + _k3_arg("city", "string", "Paris") + _k3_arg("zzz", "string", "x"),
318 + )
319 + ),
320 + # required schema but no argument tags
321 + _k3_response() + _k3_tools(_k3_call("get_weather", "")),
322 + # missing tools close marker
323 + _k3_response()
324 + + _K3_TOOLS_OPEN
325 + + _k3_call("get_weather", _k3_arg("city", "string", "Paris")),
326 + # required but no tool call
327 + _k3_response("hello"),
328 + ],
329 +)
330 +def test_kimi_k3_required_rejects_invalid(body: str):
331 + assert not _is_grammar_accept_string(_k3_grammar("required"), body)
332 +
333 +
334 +def test_kimi_k3_schema_without_required_accepts_empty_call():
335 + tools = [
336 + ChatCompletionToolsParam(
337 + type="function",
338 + function={
339 + "name": "get_weather",
340 + "parameters": {
341 + "type": "object",
342 + "properties": {"city": {"type": "string"}},
343 + },
344 + },
345 + )
346 + ]
347 + grammar = _k3_grammar("required", tools=tools)
348 + body = _k3_response() + _k3_tools(_k3_call("get_weather", ""))
349 +
350 + assert _is_grammar_accept_string(grammar, body)
351 +
352 +
353 +def test_kimi_k3_auto_strict_allows_response_only(sample_tools_strict):
354 + # With a strict tool the tag is built; the tools channel is optional so a
355 + # plain response (no tool call) is still valid.
356 + grammar = _k3_grammar("auto", tools=sample_tools_strict)
357 + assert _is_grammar_accept_string(grammar, _k3_response("Just answering."))
358 +
359 +
360 @pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS))
361 def test_get_model_structural_tag_supports_named_tool_choice(
362 model: str,
@@ -339,3 +531,151 @@ def test_get_function_parameters_relaxes_function_strict_false():
531 )
532
533 assert _get_function_parameters(function) is True
534 +
535 +
536 +def _k3_tools_with_root_defs() -> list[ChatCompletionToolsParam]:
537 + return [
538 + ChatCompletionToolsParam(
539 + type="function",
540 + function={
541 + "name": "make_config",
542 + "parameters": {
543 + "type": "object",
544 + "properties": {
545 + "config": {
546 + "type": "object",
547 + "properties": {
548 + "build": {"$ref": "#/$defs/build"},
549 + "index": {"type": "string"},
550 + },
551 + "required": ["index"],
552 + "additionalProperties": False,
553 + },
554 + },
555 + "required": ["config"],
556 + "$defs": {
557 + "build": {
558 + "type": "object",
559 + "properties": {"outDir": {"type": "string"}},
560 + "additionalProperties": False,
561 + }
562 + },
563 + },
564 + },
565 + )
566 + ]
567 +
568 +
569 +def test_kimi_k3_property_ref_to_root_defs_compiles_and_accepts():
570 + # Root-level $defs referenced from inside a property schema (the walle
571 + # TestReferences shape). Slicing the property out of the parameters
572 + # document orphans "#/$defs/..." unless the builder re-attaches $defs;
573 + # before the fix Grammar.from_structural_tag raised on the dangling ref.
574 + grammar = _k3_grammar("required", tools=_k3_tools_with_root_defs())
575 +
576 + body = _k3_response() + _k3_tools(
577 + _k3_call(
578 + "make_config",
579 + _k3_arg(
580 + "config",
581 + "object",
582 + '{"build": {"outDir": "dist"}, "index": "a.html"}',
583 + ),
584 + )
585 + )
586 + assert _is_grammar_accept_string(grammar, body)
587 +
588 +
589 +def _k3_tools_with_string_enum() -> list[ChatCompletionToolsParam]:
590 + return [
591 + ChatCompletionToolsParam(
592 + type="function",
593 + function={
594 + "name": "set_unit",
595 + "parameters": {
596 + "type": "object",
597 + "properties": {
598 + "unit": {
599 + "type": "string",
600 + "enum": ["celsius", " fahrenheit", "\tkelvin"],
601 + },
602 + },
603 + "required": ["unit"],
604 + },
605 + },
606 + )
607 + ]
608 +
609 +
610 +@pytest.mark.parametrize("value", ["celsius", " fahrenheit", "\tkelvin"])
611 +def test_kimi_k3_string_enum_accepts_exact_values(value: str):
612 + # Raw string channel with enum: constrained to the exact enum values,
613 + # including leading-whitespace variants the model otherwise flubs.
614 + grammar = _k3_grammar("required", tools=_k3_tools_with_string_enum())
615 + body = _k3_response() + _k3_tools(
616 + _k3_call("set_unit", _k3_arg("unit", "string", value))
617 + )
618 + assert _is_grammar_accept_string(grammar, body)
619 +
620 +
621 +@pytest.mark.parametrize("value", ["kelvin", "Celsius", "celsius ", ""])
622 +def test_kimi_k3_string_enum_rejects_non_members(value: str):
623 + grammar = _k3_grammar("required", tools=_k3_tools_with_string_enum())
624 + body = _k3_response() + _k3_tools(
625 + _k3_call("set_unit", _k3_arg("unit", "string", value))
626 + )
627 + assert not _is_grammar_accept_string(grammar, body)
628 +
629 +
630 +def _k3_tools_with_maxlen() -> list[ChatCompletionToolsParam]:
631 + return [
632 + ChatCompletionToolsParam(
633 + type="function",
634 + function={
635 + "name": "set_note",
636 + "parameters": {
637 + "type": "object",
638 + "properties": {
639 + "note": {"type": "string", "maxLength": 8, "minLength": 2},
640 + },
641 + "required": ["note"],
642 + },
643 + },
644 + )
645 + ]
646 +
647 +
648 +def test_kimi_k3_string_maxlength_bounds_raw_channel():
649 + # Raw string channel with maxLength/minLength: enforced via a bounded
650 + # regex that keeps the "<|" marker prefix unambiguous but still allows
651 + # a bare '<' inside values.
652 + grammar = _k3_grammar("required", tools=_k3_tools_with_maxlen())
653 +
654 + def body(val: str) -> str:
655 + return _k3_response() + _k3_tools(
656 + _k3_call("set_note", _k3_arg("note", "string", val))
657 + )
658 +
659 + assert _is_grammar_accept_string(grammar, body("ab"))
660 + assert _is_grammar_accept_string(grammar, body("a<b then"))
661 + assert not _is_grammar_accept_string(grammar, body("way too long note"))
662 + assert not _is_grammar_accept_string(grammar, body("a")) # under minLength
663 +
664 +
665 +def test_kimi_k3_forced_tool_choice_builds_single_mandatory_call():
666 + # Named tool choice normalizes to "forced": the tag must require exactly
667 + # the named tool's call (no response-only escape).
668 + grammar = _k3_grammar(
669 + ChatCompletionNamedToolChoiceParam(
670 + type="function",
671 + function=ChatCompletionNamedFunction(name="get_weather"),
672 + ),
673 + tools=_k3_tools_by_name(),
674 + )
675 +
676 + ok = _k3_response() + _k3_tools(
677 + _k3_call("get_weather", _k3_arg("city", "string", "Paris"))
678 + )
679 + response_only = _k3_response("no call here")
680 + assert _is_grammar_accept_string(grammar, ok)
681 + assert not _is_grammar_accept_string(grammar, response_only)
tests/tool_use/test_kimi_k3_tool_parser.py new
+600
@@ -0,0 +1,600 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import json
5 +
6 +import pytest
7 +
8 +from vllm.entrypoints.openai.chat_completion.protocol import (
9 + ChatCompletionRequest,
10 +)
11 +from vllm.entrypoints.openai.engine.protocol import DeltaMessage
12 +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
13 +from vllm.entrypoints.openai.responses.utils import build_response_output_items
14 +from vllm.exceptions import VLLMValidationError
15 +from vllm.parser.kimi_k3 import KimiK3Parser
16 +from vllm.parser.parser_manager import ParserManager
17 +from vllm.reasoning.kimi_k3_reasoning_parser import KimiK3ReasoningParser
18 +from vllm.tool_parsers.kimi_k3_tool_parser import KimiK3ToolParser
19 +
20 +OPEN = "<|open|>"
21 +CLOSE = "<|close|>"
22 +SEP = "<|sep|>"
23 +THINK_OPEN = f"{OPEN}think{SEP}"
24 +THINK_CLOSE = f"{CLOSE}think{SEP}"
25 +RESPONSE_CLOSE = f"{CLOSE}response{SEP}"
26 +
27 +
28 +class DummyTokenizer:
29 + def get_vocab(self) -> dict[str, int]:
30 + return {}
31 +
32 + def encode(self, text: str, add_special_tokens: bool = False) -> list[int]:
33 + if text == THINK_OPEN:
34 + return [1, 2, 3]
35 + if text == THINK_CLOSE:
36 + return [4, 2, 3]
37 + return [ord(ch) for ch in text]
38 +
39 +
40 +class KimiK3DelegatingParser(KimiK3Parser):
41 + reasoning_parser_cls = KimiK3ReasoningParser
42 + tool_parser_cls = KimiK3ToolParser
43 +
44 +
45 +def test_parser_manager_selects_kimi_k3_parser():
46 + parser_cls = ParserManager.get_parser(
47 + tool_parser_name="kimi_k3",
48 + reasoning_parser_name="kimi_k3",
49 + enable_auto_tools=True,
50 + )
51 +
52 + assert parser_cls is not None
53 + assert issubclass(parser_cls, KimiK3Parser)
54 + assert parser_cls.reasoning_parser_cls is KimiK3ReasoningParser
55 + assert parser_cls.tool_parser_cls is KimiK3ToolParser
56 +
57 +
58 +def _request() -> ChatCompletionRequest:
59 + return ChatCompletionRequest(
60 + model="test-model",
61 + messages=[],
62 + tools=[
63 + {
64 + "type": "function",
65 + "function": {
66 + "name": "calc",
67 + "parameters": {"type": "object", "properties": {}},
68 + },
69 + }
70 + ],
71 + tool_choice="auto",
72 + )
73 +
74 +
75 +def _named_request() -> ChatCompletionRequest:
76 + return ChatCompletionRequest(
77 + model="test-model",
78 + messages=[],
79 + tools=[
80 + {
81 + "type": "function",
82 + "function": {
83 + "name": "calc",
84 + "parameters": {"type": "object", "properties": {}},
85 + },
86 + }
87 + ],
88 + tool_choice={"type": "function", "function": {"name": "calc"}},
89 + )
90 +
91 +
92 +def _responses_request(*, tool_choice="auto") -> ResponsesRequest:
93 + return ResponsesRequest.model_validate(
94 + {
95 + "model": "test-model",
96 + "input": "Call the calc tool.",
97 + "tools": [
98 + {
99 + "type": "function",
100 + "name": "calc",
101 + "parameters": {"type": "object", "properties": {}},
102 + }
103 + ],
104 + "tool_choice": tool_choice,
105 + }
106 + )
107 +
108 +
109 +def _arg(key: str, typ: str, value: str) -> str:
110 + return f'{OPEN}argument key="{key}" type="{typ}"{SEP}{value}{CLOSE}argument{SEP}'
111 +
112 +
113 +def _call(tool: str, index: int, *args: str) -> str:
114 + body = "".join(args)
115 + return f'{OPEN}call tool="{tool}" index="{index}"{SEP}{body}{CLOSE}call{SEP}'
116 +
117 +
118 +def _response(content: str) -> str:
119 + return f"{OPEN}response{SEP}{content}{RESPONSE_CLOSE}"
120 +
121 +
122 +def _tools(*calls: str) -> str:
123 + return f"{OPEN}tools{SEP}{''.join(calls)}{CLOSE}tools{SEP}"
124 +
125 +
126 +def test_extract_tool_calls_with_response_and_typed_arguments():
127 + parser = KimiK3ToolParser(DummyTokenizer())
128 +
129 + output = _response("answer") + _tools(
130 + _call(
131 + "calc",
132 + 1,
133 + _arg("x", "number", "1"),
134 + _arg("flag", "boolean", "true"),
135 + _arg("text", "string", "raw"),
136 + )
137 + )
138 + extracted = parser.extract_tool_calls(output, _request())
139 +
140 + assert extracted.tools_called is True
141 + assert extracted.content == "answer"
142 + assert len(extracted.tool_calls) == 1
143 + tool_call = extracted.tool_calls[0]
144 + assert tool_call.id == "calc:0"
145 + assert tool_call.function.name == "calc"
146 + assert json.loads(tool_call.function.arguments) == {
147 + "x": 1,
148 + "flag": True,
149 + "text": "raw",
150 + }
151 +
152 +
153 +def test_delegating_parser_preserves_tool_calls_after_reasoning():
154 + parser = KimiK3DelegatingParser(DummyTokenizer())
155 + output = (
156 + f"{THINK_OPEN}step{THINK_CLOSE}"
157 + + _response("answer")
158 + + _tools(_call("calc", 1, _arg("x", "number", "1")))
159 + )
160 +
161 + reasoning, content, tool_calls = parser.parse(
162 + output,
163 + _request(),
164 + enable_auto_tools=True,
165 + )
166 +
167 + assert reasoning == "step"
168 + assert content == "answer"
169 + assert tool_calls is not None
170 + assert len(tool_calls) == 1
171 + assert tool_calls[0].id == "calc:0"
172 + assert tool_calls[0].name == "calc"
173 + assert json.loads(tool_calls[0].arguments) == {"x": 1}
174 +
175 +
176 +def test_delegating_parser_required_tool_choice_uses_xtml_parser():
177 + parser = KimiK3DelegatingParser(DummyTokenizer())
178 + request = _request().model_copy(update={"tool_choice": "required"})
179 + output = (
180 + f"{THINK_OPEN}step{THINK_CLOSE}"
181 + + _response("")
182 + + _tools(_call("calc", 1, _arg("x", "number", "1")))
183 + )
184 +
185 + reasoning, content, tool_calls = parser.parse(
186 + output,
187 + request,
188 + enable_auto_tools=True,
189 + )
190 +
191 + assert reasoning == "step"
192 + assert content is None
193 + assert tool_calls is not None
194 + assert len(tool_calls) == 1
195 + assert tool_calls[0].name == "calc"
196 + assert json.loads(tool_calls[0].arguments) == {"x": 1}
197 +
198 +
199 +def test_delegating_parser_named_tool_choice_uses_xtml_parser():
200 + parser = KimiK3DelegatingParser(DummyTokenizer())
201 + output = (
202 + f"{THINK_OPEN}step{THINK_CLOSE}"
203 + + _response("")
204 + + _tools(_call("calc", 1, _arg("x", "number", "1")))
205 + )
206 +
207 + reasoning, content, tool_calls = parser.parse(
208 + output,
209 + _named_request(),
210 + enable_auto_tools=True,
211 + )
212 +
213 + assert reasoning == "step"
214 + assert content is None
215 + assert tool_calls is not None
216 + assert len(tool_calls) == 1
217 + assert tool_calls[0].name == "calc"
218 + assert json.loads(tool_calls[0].arguments) == {"x": 1}
219 +
220 +
221 +def test_delegating_parser_auto_no_call_strips_consumed_response_prefix():
222 + parser = KimiK3DelegatingParser(
223 + DummyTokenizer(), chat_template_kwargs={"thinking": False}
224 + )
225 + request = _request().model_copy(
226 + update={"chat_template_kwargs": {"thinking": False}}
227 + )
228 +
229 + reasoning, content, tool_calls = parser.parse(
230 + f"answer{RESPONSE_CLOSE}",
231 + request,
232 + enable_auto_tools=True,
233 + )
234 +
235 + assert reasoning is None
236 + assert content == "answer"
237 + assert tool_calls is None
238 +
239 +
240 +def test_delegating_parser_required_call_strips_consumed_response_prefix():
241 + parser = KimiK3DelegatingParser(
242 + DummyTokenizer(), chat_template_kwargs={"thinking": False}
243 + )
244 + request = _request().model_copy(
245 + update={
246 + "tool_choice": "required",
247 + "chat_template_kwargs": {"thinking": False},
248 + }
249 + )
250 + output = RESPONSE_CLOSE + _tools(_call("calc", 1, _arg("x", "number", "1")))
251 +
252 + reasoning, content, tool_calls = parser.parse(
253 + output,
254 + request,
255 + enable_auto_tools=True,
256 + )
257 +
258 + assert reasoning is None
259 + assert content is None
260 + assert tool_calls is not None
261 + assert len(tool_calls) == 1
262 + assert tool_calls[0].name == "calc"
263 + assert json.loads(tool_calls[0].arguments) == {"x": 1}
264 +
265 +
266 +def test_delegating_parser_truncated_tools_do_not_leak_xtml():
267 + parser = KimiK3DelegatingParser(
268 + DummyTokenizer(), chat_template_kwargs={"thinking": False}
269 + )
270 + request = _request().model_copy(
271 + update={
272 + "tool_choice": "required",
273 + "chat_template_kwargs": {"thinking": False},
274 + }
275 + )
276 +
277 + reasoning, content, tool_calls = parser.parse(
278 + (f'{RESPONSE_CLOSE}{OPEN}tools{SEP}{OPEN}call tool="calc" index="1"'),
279 + request,
280 + enable_auto_tools=True,
281 + )
282 +
283 + assert reasoning is None
284 + assert content is None
285 + assert tool_calls is None
286 +
287 +
288 +def test_extract_tool_calls_unescapes_attributes():
289 + parser = KimiK3ToolParser(DummyTokenizer())
290 +
291 + output = _tools(_call("a&amp;b&quot;c", 1, _arg("k&amp;q", "string", "v")))
292 + extracted = parser.extract_tool_calls(output, _request())
293 +
294 + assert extracted.tools_called is True
295 + assert extracted.tool_calls[0].function.name == 'a&b"c'
296 + assert json.loads(extracted.tool_calls[0].function.arguments) == {"k&q": "v"}
297 +
298 +
299 +def test_extract_tool_calls_allows_less_than_in_attributes():
300 + parser = KimiK3ToolParser(DummyTokenizer())
301 +
302 + output = _tools(_call("calc<beta", 1, _arg("foo<bar", "string", "raw")))
303 + extracted = parser.extract_tool_calls(output, _request())
304 +
305 + assert extracted.tools_called is True
306 + assert extracted.tool_calls[0].function.name == "calc<beta"
307 + assert json.loads(extracted.tool_calls[0].function.arguments) == {"foo<bar": "raw"}
308 +
309 +
310 +def test_extract_content_from_whitespace_degraded_markers():
311 + parser = KimiK3ToolParser(DummyTokenizer())
312 +
313 + extracted = parser.extract_tool_calls(
314 + f"{OPEN} response {SEP}answer{CLOSE} response {SEP}",
315 + _request(),
316 + )
317 +
318 + assert extracted.tools_called is False
319 + assert extracted.content == "answer"
320 +
321 +
322 +def test_streaming_split_markers_do_not_leak():
323 + parser = KimiK3ToolParser(DummyTokenizer())
324 + request = _request()
325 + previous_text = ""
326 + previous_ids: list[int] = []
327 + messages: list[DeltaMessage] = []
328 + chunks = [
329 + OPEN,
330 + "response",
331 + f"{SEP}Hi",
332 + OPEN,
333 + "tools",
334 + SEP,
335 + f'{OPEN}call tool="calc" index="1"{SEP}',
336 + _arg("x", "number", "1"),
337 + f"{CLOSE}call",
338 + SEP,
339 + ]
340 +
341 + for i, chunk in enumerate(chunks, start=1):
342 + current_text = previous_text + chunk
343 + current_ids = previous_ids + [i]
344 + delta = parser.extract_tool_calls_streaming(
345 + previous_text=previous_text,
346 + current_text=current_text,
347 + delta_text=chunk,
348 + previous_token_ids=previous_ids,
349 + current_token_ids=current_ids,
350 + delta_token_ids=[i],
351 + request=request,
352 + )
353 + if delta is not None:
354 + messages.append(delta)
355 + previous_text = current_text
356 + previous_ids = current_ids
357 +
358 + content = "".join(message.content or "" for message in messages)
359 + tool_deltas = [
360 + tool_call for message in messages for tool_call in (message.tool_calls or [])
361 + ]
362 +
363 + assert content == "Hi"
364 + assert OPEN not in content
365 + assert SEP not in content
366 + assert len(tool_deltas) == 1
367 + assert tool_deltas[0].id == "calc:0"
368 + assert tool_deltas[0].function.name == "calc"
369 + assert json.loads(tool_deltas[0].function.arguments) == {"x": 1}
370 +
371 +
372 +def test_streaming_consumed_response_prefix_no_call_keeps_content():
373 + parser = KimiK3ToolParser(DummyTokenizer())
374 + request = _request()
375 + previous_text = ""
376 + previous_ids: list[int] = []
377 + messages: list[DeltaMessage] = []
378 + chunks = ["O", "K", CLOSE, f"response{SEP}"]
379 +
380 + for i, chunk in enumerate(chunks, start=1):
381 + current_text = previous_text + chunk
382 + current_ids = previous_ids + [i]
383 + delta = parser.extract_tool_calls_streaming(
384 + previous_text=previous_text,
385 + current_text=current_text,
386 + delta_text=chunk,
387 + previous_token_ids=previous_ids,
388 + current_token_ids=current_ids,
389 + delta_token_ids=[i],
390 + request=request,
391 + )
392 + if delta is not None:
393 + messages.append(delta)
394 + previous_text = current_text
395 + previous_ids = current_ids
396 +
397 + assert "".join(message.content or "" for message in messages) == "OK"
398 + assert all(CLOSE not in (message.content or "") for message in messages)
399 +
400 +
401 +def test_delegating_parser_tool_choice_none_strips_xtml_and_suppresses_calls():
402 + parser = KimiK3DelegatingParser(
403 + DummyTokenizer(), chat_template_kwargs={"thinking": False}
404 + )
405 + request = _request().model_copy(
406 + update={
407 + "tool_choice": "none",
408 + "chat_template_kwargs": {"thinking": False},
409 + }
410 + )
411 + messages: list[DeltaMessage] = []
412 + chunks = [
413 + OPEN,
414 + "response",
415 + f"{SEP}answer",
416 + RESPONSE_CLOSE,
417 + _tools(_call("calc", 1, _arg("x", "number", "1"))),
418 + ]
419 +
420 + for index, chunk in enumerate(chunks, start=1):
421 + delta = parser.parse_delta(
422 + delta_text=chunk,
423 + delta_token_ids=[index],
424 + request=request,
425 + prompt_token_ids=[1],
426 + finished=index == len(chunks),
427 + )
428 + if delta is not None:
429 + messages.append(delta)
430 +
431 + content = "".join(message.content or "" for message in messages)
432 + assert content == "answer"
433 + assert OPEN not in content
434 + assert CLOSE not in content
435 + assert SEP not in content
436 + assert all(not message.tool_calls for message in messages)
437 +
438 +
439 +def test_adjust_request_keeps_xtml_markers_contiguous():
440 + parser = KimiK3ToolParser(DummyTokenizer())
441 + request = _request()
442 +
443 + adjusted = parser.adjust_request(request)
444 +
445 + assert adjusted.skip_special_tokens is False
446 + if hasattr(adjusted, "spaces_between_special_tokens"):
447 + assert adjusted.spaces_between_special_tokens is False
448 + assert KimiK3ToolParser.supports_required_and_named is False
449 +
450 +
451 +def test_adjust_request_required_uses_xtml_parser_not_json_guidance():
452 + parser = KimiK3ToolParser(DummyTokenizer())
453 + request = _request().model_copy(update={"tool_choice": "required"})
454 +
455 + adjusted = parser.adjust_request(request)
456 +
457 + assert adjusted.structured_outputs is None
458 + assert adjusted.skip_special_tokens is False
459 + if hasattr(adjusted, "spaces_between_special_tokens"):
460 + assert adjusted.spaces_between_special_tokens is False
461 +
462 +
463 +@pytest.mark.parametrize(
464 + "tool_request",
465 + [
466 + _named_request(),
467 + _responses_request(
468 + tool_choice={"type": "function", "name": "calc"},
469 + ),
470 + ],
471 +)
472 +def test_adjust_request_rejects_named_tool_choice(tool_request):
473 + parser = KimiK3ToolParser(DummyTokenizer())
474 +
475 + with pytest.raises(VLLMValidationError) as exc_info:
476 + parser.adjust_request(tool_request)
477 +
478 + assert exc_info.value.parameter == "tool_choice"
479 + assert "requires strict tool calling" in str(exc_info.value)
480 +
481 +
482 +def test_responses_chat_params_carries_tool_choice_metadata():
483 + request = _responses_request(tool_choice="required")
484 +
485 + chat_params = request.build_chat_params(
486 + default_template=None,
487 + default_template_content_format="auto",
488 + )
489 +
490 + assert chat_params.tool_choice == "required"
491 +
492 +
493 +def test_responses_chat_params_keeps_template_tool_choice_when_api_auto():
494 + request = _responses_request().model_copy(
495 + update={"chat_template_kwargs": {"tool_choice": "required"}}
496 + )
497 +
498 + chat_params = request.build_chat_params(
499 + default_template=None,
500 + default_template_content_format="auto",
501 + )
502 +
503 + assert chat_params.chat_template_kwargs["tool_choice"] == "required"
504 + assert chat_params.tool_choice == "auto"
505 +
506 +
507 +def test_responses_required_tool_choice_uses_xtml_parser():
508 + parser = KimiK3DelegatingParser(
509 + DummyTokenizer(), chat_template_kwargs={"thinking": False}
510 + )
511 + request = _responses_request(tool_choice="required").model_copy(
512 + update={"chat_template_kwargs": {"thinking": False}}
513 + )
514 + output = RESPONSE_CLOSE + _tools(_call("calc", 1, _arg("x", "number", "1")))
515 +
516 + reasoning, content, tool_calls = parser.parse(
517 + output, request, enable_auto_tools=True, model_output_token_ids=[]
518 + )
519 + response_outputs = build_response_output_items(
520 + reasoning=reasoning,
521 + content=content,
522 + tool_calls=tool_calls,
523 + tools=request.tools,
524 + )
525 +
526 + assert len(response_outputs) == 1
527 + tool_call = response_outputs[0]
528 + assert tool_call.type == "function_call"
529 + assert tool_call.name == "calc"
530 + assert json.loads(tool_call.arguments) == {"x": 1}
531 +
532 +
533 +def test_responses_named_tool_choice_uses_xtml_parser():
534 + parser = KimiK3DelegatingParser(
535 + DummyTokenizer(), chat_template_kwargs={"thinking": False}
536 + )
537 + request = _responses_request(
538 + tool_choice={"type": "function", "name": "calc"}
539 + ).model_copy(update={"chat_template_kwargs": {"thinking": False}})
540 + output = RESPONSE_CLOSE + _tools(_call("calc", 1, _arg("x", "number", "1")))
541 +
542 + reasoning, content, tool_calls = parser.parse(
543 + output, request, enable_auto_tools=True, model_output_token_ids=[]
544 + )
545 + response_outputs = build_response_output_items(
546 + reasoning=reasoning,
547 + content=content,
548 + tool_calls=tool_calls,
549 + tools=request.tools,
550 + )
551 +
552 + assert len(response_outputs) == 1
553 + tool_call = response_outputs[0]
554 + assert tool_call.type == "function_call"
555 + assert tool_call.name == "calc"
556 + assert json.loads(tool_call.arguments) == {"x": 1}
557 +
558 +
559 +def test_chat_params_carries_tool_choice_metadata():
560 + request = _request().model_copy(update={"tool_choice": "required"})
561 +
562 + chat_params = request.build_chat_params(
563 + default_template=None,
564 + default_template_content_format="auto",
565 + )
566 +
567 + assert chat_params.tool_choice == "required"
568 +
569 +
570 +def test_chat_params_carries_response_format_metadata():
571 + request = ChatCompletionRequest(
572 + model="test-model",
573 + messages=[],
574 + response_format={"type": "json_object"},
575 + )
576 +
577 + chat_params = request.build_chat_params(
578 + default_template=None,
579 + default_template_content_format="auto",
580 + )
581 +
582 + assert chat_params.response_format is request.response_format
583 + assert chat_params.tool_choice is None
584 +
585 +
586 +def test_chat_params_keeps_template_tool_choice_when_api_auto():
587 + request = _request().model_copy(
588 + update={
589 + "tool_choice": "auto",
590 + "chat_template_kwargs": {"tool_choice": "required"},
591 + }
592 + )
593 +
594 + chat_params = request.build_chat_params(
595 + default_template=None,
596 + default_template_content_format="auto",
597 + )
598 +
599 + assert chat_params.chat_template_kwargs["tool_choice"] == "required"
600 + assert chat_params.tool_choice == "auto"
vllm/config/model.py
+12 -1
@@ -84,7 +84,14 @@ RunnerOption = Literal["auto", RunnerType]
84 ConvertType = Literal["none", "embed", "classify"]
85 ConvertOption = Literal["auto", ConvertType]
86 TokenizerMode = Literal[
87 - "auto", "hf", "slow", "mistral", "deepseek_v32", "deepseek_v4", "inkling"
87 + "auto",
88 + "hf",
89 + "slow",
90 + "mistral",
91 + "deepseek_v32",
92 + "deepseek_v4",
93 + "inkling",
94 + "kimi_k3",
95 ]
96 ModelDType = Literal["auto", "half", "float16", "bfloat16", "float", "float32"]
97 LogprobsMode = Literal[
@@ -141,6 +148,8 @@ class ModelConfig:
148 - "mistral" will always use the tokenizer from `mistral_common`.
149 - "deepseek_v32" will always use the tokenizer from `deepseek_v32`.
150 - "deepseek_v4" will always use the tokenizer from `deepseek_v4`.
151 + - "kimi_k3" will always use the "hf" tokenizer but render chat prompts
152 + with Kimi K3's Python XTML encoding instead of a Jinja template.
153 - Other custom values can be supported via plugins.
154
155 To swap the Rust BPE backend that powers HF fast tokenizers for the
@@ -634,6 +643,8 @@ class ModelConfig:
643 self.tokenizer_mode = "terratorch"
644 elif arch == "MoonshotKimiaForCausalLM":
645 self.tokenizer_mode = "kimi_audio"
646 + elif arch == "KimiK3ForConditionalGeneration":
647 + self.tokenizer_mode = "kimi_k3"
648 elif arch == "DeepseekV32ForCausalLM":
649 self.tokenizer_mode = "deepseek_v32"
650 elif arch == "DeepseekV4ForCausalLM":
vllm/entrypoints/chat_utils.py
+8 -2
@@ -2024,13 +2024,19 @@ def get_history_tool_calls_cnt(conversation: list[ConversationMessage]):
2024 return idx
2025
2026
2027 -_KIMI_MODEL_TYPES = ("kimi_k2", "kimi_k25")
2027 +_KIMI_MODEL_TYPES = ("kimi_k2", "kimi_k25", "kimi_k3")
2028
2029
2030 def get_tool_call_id_type(model_config: ModelConfig) -> str:
2031 """Return the tool-call ID type for a given model configuration."""
2032 hf_overrides = getattr(model_config, "hf_overrides", None)
2033 - if model_config.hf_text_config.model_type in _KIMI_MODEL_TYPES or (
2033 + hf_config = getattr(model_config, "hf_config", None)
2034 + hf_text_config = getattr(model_config, "hf_text_config", None)
2035 + model_types = (
2036 + getattr(hf_config, "model_type", None),
2037 + getattr(hf_text_config, "model_type", None),
2038 + )
2039 + if any(model_type in _KIMI_MODEL_TYPES for model_type in model_types) or (
2040 isinstance(hf_overrides, dict)
2041 and hf_overrides.get("model_type") in _KIMI_MODEL_TYPES
2042 ):
vllm/entrypoints/openai/chat_completion/protocol.py
+6
@@ -565,6 +565,12 @@ class ChatCompletionRequest(OpenAIBaseModel):
565 ),
566 media_io_kwargs=self.media_io_kwargs,
567 return_assistant_tokens_mask=bool(self.return_assistant_tokens_mask),
568 + # No-tools requests default to tool_choice="none" at the API
569 + # layer. Collapse that default before rendering, so K3 emits a
570 + # model-visible tool-choice instruction only for requests with a
571 + # tools block.
572 + tool_choice=self.tool_choice if self.tools else None,
573 + response_format=self.response_format,
574 )
575
576 def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
vllm/entrypoints/openai/responses/protocol.py
+1
@@ -337,6 +337,7 @@ class ResponsesRequest(OpenAIBaseModel):
337 extra_kwargs,
338 ),
339 media_io_kwargs=self.media_io_kwargs,
340 + tool_choice=self.tool_choice if self.tools else None,
341 )
342
343 def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
vllm/parser/kimi_k3.py new
+135
@@ -0,0 +1,135 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from __future__ import annotations
5 +
6 +from collections.abc import Sequence
7 +from typing import TYPE_CHECKING
8 +
9 +from vllm.entrypoints.openai.engine.protocol import (
10 + DeltaMessage,
11 + FunctionCall,
12 +)
13 +from vllm.parser.abstract_parser import DelegatingParser
14 +from vllm.reasoning.kimi_k3_reasoning_parser import KimiK3ReasoningParser
15 +
16 +if TYPE_CHECKING:
17 + from vllm.entrypoints.openai.chat_completion.protocol import (
18 + ChatCompletionRequest,
19 + )
20 + from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
21 +
22 +
23 +class KimiK3Parser(DelegatingParser):
24 + """Compose the Kimi K3 reasoning and tool parsers for XTML output."""
25 +
26 + # TODO: Switch Kimi K3 to the parser engine once its XTML reasoning/tool
27 + # path is covered there.
28 + def _extract_tool_calls(
29 + self,
30 + content: str | None,
31 + request: ChatCompletionRequest | ResponsesRequest,
32 + enable_auto_tools: bool = False,
33 + ) -> tuple[list[FunctionCall] | None, str | None]:
34 + if self._tool_parser is None or not enable_auto_tools:
35 + return super()._extract_tool_calls(content, request, enable_auto_tools)
36 +
37 + tool_call_info = self.extract_tool_calls(content or "", request=request)
38 + if request.tool_choice == "none":
39 + return [], tool_call_info.content
40 + if not tool_call_info.tools_called:
41 + return None, tool_call_info.content
42 +
43 + tool_calls = [
44 + FunctionCall(
45 + id=tool_call.id,
46 + name=tool_call.function.name,
47 + arguments=tool_call.function.arguments,
48 + )
49 + for tool_call in tool_call_info.tool_calls
50 + ]
51 + parsed_content = tool_call_info.content
52 + if parsed_content and parsed_content.strip() == "":
53 + parsed_content = None
54 + return tool_calls, parsed_content
55 +
56 + def _extract_tool_calls_streaming(
57 + self,
58 + previous_text: str,
59 + current_text: str,
60 + delta_text: str,
61 + previous_token_ids: Sequence[int],
62 + current_token_ids: Sequence[int],
63 + delta_token_ids: Sequence[int],
64 + request: ChatCompletionRequest | ResponsesRequest,
65 + tool_call_idx: int | None = None,
66 + tool_call_id_type: str = "random",
67 + function_name_returned: bool = False,
68 + ) -> tuple[DeltaMessage | None, bool]:
69 + if request.tool_choice != "none":
70 + return super()._extract_tool_calls_streaming(
71 + previous_text,
72 + current_text,
73 + delta_text,
74 + previous_token_ids,
75 + current_token_ids,
76 + delta_token_ids,
77 + request,
78 + tool_call_idx=tool_call_idx,
79 + tool_call_id_type=tool_call_id_type,
80 + function_name_returned=function_name_returned,
81 + )
82 +
83 + delta_message = self.extract_tool_calls_streaming(
84 + previous_text,
85 + current_text,
86 + delta_text,
87 + previous_token_ids,
88 + current_token_ids,
89 + delta_token_ids,
90 + request,
91 + )
92 + if delta_message is not None:
93 + delta_message.tool_calls = []
94 + return delta_message, False
95 +
96 + def parse_delta(
97 + self,
98 + delta_text: str,
99 + delta_token_ids: list[int],
100 + request: ChatCompletionRequest | ResponsesRequest,
101 + prompt_token_ids: list[int] | None = None,
102 + *,
103 + finished: bool,
104 + ) -> DeltaMessage | None:
105 + state = self._stream_state
106 + previous_content = state.previous_text if state.reasoning_ended else ""
107 + delta_message = super().parse_delta(
108 + delta_text,
109 + delta_token_ids,
110 + request,
111 + prompt_token_ids,
112 + finished=finished,
113 + )
114 +
115 + if (
116 + self._tool_parser is not None
117 + or not isinstance(self._reasoning_parser, KimiK3ReasoningParser)
118 + or not state.reasoning_ended
119 + or delta_message is None
120 + ):
121 + return delta_message
122 +
123 + stripped = self._reasoning_parser.strip_content_streaming(
124 + previous_text=previous_content,
125 + current_text=state.previous_text,
126 + )
127 + delta_message.content = stripped.content if stripped is not None else None
128 + if (
129 + delta_message.role is None
130 + and delta_message.content is None
131 + and delta_message.reasoning is None
132 + and not delta_message.tool_calls
133 + ):
134 + return None
135 + return delta_message
vllm/parser/parser_manager.py
+12
@@ -125,6 +125,18 @@ class ParserManager:
125 MistralParser.tool_parser_cls = tool_parser_cls
126 return MistralParser
127
128 + if reasoning_parser_name == "kimi_k3" or tool_parser_name == "kimi_k3":
129 + from vllm.parser.kimi_k3 import KimiK3Parser
130 +
131 + r_cls = reasoning_parser_cls
132 + t_cls = tool_parser_cls
133 +
134 + class _KimiK3Parser(KimiK3Parser):
135 + reasoning_parser_cls = r_cls
136 + tool_parser_cls = t_cls
137 +
138 + return _KimiK3Parser
139 +
140 from vllm.parser.abstract_parser import DelegatingParser
141
142 r_cls = reasoning_parser_cls
vllm/reasoning/__init__.py
+4
@@ -84,6 +84,10 @@ _REASONING_PARSERS_TO_REGISTER = {
84 "kimi_k2_reasoning_parser",
85 "KimiK2ReasoningParser",
86 ),
87 + "kimi_k3": (
88 + "kimi_k3_reasoning_parser",
89 + "KimiK3ReasoningParser",
90 + ),
91 "mimo": (
92 "qwen3_engine_reasoning_parser",
93 "Qwen3ParserReasoningAdapter",
vllm/reasoning/kimi_k3_reasoning_parser.py new
+371
@@ -0,0 +1,371 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Reasoning parser for the Kimi K3 (XTML) chat format.
4 +
5 +This strips the ``think`` channel out of generated text and hands the remainder
6 +(``response`` + ``tools`` channels) downstream. Kimi K3 wraps the thinking
7 +channel as an XTML element built from special tokens::
8 +
9 + <|open|>think<|sep|> <reasoning> <|close|>think<|sep|>
10 +
11 +Two subtleties drive the implementation:
12 + * Unlike Kimi-K2 (a single ``<think>`` token), each K3 marker is a 3-token
13 + sequence, so the token-id helpers search for the marker *subsequence*
14 + rather than a single id.
15 + * In thinking mode the serving layer may feed ``<|open|>think<|sep|>`` as the
16 + generation prefix, so the model's output can begin *inside* the think
17 + channel with no open marker. The text paths therefore treat a missing open
18 + marker as "reasoning starts at offset 0".
19 +
20 +When thinking is disabled (``chat_template_kwargs={"thinking": False}`` or
21 +``{"enable_thinking": False}``, i.e. instruct mode) the parser returns every
22 +delta as normal content; there is simply no think channel to extract.
23 +"""
24 +
25 +from collections.abc import Sequence
26 +from typing import TYPE_CHECKING
27 +
28 +import regex as re
29 +from transformers import PreTrainedTokenizerBase
30 +
31 +from vllm.entrypoints.openai.engine.protocol import DeltaMessage
32 +from vllm.reasoning import ReasoningParser
33 +
34 +if TYPE_CHECKING:
35 + from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
36 + from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
37 +
38 +
39 +def _subseq_index(haystack: Sequence[int], needle: Sequence[int]) -> int:
40 + """Return start index of the last occurrence of needle in haystack, or -1."""
41 + n = len(needle)
42 + if n == 0:
43 + return -1
44 + for i in range(len(haystack) - n, -1, -1):
45 + if list(haystack[i : i + n]) == list(needle):
46 + return i
47 + return -1
48 +
49 +
50 +class KimiK3ReasoningParser(ReasoningParser):
51 + """Reasoning parser for the Kimi K3 (XTML) think channel."""
52 +
53 + def __init__(self, tokenizer: PreTrainedTokenizerBase, *args, **kwargs):
54 + super().__init__(tokenizer)
55 +
56 + if not self.model_tokenizer:
57 + raise ValueError(
58 + "The model tokenizer must be passed to the ReasoningParser "
59 + "constructor during construction."
60 + )
61 +
62 + # thinking can be disabled via chat_template_kwargs -> identity fallthrough
63 + chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {}
64 + thinking = chat_kwargs.get("thinking", None)
65 + if thinking is None:
66 + thinking = chat_kwargs.get("enable_thinking", True)
67 + self._thinking_enabled = bool(thinking)
68 +
69 + # XTML markers as literal strings (skip_special_tokens=False at serve time)
70 + self._think_open = "<|open|>think<|sep|>"
71 + self._think_close = "<|close|>think<|sep|>"
72 +
73 + # Content-channel markers that must be stripped from the final output.
74 + # The tool parser handles these when active, but when tool_choice is
75 + # "none" (the default for requests without tools) the tool parser is
76 + # bypassed and the reasoning parser must strip them itself.
77 + self._response_open = "<|open|>response<|sep|>"
78 + self._response_close = "<|close|>response<|sep|>"
79 + self._message_close = "<|close|>message<|sep|>"
80 +
81 + # Tolerant matchers: defense-in-depth against vLLM's added-token spacing
82 + # ("<|open|> think <|sep|>"). The `\s*` is a no-op on clean input, so the
83 + # normal path stays byte-exact; it only helps if some serving path leaves
84 + # spaces_between_special_tokens on. See adjust_request for the real fix.
85 + open_marker = r"<\|open\|>"
86 + close_marker = r"<\|close\|>"
87 + sep_marker = r"<\|sep\|>"
88 + self._think_open_re = re.compile(open_marker + r"\s*think\s*" + sep_marker)
89 + self._think_close_re = re.compile(close_marker + r"\s*think\s*" + sep_marker)
90 + self._response_open_re = re.compile(
91 + open_marker + r"\s*response\s*" + sep_marker
92 + )
93 + self._response_close_re = re.compile(
94 + close_marker + r"\s*response\s*" + sep_marker
95 + )
96 + self._message_close_re = re.compile(
97 + close_marker + r"\s*message\s*" + sep_marker
98 + )
99 +
100 + # marker token-id subsequences (each marker is 3 tokens)
101 + self._think_open_ids = tokenizer.encode(
102 + self._think_open, add_special_tokens=False
103 + )
104 + self._think_close_ids = tokenizer.encode(
105 + self._think_close, add_special_tokens=False
106 + )
107 + self._last_streaming_delta_token_ids: tuple[int, ...] | None = None
108 + self._last_streaming_content_token_ids: list[int] | None = None
109 +
110 + @property
111 + def reasoning_start_str(self) -> str | None:
112 + return self._think_open
113 +
114 + @property
115 + def reasoning_end_str(self) -> str | None:
116 + return self._think_close
117 +
118 + def adjust_request(
119 + self,
120 + request: "ChatCompletionRequest | ResponsesRequest",
121 + ) -> "ChatCompletionRequest | ResponsesRequest":
122 + request.skip_special_tokens = False
123 + if hasattr(request, "spaces_between_special_tokens"):
124 + request.spaces_between_special_tokens = False
125 + return request
126 +
127 + def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
128 + if not self._thinking_enabled:
129 + return True
130 + # Reasoning has ended only if the *most recent* think block is closed:
131 + # the last close marker must come after the last open marker. A plain
132 + # "a close marker exists anywhere" check false-positives in multi-turn /
133 + # agent continuations, where the chat template keeps a prior turn's
134 + # think channel (with its <|close|>think<|sep|>) in the prompt while the
135 + # current turn is still reasoning (its <|open|>think<|sep|> is the newest
136 + # marker). See _subseq_index (returns the last occurrence).
137 + last_close = _subseq_index(input_ids, self._think_close_ids)
138 + last_open = _subseq_index(input_ids, self._think_open_ids)
139 + if last_open == -1:
140 + # No open marker in scope (e.g. the open was consumed as the
141 + # generation prefix): a close marker means reasoning has ended.
142 + return last_close != -1
143 + return last_close > last_open
144 +
145 + def _extract_content_ids(self, input_ids: list[int]) -> list[int]:
146 + if not self._thinking_enabled:
147 + return input_ids
148 + idx = _subseq_index(input_ids, self._think_close_ids)
149 + if idx == -1:
150 + return [] # still reasoning
151 + return input_ids[idx + len(self._think_close_ids) :]
152 +
153 + def extract_content_ids(self, input_ids: list[int]) -> list[int]:
154 + cached_delta_ids = self._last_streaming_delta_token_ids
155 + cached_content_ids = self._last_streaming_content_token_ids
156 + self._last_streaming_delta_token_ids = None
157 + self._last_streaming_content_token_ids = None
158 + if cached_delta_ids == tuple(input_ids) and cached_content_ids is not None:
159 + return cached_content_ids
160 + return self._extract_content_ids(input_ids)
161 +
162 + def _strip_content_wrapper(self, text: str) -> str:
163 + """Strip ``<|open|>response<|sep|>…<|close|>response<|sep|>`` wrapper and
164 + ``<|close|>message<|sep|>`` from *text*.
165 +
166 + When Kimi K3 tool parsing is active it needs the raw XTML
167 + ``response`` + ``tools`` channels. Otherwise the reasoning parser cleans
168 + up the response wrapper itself so API users do not see XTML markers.
169 + """
170 + # Try structured unwrap first: extract body of response channel
171 + m_ro = self._response_open_re.search(text)
172 + m_rc = self._response_close_re.search(text, m_ro.end() if m_ro else 0)
173 + if m_ro is not None and m_rc is not None:
174 + text = text[m_ro.end() : m_rc.start()]
175 + elif m_ro is not None:
176 + # response opened but not closed (truncated)
177 + text = text[m_ro.end() :]
178 + else:
179 + # No response wrapper — strip stray markers if present
180 + text = self._response_open_re.sub("", text)
181 + text = self._response_close_re.sub("", text)
182 + text = self._message_close_re.sub("", text)
183 + return text
184 +
185 + @staticmethod
186 + def _should_preserve_tool_channels(
187 + request: "ChatCompletionRequest | ResponsesRequest",
188 + ) -> bool:
189 + return bool(getattr(request, "tools", None)) and (
190 + getattr(request, "tool_choice", None) != "none"
191 + )
192 +
193 + def _content_after_reasoning(
194 + self,
195 + text: str,
196 + request: "ChatCompletionRequest | ResponsesRequest",
197 + ) -> str | None:
198 + if self._should_preserve_tool_channels(request):
199 + return text or None
200 + return self._strip_content_wrapper(text) or None
201 +
202 + def extract_reasoning(
203 + self, model_output: str, request: "ChatCompletionRequest | ResponsesRequest"
204 + ) -> tuple[str | None, str | None]:
205 + """Split full text into ``(reasoning, rest)`` for the non-streaming path.
206 +
207 + Handles three shapes:
208 + * no think channel at all -> ``(None, model_output)`` (all content)
209 + * open marker present -> reasoning starts after ``<|open|>think<|sep|>``
210 + * open marker absent but a close marker exists (gen-prefix consumed
211 + the open) -> reasoning starts at offset 0
212 + ``rest`` is whatever follows the close marker, fed on to the tool parser.
213 + """
214 + if not self._thinking_enabled:
215 + return None, self._content_after_reasoning(model_output, request)
216 +
217 + m_open = self._think_open_re.search(model_output)
218 + # reasoning content begins right after think-open (or at start if the
219 + # open marker was already consumed as a generation prefix)
220 + content_start = m_open.end() if m_open is not None else 0
221 + # if there is no think channel at all, everything is content
222 + if m_open is None and self._think_close_re.search(model_output) is None:
223 + return None, self._content_after_reasoning(model_output, request)
224 +
225 + m_close = self._think_close_re.search(model_output, content_start)
226 + if m_close is not None:
227 + reasoning = model_output[content_start : m_close.start()]
228 + rest = model_output[m_close.end() :]
229 + return (reasoning or None, self._content_after_reasoning(rest, request))
230 + # think not closed -> still reasoning, no content yet
231 + return (model_output[content_start:] or None, None)
232 +
233 + def _reasoning_text_ready_to_emit(self, text: str) -> str:
234 + """Return the reasoning prefix that is safe to stream now.
235 +
236 + Work from accumulated text, not from the current delta alone. That turns
237 + split-open and split-close handling into the same prefix-diff problem.
238 +
239 + Example 1: split think-open marker.
240 + chunks: ``<|open|>`` / ``think`` / ``<|sep|>reasoning``
241 + current text after chunk 1: ``<|open|>`` -> emit ``""``
242 + current text after chunk 2: ``<|open|>think`` -> emit ``""``
243 + current text after chunk 3: ``<|open|>think<|sep|>reasoning``
244 + -> emit ``reasoning``
245 +
246 + Example 2: split think-close marker.
247 + chunks: ``reasoning`` / ``<|close|>`` / ``think<|sep|>...``
248 + after ``<|close|>``, the suffix is only a partial close marker, so the
249 + sendable reasoning is still ``reasoning`` and the delta is empty. Once
250 + ``think<|sep|>`` arrives, the close branch hands the following response
251 + or tools text to the downstream parser.
252 + """
253 + m_open = self._think_open_re.search(text)
254 + if m_open is not None:
255 + text = text[m_open.end() :]
256 + overlap = 0
257 + for marker in (self._think_open, self._think_close):
258 + max_check = min(len(marker) - 1, len(text))
259 + for n in range(max_check, 0, -1):
260 + if text.endswith(marker[:n]):
261 + overlap = max(overlap, n)
262 + break
263 + return text[:-overlap] if overlap else text
264 +
265 + def _content_ready_to_emit(self, text: str) -> str:
266 + """Return the content prefix that is safe to stream now.
267 +
268 + Mirrors ``_reasoning_text_ready_to_emit`` but for the post-reasoning
269 + content phase. Strips the ``<|open|>response<|sep|>`` prefix, holds back
270 + any partial marker suffix, and removes complete
271 + ``<|close|>response<|sep|>`` / ``<|close|>message<|sep|>`` markers.
272 + """
273 + # Strip response-open prefix
274 + m_open = self._response_open_re.search(text)
275 + if m_open is not None:
276 + text = text[m_open.end() :]
277 +
278 + # Remove complete close/message markers
279 + text = self._response_close_re.sub("", text)
280 + text = self._message_close_re.sub("", text)
281 +
282 + # Hold back partial markers at the end
283 + overlap = 0
284 + for marker in (
285 + self._response_open,
286 + self._response_close,
287 + self._message_close,
288 + ):
289 + max_check = min(len(marker) - 1, len(text))
290 + for n in range(max_check, 0, -1):
291 + if text.endswith(marker[:n]):
292 + overlap = max(overlap, n)
293 + break
294 + return text[:-overlap] if overlap else text
295 +
296 + def strip_content_streaming(
297 + self,
298 + previous_text: str,
299 + current_text: str,
300 + ) -> DeltaMessage | None:
301 + """Strip XTML content wrappers from streaming deltas after reasoning.
302 +
303 + Called by KimiK3Parser when no tool parser is configured, so the
304 + reasoning parser handles ``<|open|>response<|sep|>`` /
305 + ``<|close|>response<|sep|>`` / ``<|close|>message<|sep|>`` stripping itself.
306 +
307 + Works from accumulated text (``previous_text`` / ``current_text``
308 + already contain only post-reasoning content).
309 + """
310 + current_safe = self._content_ready_to_emit(current_text)
311 + previous_safe = self._content_ready_to_emit(previous_text)
312 + if current_safe.startswith(previous_safe):
313 + delta = current_safe[len(previous_safe) :]
314 + else:
315 + delta = current_safe
316 + return DeltaMessage(content=delta) if delta else None
317 +
318 + def extract_reasoning_streaming(
319 + self,
320 + previous_text: str,
321 + current_text: str,
322 + delta_text: str,
323 + previous_token_ids: Sequence[int],
324 + current_token_ids: Sequence[int],
325 + delta_token_ids: Sequence[int],
326 + ) -> DeltaMessage | None:
327 + self._last_streaming_delta_token_ids = None
328 + self._last_streaming_content_token_ids = None
329 + if not self._thinking_enabled:
330 + return DeltaMessage(content=delta_text)
331 +
332 + # reasoning already ended -> downstream content
333 + if self._think_close_re.search(previous_text):
334 + return DeltaMessage(content=delta_text)
335 +
336 + # the close marker completes within this delta's accumulated text:
337 + # split the buffer at the close marker into reasoning vs trailing content.
338 + m_close = self._think_close_re.search(current_text)
339 + if m_close is not None:
340 + self._last_streaming_delta_token_ids = tuple(delta_token_ids)
341 + self._last_streaming_content_token_ids = self._extract_content_ids(
342 + list(current_token_ids)
343 + )
344 + m_open = self._think_open_re.search(current_text)
345 + r_start = m_open.end() if m_open is not None else 0
346 + reasoning = current_text[r_start : m_close.start()]
347 + already_sent = self._reasoning_text_ready_to_emit(previous_text)
348 + if reasoning.startswith(already_sent):
349 + reasoning_delta = reasoning[len(already_sent) :]
350 + else:
351 + reasoning_delta = reasoning
352 + content = current_text[m_close.end() :]
353 + return DeltaMessage(
354 + reasoning=reasoning_delta or None,
355 + content=content or None,
356 + )
357 +
358 + current_reasoning = self._reasoning_text_ready_to_emit(current_text)
359 + previous_reasoning = self._reasoning_text_ready_to_emit(previous_text)
360 + if current_reasoning.startswith(previous_reasoning):
361 + reasoning_delta = current_reasoning[len(previous_reasoning) :]
362 + else:
363 + reasoning_delta = current_reasoning
364 + if not reasoning_delta:
365 + return None
366 + return DeltaMessage(reasoning=reasoning_delta)
367 +
368 + # Backward-compatible aliases for existing unit tests and downstream users
369 + # that still call the pre-split method names.
370 + extract_reasoning_content = extract_reasoning
371 + extract_reasoning_content_streaming = extract_reasoning_streaming
vllm/renderers/kimi_k3.py new
+220
@@ -0,0 +1,220 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +from typing import Any, cast
4 +
5 +from vllm.config import VllmConfig
6 +from vllm.entrypoints.chat_utils import (
7 + ChatCompletionMessageParam,
8 + ConversationMessage,
9 + parse_chat_messages,
10 + parse_chat_messages_async,
11 +)
12 +from vllm.exceptions import VLLMValidationError
13 +from vllm.multimodal.media.connector import merge_media_io_kwargs
14 +from vllm.tokenizers.hf import HfTokenizer
15 +from vllm.utils.async_utils import make_async
16 +
17 +from .base import BaseRenderer
18 +from .inputs import DictPrompt
19 +from .inputs.preprocess import parse_dec_only_prompt
20 +from .params import ChatParams
21 +
22 +# Keep the original image mode (including the alpha channel) for K3 instead of
23 +# flattening images onto a background color. Server-level (--media-io-kwargs)
24 +# and request-level media_io_kwargs still take precedence over this default.
25 +_K3_MEDIA_IO_DEFAULTS: dict[str, dict[str, Any]] = {"image": {"image_mode": None}}
26 +_K3_THINKING_EFFORTS = ("low", "high", "max")
27 +
28 +
29 +def _merge_k3_media_io_kwargs(
30 + media_io_kwargs: dict[str, dict[str, Any]] | None,
31 +) -> dict[str, dict[str, Any]] | None:
32 + return merge_media_io_kwargs(_K3_MEDIA_IO_DEFAULTS, media_io_kwargs)
33 +
34 +
35 +def _dump_k3_template_value(value: Any) -> Any:
36 + model_dump = getattr(value, "model_dump", None)
37 + if callable(model_dump):
38 + return model_dump(mode="json", exclude_none=True)
39 +
40 + to_dict = getattr(value, "to_dict", None)
41 + if callable(to_dict):
42 + return to_dict()
43 +
44 + return value
45 +
46 +
47 +def _apply_k3_thinking_kwargs(kwargs: dict[str, Any]) -> None:
48 + if (enable_thinking := kwargs.pop("enable_thinking", None)) is not None:
49 + kwargs.setdefault("thinking", enable_thinking)
50 +
51 + reasoning_effort = kwargs.pop("reasoning_effort", None)
52 + if reasoning_effort == "none":
53 + kwargs.setdefault("thinking", False)
54 + elif reasoning_effort is not None:
55 + kwargs.setdefault("thinking_effort", reasoning_effort)
56 +
57 + thinking_effort = kwargs.get("thinking_effort")
58 + if thinking_effort is not None and thinking_effort not in _K3_THINKING_EFFORTS:
59 + supported = ", ".join(_K3_THINKING_EFFORTS)
60 + raise VLLMValidationError(
61 + f"Kimi K3 supports thinking_effort values: {supported}",
62 + parameter="thinking_effort",
63 + value=thinking_effort,
64 + )
65 +
66 +
67 +def _normalize_k3_tool_messages(
68 + conversation: list[ConversationMessage],
69 +) -> list[dict[str, Any]]:
70 + """Reorder tool-result messages to match assistant tool_call order.
71 +
72 + Supports matching by ``tool_call_id`` or by the synthetic
73 + ``"{tool}:{zero_based_index}"`` alias. When any tool message in a
74 + block cannot be resolved, the whole block is left in its original
75 + order (graceful fallback).
76 +
77 + Returns a new list; caller-owned message dicts are not mutated.
78 + """
79 + normalized: list[dict[str, Any]] = []
80 + i = 0
81 +
82 + while i < len(conversation):
83 + message = conversation[i]
84 + normalized.append(dict(message))
85 + i += 1
86 +
87 + if message.get("role") != "assistant":
88 + continue
89 +
90 + tool_calls = message.get("tool_calls")
91 + if not tool_calls:
92 + continue
93 +
94 + # Build the lookup table from the assistant's tool_calls.
95 + targets_by_id: dict[str, tuple[int, str]] = {}
96 + for position, tool_call in enumerate(tool_calls):
97 + function = tool_call.get("function")
98 + if not isinstance(function, dict):
99 + continue
100 + name = function.get("name")
101 + if not isinstance(name, str) or not name:
102 + continue
103 +
104 + call_target = (position, name)
105 + aliases = [f"{name}:{position}"]
106 + if tool_call_id := tool_call.get("id"):
107 + aliases.insert(0, str(tool_call_id))
108 + for alias in aliases:
109 + targets_by_id.setdefault(alias, call_target)
110 +
111 + # Collect consecutive tool messages.
112 + block_start = i
113 + while i < len(conversation) and conversation[i].get("role") == "tool":
114 + i += 1
115 + if i == block_start:
116 + continue
117 +
118 + # Try to resolve every tool message in the block.
119 + resolved: list[tuple[int, int, dict[str, Any]]] = []
120 + for order, tool_message in enumerate(conversation[block_start:i]):
121 + tool_call_id = tool_message.get("tool_call_id")
122 + resolved_target = (
123 + targets_by_id.get(str(tool_call_id))
124 + if tool_call_id is not None
125 + else None
126 + )
127 + if resolved_target is None:
128 + normalized.extend(dict(item) for item in conversation[block_start:i])
129 + break
130 +
131 + position, name = resolved_target
132 + enriched = dict(tool_message)
133 + enriched["tool"] = name
134 + enriched["index"] = position + 1
135 + resolved.append((position, order, enriched))
136 + else:
137 + resolved.sort(key=lambda item: (item[0], item[1]))
138 + normalized.extend(item for _, _, item in resolved)
139 +
140 + return normalized
141 +
142 +
143 +class KimiK3Renderer(BaseRenderer[HfTokenizer]):
144 + """Render chat prompts with Kimi K3's Python XTML encoding.
145 +
146 + K3 ships no Jinja chat template; its tokenizer renders messages through
147 + ``encoding_k3`` instead. We tokenize eagerly so the structural markers keep
148 + their special-token ids while user- and tool-supplied text stays ordinary.
149 + """
150 +
151 + def __init__(self, config: VllmConfig, tokenizer: HfTokenizer | None) -> None:
152 + super().__init__(config, tokenizer)
153 +
154 + self._apply_chat_template_async = make_async(
155 + self._apply_chat_template, executor=self._executor
156 + )
157 +
158 + def _apply_chat_template(
159 + self,
160 + conversation: list[dict[str, Any]],
161 + params: ChatParams,
162 + ) -> list[int]:
163 + # Tokenize eagerly: K3 encodes structural markers as special tokens and
164 + # user/tool text as ordinary tokens, so we cannot defer to a plain
165 + # re-tokenization of the rendered string downstream.
166 + kwargs = params.get_apply_chat_template_kwargs()
167 + _apply_k3_thinking_kwargs(kwargs)
168 + if params.tool_choice not in (None, "auto"):
169 + kwargs["tool_choice"] = _dump_k3_template_value(params.tool_choice)
170 + if params.response_format is not None:
171 + kwargs["response_format"] = _dump_k3_template_value(params.response_format)
172 + kwargs["tokenize"] = True
173 + return self.get_tokenizer().apply_chat_template(conversation, **kwargs)
174 +
175 + def render_messages(
176 + self,
177 + messages: list[ChatCompletionMessageParam],
178 + params: ChatParams,
179 + ) -> tuple[list[ConversationMessage], DictPrompt]:
180 + conversation, mm_data, mm_uuids = parse_chat_messages(
181 + messages,
182 + self.model_config,
183 + content_format="string",
184 + media_io_kwargs=_merge_k3_media_io_kwargs(params.media_io_kwargs),
185 + mm_processor_kwargs=params.mm_processor_kwargs,
186 + )
187 +
188 + rendered_conversation = _normalize_k3_tool_messages(conversation)
189 + prompt = parse_dec_only_prompt(
190 + self._apply_chat_template(rendered_conversation, params)
191 + )
192 + if mm_data is not None:
193 + prompt["multi_modal_data"] = mm_data
194 + if mm_uuids is not None:
195 + prompt["multi_modal_uuids"] = mm_uuids
196 +
197 + return cast(list[ConversationMessage], rendered_conversation), prompt
198 +
199 + async def render_messages_async(
200 + self,
201 + messages: list[ChatCompletionMessageParam],
202 + params: ChatParams,
203 + ) -> tuple[list[ConversationMessage], DictPrompt]:
204 + conversation, mm_data, mm_uuids = await parse_chat_messages_async(
205 + messages,
206 + self.model_config,
207 + content_format="string",
208 + media_io_kwargs=_merge_k3_media_io_kwargs(params.media_io_kwargs),
209 + mm_processor_kwargs=params.mm_processor_kwargs,
210 + )
211 +
212 + rendered_conversation = _normalize_k3_tool_messages(conversation)
213 + token_ids = await self._apply_chat_template_async(rendered_conversation, params)
214 + prompt = parse_dec_only_prompt(token_ids)
215 + if mm_data is not None:
216 + prompt["multi_modal_data"] = mm_data
217 + if mm_uuids is not None:
218 + prompt["multi_modal_uuids"] = mm_uuids
219 +
220 + return cast(list[ConversationMessage], rendered_conversation), prompt
vllm/renderers/params.py
+8
@@ -90,6 +90,12 @@ class ChatParams:
90 return_assistant_tokens_mask: bool = False
91 """Request a per-token assistant mask from apply_chat_template."""
92
93 + tool_choice: Any | None = None
94 + """Request-level tool choice for renderers that need API metadata."""
95 +
96 + response_format: Any | None = None
97 + """Request-level response format for renderers that need API metadata."""
98 +
99 def with_defaults(
100 self,
101 default_chat_template_kwargs: dict[str, Any] | None = None,
@@ -119,6 +125,8 @@ class ChatParams:
125 self.mm_processor_kwargs,
126 ),
127 return_assistant_tokens_mask=self.return_assistant_tokens_mask,
128 + tool_choice=self.tool_choice,
129 + response_format=self.response_format,
130 )
131
132 def get_apply_chat_template_kwargs(self) -> dict[str, Any]:
vllm/renderers/registry.py
+1
@@ -24,6 +24,7 @@ _VLLM_RENDERERS = {
24 "deepseek_v4": ("deepseek_v4", "DeepseekV4Renderer"),
25 "hf": ("hf", "HfRenderer"),
26 "kimi_audio": ("hf", "HfRenderer"),
27 + "kimi_k3": ("kimi_k3", "KimiK3Renderer"),
28 "mistral": ("mistral", "MistralRenderer"),
29 "terratorch": ("terratorch", "TerratorchRenderer"),
30 "inkling": ("inkling", "InklingRenderer"),
vllm/tokenizers/registry.py
+1
@@ -44,6 +44,7 @@ _VLLM_TOKENIZERS = {
44 "deepseek_v4": ("deepseek_v4", "DeepseekV4Tokenizer"),
45 "hf": ("hf", "CachedHfTokenizer"),
46 "kimi_audio": ("kimi_audio", "KimiAudioTokenizer"),
47 + "kimi_k3": ("hf", "CachedHfTokenizer"),
48 "mistral": ("mistral", "MistralTokenizer"),
49 # Inkling uses the plain HF tokenizer for token operations; the "inkling"
50 # mode exists to select the InklingRenderer, which renders chat to
vllm/tool_parsers/__init__.py
+4
@@ -102,6 +102,10 @@ _TOOL_PARSERS_TO_REGISTER = {
102 "kimi_k2_tool_parser",
103 "KimiK2ToolParser",
104 ),
105 + "kimi_k3": (
106 + "kimi_k3_tool_parser",
107 + "KimiK3ToolParser",
108 + ),
109 "llama3_json": (
110 "llama_tool_parser",
111 "Llama3JsonToolParser",
vllm/tool_parsers/kimi_k3_tool_parser.py new
+405
@@ -0,0 +1,405 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Tool-call parser for the Kimi K3 (XTML) chat format.
4 +
5 +This turns the generated XTML ``response`` and ``tools`` channels back into
6 +OpenAI-compatible ``content`` and ``tool_calls``.
7 +
8 +K3 assistant tool calls live in a nested ``tools`` channel::
9 +
10 + <|open|>tools<|sep|>
11 + <|open|>call tool="python" index="1"<|sep|>
12 + <|open|>argument key="code" type="string"<|sep|>print(1)<|close|>argument<|sep|>
13 + <|open|>argument key="opts" type="object"<|sep|>{"a":1}<|close|>argument<|sep|>
14 + <|close|>call<|sep|>
15 + <|close|>tools<|sep|>
16 +
17 +where ``<|open|>``, ``<|close|>``, ``<|sep|>`` are dedicated special tokens. The plain
18 +reply lives in a sibling ``response`` channel which is unwrapped into content.
19 +``response`` always precedes ``tools`` (see the protocol channel grammar).
20 +
21 +Argument decoding mirrors the template's type tagging (inverse encoding):
22 + * type="string" -> value is the RAW text, no unescaping (template emits it raw)
23 + * other types -> value is JSON-decoded (number/boolean/null/object/array)
24 +
25 +Attribute values (``tool=``, ``index=``, ``key=``, ``type=``) ARE escaped on the
26 +encode side (``&`` -> ``&amp;``, ``"`` -> ``&quot;``), so ``_attrs`` reverses
27 +them on decode (``&quot;`` before ``&amp;`` -- reverse of encode order).
28 +
29 +Known limitation: because string argument and response bodies are emitted raw,
30 +a value that literally contains ``<|close|>argument<|sep|>`` or
31 +``<|close|>response<|sep|>`` is indistinguishable from a real closing marker.
32 +"""
33 +
34 +import json
35 +from collections.abc import Sequence
36 +
37 +import regex as re
38 +from openai.types.responses import ToolChoiceFunction
39 +
40 +from vllm.entrypoints.openai.chat_completion.protocol import (
41 + ChatCompletionNamedToolChoiceParam,
42 + ChatCompletionRequest,
43 +)
44 +from vllm.entrypoints.openai.engine.protocol import (
45 + DeltaFunctionCall,
46 + DeltaMessage,
47 + DeltaToolCall,
48 + ExtractedToolCallInformation,
49 + FunctionCall,
50 + ToolCall,
51 +)
52 +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
53 +from vllm.exceptions import VLLMValidationError
54 +from vllm.logger import init_logger
55 +from vllm.tokenizers import TokenizerLike
56 +from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser
57 +
58 +logger = init_logger(__name__)
59 +
60 +_O, _C, _S = r"<\|open\|>", r"<\|close\|>", r"<\|sep\|>"
61 +_TEXT_UNTIL_SEP = r"(?:(?!" + _S + r").)*?"
62 +
63 +
64 +def _partial_tag_overlap(text: str, tag: str) -> int:
65 + max_len = min(len(text), len(tag) - 1)
66 + for n in range(max_len, 0, -1):
67 + if text.endswith(tag[:n]):
68 + return n
69 + return 0
70 +
71 +
72 +class KimiK3ToolParser(ToolParser):
73 + supports_required_and_named = False
74 + # Enables the vLLM-side XTML structural tag builder
75 + # (``get_kimi_k3_structural_tag`` in ``structural_tag_registry``). With
76 + # ``VLLM_ENFORCE_STRICT_TOOL_CALLING`` on (default), ``_apply_structural_tag``
77 + # constrains generation to K3's ``<|open|>tools<|sep|>`` channel for
78 + # ``required`` (and ``auto`` when a tool sets ``strict``) instead of the
79 + # generic JSON guided decoding, which conflicts with the XTML format.
80 + structural_tag_model = "kimi_k3"
81 +
82 + def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
83 + super().__init__(tokenizer, tools)
84 +
85 + self.tools_open = "<|open|>tools<|sep|>"
86 + self.tools_close = "<|close|>tools<|sep|>"
87 + self.response_open = "<|open|>response<|sep|>"
88 + self.response_close = "<|close|>response<|sep|>"
89 +
90 + # Regexes operate on detokenized text. The XTML markers reach us as the
91 + # literal strings <|open|>/<|close|>/<|sep|>. adjust_request keeps them
92 + # from being stripped and suppresses spacing between adjacent token
93 + # pieces. As DEFENSE-IN-DEPTH we still tolerate optional whitespace
94 + # WITHIN a marker (e.g. "<|open|> tools <|sep|>") in case some serving
95 + # path leaves vLLM's added-token spacing on -- this `\s*` is a no-op on
96 + # clean input, so the normal byte-exact path is unaffected. We do NOT
97 + # strip body whitespace, so clean content stays byte-exact.
98 + # All bodies use non-greedy (.*?) so each block stops at its own first
99 + # closing marker -- see the module-level KNOWN LIMITATION about
100 + # literal-marker values.
101 + self._tools_open_re = re.compile(_O + r"\s*tools\s*" + _S)
102 + self._tools_close_re = re.compile(_C + r"\s*tools\s*" + _S)
103 + self._response_open_re = re.compile(_O + r"\s*response\s*" + _S)
104 + self._response_close_re = re.compile(_C + r"\s*response\s*" + _S)
105 + self._message_close_re = re.compile(_C + r"\s*message\s*" + _S)
106 + self._call_re = re.compile(
107 + _O
108 + + r"\s*call\s+(?P<attrs>"
109 + + _TEXT_UNTIL_SEP
110 + + r")"
111 + + _S
112 + + r"(?P<body>.*?)"
113 + + _C
114 + + r"\s*call\s*"
115 + + _S,
116 + re.DOTALL,
117 + )
118 + self._arg_re = re.compile(
119 + _O
120 + + r"\s*argument\s+(?P<attrs>"
121 + + _TEXT_UNTIL_SEP
122 + + r")"
123 + + _S
124 + + r"(?P<val>.*?)"
125 + + _C
126 + + r"\s*argument\s*"
127 + + _S,
128 + re.DOTALL,
129 + )
130 + # attr segment: key="value" (value already escaped on the encode side)
131 + self._attr_re = re.compile(r'(?P<k>\w+)="(?P<v>[^"]*)"')
132 + self._response_re = re.compile(
133 + _O + r"\s*response\s*" + _S + r"(?P<c>.*?)" + _C + r"\s*response\s*" + _S,
134 + re.DOTALL,
135 + )
136 +
137 + # streaming state
138 + self._sent_content_idx = 0
139 + self._sent_tool_call_count = 0
140 +
141 + if not self.model_tokenizer:
142 + raise ValueError(
143 + "The model tokenizer must be passed to the ToolParser "
144 + "constructor during construction."
145 + )
146 +
147 + def adjust_request(
148 + self, request: ChatCompletionRequest | ResponsesRequest
149 + ) -> ChatCompletionRequest | ResponsesRequest:
150 + named = isinstance(
151 + request.tool_choice,
152 + (ChatCompletionNamedToolChoiceParam, ToolChoiceFunction),
153 + )
154 + structured_outputs = getattr(request, "structured_outputs", None)
155 + has_structural_tag = (
156 + structured_outputs is not None
157 + and structured_outputs.structural_tag is not None
158 + )
159 + if named and not has_structural_tag:
160 + # Without the XTML structural tag there is no way to force the
161 + # named call (the generic JSON guided-decoding path conflicts
162 + # with the XTML channel format).
163 + raise VLLMValidationError(
164 + "Named tool choice for Kimi K3 requires strict tool calling "
165 + "(VLLM_ENFORCE_STRICT_TOOL_CALLING) so the XTML structural "
166 + "tag can force the call. Otherwise use `tool_choice` set to "
167 + '"auto", "required", or "none".',
168 + parameter="tool_choice",
169 + value=request.tool_choice,
170 + )
171 +
172 + if request.tools and (request.tool_choice == "required" or named):
173 + # K3 emits tool calls in XTML. When strict tool calling is enabled,
174 + # DelegatingParser._apply_structural_tag has already attached the K3
175 + # XTML structural tag (structural_tag_model = "kimi_k3"). We return
176 + # early to skip the generic parent path, which would otherwise attach
177 + # JSON guided decoding when strict calling is off -- that JSON
178 + # constraint conflicts with the <|open|>tools<|sep|> channel.
179 + request.skip_special_tokens = False
180 + if hasattr(request, "spaces_between_special_tokens"):
181 + request.spaces_between_special_tokens = False
182 + return request
183 +
184 + request = super().adjust_request(request)
185 + # The XTML markers (<|open|>/<|close|>/<|sep|>,
186 + # <|open|>response<|sep|> ...) must
187 + # reach this parser as CONTIGUOUS literal text. Two request flags govern
188 + # that, and vLLM's detokenizer couples them:
189 + # effective_spaces_between =
190 + # skip_special_tokens OR spaces_between_special_tokens
191 + # (vllm/v1/engine/detokenizer.py).
192 + # The detokenizer treats these control tokens as separate sub-texts and,
193 + # when the effective flag is True, joins them with spaces ->
194 + # "<|open|> response <|sep|>", which the regexes below would NOT match.
195 + # Forcing BOTH flags off is the only way to suppress that spacing. We
196 + # set both unconditionally: the response channel is unwrapped from these
197 + # markers even with no tools.
198 + request.skip_special_tokens = False
199 + if hasattr(request, "spaces_between_special_tokens"):
200 + request.spaces_between_special_tokens = False
201 + return request
202 +
203 + def _attrs(self, s: str) -> dict[str, str]:
204 + return {
205 + m["k"]: m["v"].replace("&quot;", '"').replace("&amp;", "&")
206 + for m in self._attr_re.finditer(s)
207 + }
208 +
209 + def _decode_call(self, attrs: str, body: str) -> ToolCall | None:
210 + """Decode one ``call`` block into a :class:`ToolCall`.
211 +
212 + ``attrs`` is the text between ``<|open|>call `` and ``<|sep|>`` (carries
213 + ``tool=`` / ``index=``); ``body`` is the sequence of ``argument`` blocks.
214 + Each argument is re-typed per its ``type=`` tag: strings pass through
215 + raw, everything else is JSON-decoded (falling back to raw text if the
216 + JSON is malformed, so a partial stream never raises). Returns ``None``
217 + when no tool name is present (an empty/garbage block is dropped).
218 + """
219 + call_attrs = self._attrs(attrs)
220 + tool_name = call_attrs.get("tool", "")
221 + tool_index = call_attrs.get("index", "")
222 + arguments: dict = {}
223 + for arg_match in self._arg_re.finditer(body):
224 + arg_attrs = self._attrs(arg_match["attrs"])
225 + key = arg_attrs.get("key", "")
226 + arg_type = arg_attrs.get("type", "string")
227 + raw_value = arg_match["val"]
228 + if arg_type == "string":
229 + arguments[key] = raw_value
230 + else:
231 + try:
232 + arguments[key] = json.loads(raw_value)
233 + except json.JSONDecodeError:
234 + arguments[key] = raw_value
235 + if not tool_name:
236 + return None
237 + tool_call_id = tool_name
238 + if tool_index:
239 + try:
240 + tool_call_id = f"{tool_name}:{int(tool_index) - 1}"
241 + except ValueError:
242 + tool_call_id = f"{tool_name}:{tool_index}"
243 + # id uses the API-side zero-based call ordinal; XTML's message index
244 + # stays one-based when rendering tool result messages.
245 + return ToolCall(
246 + id=tool_call_id,
247 + type="function",
248 + function=FunctionCall(
249 + name=tool_name,
250 + arguments=json.dumps(arguments, ensure_ascii=False),
251 + ),
252 + )
253 +
254 + def _strip_response_content(self, text: str) -> str | None:
255 + """Strip XTML response/message markers from generated response text.
256 +
257 + In chat serving, ``<|open|>response<|sep|>`` is often part of the prompt
258 + generation prefix, so the model output may only contain the body plus
259 + ``<|close|>response<|sep|>``. Handle both that consumed-prefix shape and a
260 + complete ``<|open|>response<|sep|>... <|close|>response<|sep|>`` wrapper.
261 + """
262 + m_open = self._response_open_re.search(text)
263 + if m_open is not None:
264 + m_close = self._response_close_re.search(text, m_open.end())
265 + if m_close is not None:
266 + text = text[m_open.end() : m_close.start()]
267 + else:
268 + text = text[m_open.end() :]
269 + else:
270 + text = self._response_close_re.sub("", text)
271 + text = self._message_close_re.sub("", text)
272 + return text or None
273 +
274 + def _content(self, model_output: str, before: str) -> str | None:
275 + # prefer the unwrapped response channel; else the text before the tools
276 + m = self._response_re.search(model_output)
277 + if m is not None:
278 + return m["c"] or None
279 + return self._strip_response_content(before)
280 +
281 + def _extract_response_content(self, current_text: str) -> str | None:
282 + # Streaming response text is computed from the accumulated text. This is
283 + # what keeps split markers from leaking:
284 + # <|open|> / response / <|sep|>Hi -> emit only "Hi" after open closes
285 + # Hi<|open|> / tools / <|sep|>... -> emit "Hi", hold the tools marker
286 + # Hi<|close|> / response / <|sep|>... -> emit "Hi", hold the close marker
287 + # Tool calls are even simpler: they are not emitted until a full
288 + # <|close|>call<|sep|> is present in current_text.
289 + m_open = self._response_open_re.search(current_text)
290 + # In the normal chat path, the response-open marker may be consumed as
291 + # the generation prefix. Then generated text starts directly with the
292 + # response body or with <|close|>response<|sep|> before a tools channel.
293 + body_start = m_open.end() if m_open is not None else 0
294 + m_tools = self._tools_open_re.search(current_text, body_start)
295 + m_rclose = self._response_close_re.search(current_text, body_start)
296 + tools_start = m_tools.start() if m_tools else -1
297 + response_end = m_rclose.start() if m_rclose else -1
298 +
299 + candidates = [i for i in (tools_start, response_end) if i != -1]
300 + if candidates:
301 + sendable_idx = min(candidates)
302 + else:
303 + overlap = max(
304 + _partial_tag_overlap(current_text, self.response_open),
305 + _partial_tag_overlap(current_text, self.response_close),
306 + _partial_tag_overlap(current_text, self.tools_open),
307 + )
308 + sendable_idx = len(current_text) - overlap
309 +
310 + if sendable_idx <= body_start:
311 + return None
312 + if self._sent_content_idx < body_start:
313 + self._sent_content_idx = body_start
314 + if sendable_idx <= self._sent_content_idx:
315 + return None
316 +
317 + content = current_text[self._sent_content_idx : sendable_idx]
318 + self._sent_content_idx = sendable_idx
319 + return content or None
320 +
321 + def extract_tool_calls(
322 + self, model_output: str, request: ChatCompletionRequest
323 + ) -> ExtractedToolCallInformation:
324 + m_open = self._tools_open_re.search(model_output)
325 + if m_open is None:
326 + # no tools channel -> content is the response channel (unwrapped)
327 + return ExtractedToolCallInformation(
328 + tools_called=False,
329 + tool_calls=[],
330 + content=self._content(model_output, model_output),
331 + )
332 + try:
333 + before = model_output[: m_open.start()]
334 + start = m_open.end()
335 + m_close = self._tools_close_re.search(model_output, start)
336 + section = (
337 + model_output[start:]
338 + if m_close is None
339 + else model_output[start : m_close.start()]
340 + )
341 +
342 + tool_calls = [
343 + tc
344 + for m in self._call_re.finditer(section)
345 + if (tc := self._decode_call(m["attrs"], m["body"])) is not None
346 + ]
347 + if not tool_calls:
348 + return ExtractedToolCallInformation(
349 + tools_called=False,
350 + tool_calls=[],
351 + content=self._content(model_output, before),
352 + )
353 + return ExtractedToolCallInformation(
354 + tools_called=True,
355 + tool_calls=tool_calls,
356 + content=self._content(model_output, before),
357 + )
358 + except Exception:
359 + logger.exception("Error extracting K3 tool calls.")
360 + return ExtractedToolCallInformation(
361 + tools_called=False, tool_calls=[], content=model_output
362 + )
363 +
364 + def extract_tool_calls_streaming(
365 + self,
366 + previous_text: str,
367 + current_text: str,
368 + delta_text: str,
369 + previous_token_ids: Sequence[int],
370 + current_token_ids: Sequence[int],
371 + delta_token_ids: Sequence[int],
372 + request: ChatCompletionRequest,
373 + ) -> DeltaMessage | None:
374 + # Conservative streaming: stream unwrapped response-channel text, then
375 + # buffer tool calls and emit each call once its block closes.
376 + content = self._extract_response_content(current_text)
377 +
378 + # tools channel is open: parse fully-closed calls we have not emitted yet
379 + m_tools = self._tools_open_re.search(current_text)
380 + if m_tools is None:
381 + return DeltaMessage(content=content) if content else None
382 +
383 + section = current_text[m_tools.end() :]
384 + calls = [
385 + tc
386 + for m in self._call_re.finditer(section)
387 + if (tc := self._decode_call(m["attrs"], m["body"])) is not None
388 + ]
389 + if len(calls) <= self._sent_tool_call_count:
390 + return DeltaMessage(content=content) if content else None
391 + new = calls[self._sent_tool_call_count :]
392 +
393 + deltas = [
394 + DeltaToolCall(
395 + index=self._sent_tool_call_count + i,
396 + id=tc.id,
397 + type="function",
398 + function=DeltaFunctionCall(
399 + name=tc.function.name, arguments=tc.function.arguments
400 + ).model_dump(exclude_none=True),
401 + )
402 + for i, tc in enumerate(new)
403 + ]
404 + self._sent_tool_call_count = len(calls)
405 + return DeltaMessage(content=content, tool_calls=deltas)
vllm/tool_parsers/structural_tag_registry.py
+262 -1
@@ -19,7 +19,12 @@ from xgrammar.structural_tag import (
19 AnyTextFormat,
20 ConstStringFormat,
21 JSONSchemaFormat,
22 + OptionalFormat,
23 + OrFormat,
24 + PlusFormat,
25 + RegexFormat,
26 SequenceFormat,
27 + StarFormat,
28 TagFormat,
29 TagsWithSeparatorFormat,
30 TriggeredTagsFormat,
@@ -66,7 +71,7 @@ XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset(
71 "deepseek_v4",
72 }
73 )
69 -VLLM_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset({"hermes"})
74 +VLLM_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset({"hermes", "kimi_k3"})
75 SUPPORTED_STRUCTURAL_TAG_MODELS = (
76 XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS | VLLM_BUILTIN_STRUCTURAL_TAG_MODELS
77 )
@@ -345,3 +350,259 @@ def get_minimax_structural_tag(
350 )
351
352 return StructuralTag(format=suffix_tag)
353 +
354 +
355 +# ---------------------------------------------------------------------------
356 +# Kimi K3 (XTML channel format)
357 +# ---------------------------------------------------------------------------
358 +# K3 assistant output after the reasoning gate (``<|close|>think<|sep|>``):
359 +# <|open|>response<|sep|> <content> <|close|>response<|sep|>
360 +# [ <|open|>tools<|sep|>
361 +# <|open|>call tool="NAME" index="1"<|sep|>
362 +# <|open|>argument key="K" type="TYPE"<|sep|>VALUE<|close|>argument<|sep|>
363 +# <|close|>call<|sep|> ...
364 +# <|close|>tools<|sep|> ]
365 +# See ``encoding_k3.py`` (_render_assistant_segments / _open_tag / _attr) and
366 +# ``kimi_k3_tool_parser.py`` for the exact byte-level encoding this mirrors.
367 +_K3_OPEN = "<|open|>"
368 +_K3_CLOSE = "<|close|>"
369 +_K3_SEP = "<|sep|>"
370 +_K3_RESPONSE_OPEN = f"{_K3_OPEN}response{_K3_SEP}"
371 +_K3_RESPONSE_CLOSE = f"{_K3_CLOSE}response{_K3_SEP}"
372 +_K3_TOOLS_OPEN = f"{_K3_OPEN}tools{_K3_SEP}"
373 +_K3_TOOLS_CLOSE = f"{_K3_CLOSE}tools{_K3_SEP}"
374 +_K3_CALL_CLOSE = f"{_K3_CLOSE}call{_K3_SEP}"
375 +_K3_ARG_CLOSE = f"{_K3_CLOSE}argument{_K3_SEP}"
376 +# The model closes the assistant turn with <|close|>message<|sep|> right before
377 +# the end-of-message token. It is generated (not part of the prompt prefix), so
378 +# the tag must permit it or the FSM would mask the model's natural terminator.
379 +_K3_MESSAGE_CLOSE = f"{_K3_CLOSE}message{_K3_SEP}"
380 +
381 +# JSON-schema type -> K3 XTML ``type=`` attribute value. Mirrors
382 +# ``encoding_k3._xtml_type`` (integer collapses onto number).
383 +_K3_JSON_TO_XTML_TYPE = {
384 + "string": "string",
385 + "integer": "number",
386 + "number": "number",
387 + "boolean": "boolean",
388 + "null": "null",
389 + "object": "object",
390 + "array": "array",
391 +}
392 +
393 +
394 +def _k3_escape_attr(value: str) -> str:
395 + """Mirror ``encoding_k3._escape_attr_value`` (``&`` then ``"``)."""
396 + return str(value).replace("&", "&amp;").replace('"', "&quot;")
397 +
398 +
399 +_K3_STRING_ATOM = r"(?:[^<]|<[^|])"
400 +"""One raw-string character: anything but the ambiguous "<|" marker prefix.
401 +Allows '<' inside values (e.g. HTML snippets); a value *ending* in '<' or
402 +containing a literal "<|" is not expressible and falls back to AnyText via
403 +the pattern checks below never matching those cases at build time (schemas
404 +cannot know values, so the only build-time effect is the length bound)."""
405 +
406 +
407 +def _k3_bounded_string_regex(prop: dict[str, Any]) -> str | None:
408 + """Length/pattern constraint for the raw string channel, if expressible.
409 +
410 + The XTML string channel emits values raw (not JSON-quoted), so
411 + JSONSchemaFormat cannot enforce string constraints there; unconstrained
412 + AnyText lets maxLength/pattern violations through (observed on the walle
413 + verifier: over-long junk strings pass the grammar and fail validation).
414 + xgrammar's regex engine has no lookahead, so the close marker is kept
415 + unambiguous by excluding the "<|" prefix from value characters.
416 +
417 + Returns a regex for the value, or None to keep permissive AnyText.
418 + """
419 + max_len = prop.get("maxLength")
420 + min_len = prop.get("minLength", 0)
421 + if not isinstance(max_len, int) or max_len < 0 or max_len > 4096:
422 + return None
423 + if not isinstance(min_len, int) or min_len < 0 or min_len > max_len:
424 + min_len = 0
425 + return _K3_STRING_ATOM + f"{{{min_len},{max_len}}}"
426 +
427 +
428 +def _k3_argument_tag(
429 + key: str,
430 + schema: dict[str, Any],
431 + root_defs: dict[str, Any] | None = None,
432 +) -> TagFormat | None:
433 + """Build one ``argument`` XTML tag for property ``key``.
434 +
435 + ``string`` values are emitted raw (bounded by the close marker); every other
436 + JSON type is emitted as JSON and validated against the property schema. A
437 + property whose type is a union / missing is left permissive (any XTML type,
438 + raw value) so a valid call is never rejected.
439 +
440 + ``root_defs`` carries the tool parameters' root-level ``$defs`` /
441 + ``definitions``: slicing a property out of the parameters document orphans
442 + its ``#/$defs/...`` references, so those tables must be re-attached to keep
443 + the embedded schema self-contained.
444 + """
445 + prop = schema if isinstance(schema, dict) else {}
446 + json_type = prop.get("type")
447 + xtml_type = (
448 + _K3_JSON_TO_XTML_TYPE.get(json_type) if isinstance(json_type, str) else None
449 + )
450 + if xtml_type is None:
451 + # Unknown / union type: constrain the key but keep the value permissive.
452 + return None
453 + begin = (
454 + f'{_K3_OPEN}argument key="{_k3_escape_attr(key)}" type="{xtml_type}"{_K3_SEP}'
455 + )
456 + if xtml_type == "string":
457 + # Raw string channel: JSONSchemaFormat can't apply (values are not
458 + # JSON-quoted), but an enum/const of strings is a finite set that can
459 + # be enforced exactly with const-string alternation. Enum semantics
460 + # are exclusive, so this never over-rejects. Fall back to permissive
461 + # AnyText for open-ended strings or non-representable enums.
462 + enum_values = prop.get("enum")
463 + if enum_values is None and isinstance(prop.get("const"), str):
464 + enum_values = [prop["const"]]
465 + if (
466 + isinstance(enum_values, list)
467 + and enum_values
468 + and len(enum_values) <= 256
469 + and all(isinstance(v, str) for v in enum_values)
470 + and not any("<|" in v for v in enum_values)
471 + ):
472 + branches = [ConstStringFormat(value=v) for v in enum_values]
473 + content: Any = (
474 + branches[0] if len(branches) == 1 else OrFormat(elements=branches)
475 + )
476 + elif (bounded := _k3_bounded_string_regex(prop)) is not None:
477 + content = RegexFormat(pattern=bounded)
478 + else:
479 + content = AnyTextFormat(excludes=[_K3_CLOSE])
480 + else:
481 + embedded = prop
482 + if root_defs:
483 + embedded = dict(prop)
484 + for defs_key, defs_value in root_defs.items():
485 + embedded.setdefault(defs_key, defs_value)
486 + content = JSONSchemaFormat(json_schema=embedded)
487 + return TagFormat(begin=begin, content=content, end=_K3_ARG_CLOSE)
488 +
489 +
490 +def _k3_permissive_argument_tag() -> TagFormat:
491 + """A key/type-agnostic ``argument`` tag: any attributes, raw value.
492 +
493 + Used as a fallback so tools with union/loose schemas still get the XTML
494 + skeleton constrained without over-rejecting the value.
495 + """
496 + return TagFormat(
497 + begin=_K3_OPEN + "argument ",
498 + content=SequenceFormat(
499 + elements=[
500 + RegexFormat(pattern=r"[^<]*" + _K3_SEP.replace("|", r"\|")),
501 + AnyTextFormat(excludes=[_K3_CLOSE]),
502 + ]
503 + ),
504 + end=_K3_ARG_CLOSE,
505 + )
506 +
507 +
508 +def _k3_arguments_block(parameters: dict[str, Any] | bool) -> Any:
509 + """Build ``argument`` tags for a tool's parameter schema.
510 +
511 + Require at least one tag when the root schema declares required properties.
512 + Otherwise, keep accepting zero-or-more tags. Arguments remain order-agnostic
513 + and non-unique.
514 + """
515 + if not isinstance(parameters, dict):
516 + return StarFormat(content=_k3_permissive_argument_tag())
517 + props = parameters.get("properties")
518 + if not isinstance(props, dict) or not props:
519 + # No declared properties: allow any argument blocks (or none).
520 + return StarFormat(content=_k3_permissive_argument_tag())
521 + root_defs = {
522 + defs_key: parameters[defs_key]
523 + for defs_key in ("$defs", "definitions")
524 + if isinstance(parameters.get(defs_key), dict)
525 + }
526 + tags: list[TagFormat] = []
527 + for key, prop in props.items():
528 + tag = _k3_argument_tag(key, prop, root_defs)
529 + tags.append(tag if tag is not None else _k3_permissive_argument_tag())
530 + inner = tags[0] if len(tags) == 1 else OrFormat(elements=list(tags))
531 + required = parameters.get("required")
532 + if isinstance(required, list) and required:
533 + return PlusFormat(content=inner)
534 + return StarFormat(content=inner)
535 +
536 +
537 +def _k3_call_tag(tool: FunctionToolParam) -> TagFormat:
538 + """One ``call`` tag: ``<|open|>call tool="N" index="<digits>"<|sep|> args``."""
539 + function = tool.function
540 + parameters = _get_function_parameters(function)
541 + begin = f'{_K3_OPEN}call tool="{_k3_escape_attr(function.name)}" index="'
542 + return TagFormat(
543 + begin=begin,
544 + content=SequenceFormat(
545 + elements=[
546 + RegexFormat(pattern=r"[0-9]+"),
547 + ConstStringFormat(value=f'"{_K3_SEP}'),
548 + _k3_arguments_block(parameters),
549 + ]
550 + ),
551 + end=_K3_CALL_CLOSE,
552 + )
553 +
554 +
555 +def _k3_response_prefix() -> list[Any]:
556 + """The response channel that always precedes the tools channel.
557 +
558 + ``response`` is generated in thinking mode (prefix ends at
559 + ``<|open|>think<|sep|>``) but is part of the generation prefix in
560 + non-thinking mode, so its open marker is optional. The body is bounded by
561 + the response close marker.
562 + """
563 + return [
564 + OptionalFormat(content=ConstStringFormat(value=_K3_RESPONSE_OPEN)),
565 + TagFormat(begin="", content=AnyTextFormat(), end=_K3_RESPONSE_CLOSE),
566 + ]
567 +
568 +
569 +def _k3_tools_channel(tools: list[FunctionToolParam]) -> TagFormat:
570 + return TagFormat(
571 + begin=_K3_TOOLS_OPEN,
572 + content=TagsWithSeparatorFormat(
573 + tags=[_k3_call_tag(tool) for tool in tools],
574 + separator="",
575 + at_least_one=True,
576 + ),
577 + end=_K3_TOOLS_CLOSE,
578 + )
579 +
580 +
581 +@register_vllm_structural_tag("kimi_k3")
582 +def get_kimi_k3_structural_tag(
583 + tools: list[FunctionToolParam],
584 + builtin_tools: list[BuiltinToolParam],
585 + tool_choice: SimplifiedToolChoice,
586 + reasoning: bool,
587 +) -> StructuralTag:
588 + del builtin_tools, reasoning
589 +
590 + trailer = OptionalFormat(content=ConstStringFormat(value=_K3_MESSAGE_CLOSE))
591 +
592 + if not tools:
593 + return StructuralTag(
594 + format=SequenceFormat(elements=[*_k3_response_prefix(), trailer])
595 + )
596 +
597 + if tool_choice == "auto":
598 + tools_part: Any = OptionalFormat(content=_k3_tools_channel(tools))
599 + elif tool_choice == "forced":
600 + # K3 rejects named tool choice upstream; treat defensively as a single
601 + # mandatory call of the first tool.
602 + tools_part = _k3_tools_channel(tools[:1])
603 + else: # required
604 + tools_part = _k3_tools_channel(tools)
605 +
606 + return StructuralTag(
607 + format=SequenceFormat(elements=[*_k3_response_prefix(), tools_part, trailer])
608 + )