@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 9bd993564f

[Frontend] Parse tool calls and reasoning from checkpoint response templates (#58604)

Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Yoni Gozlan <yonigozlan@users.noreply.github.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>

Yoni Gozlan committed Oct 1, 2026 at 22:33 UTC 9bd993564f5f3d172f9da398fd56b64db1074d9e

18 files changed +2298 -21

docs/features/tool_calling.md

+23

@@ -171,6 +171,29 @@ from HuggingFace; and you can find an example of this in a `tokenizer_config.jso

171

172 If your favorite tool-calling model is not supported, please feel free to contribute a parser & tool use chat template!

173

174 +### Checkpoint Response Templates (`hf`)

175 +

176 +Checkpoints can define a `response_template` in `tokenizer_config.json` to describe their reasoning, content, and tool-call wire format. Select the

177 +`hf` parser to parse output from this metadata instead of a model-specific parser. Like `--tokenizer-mode hf`, it relies on the checkpoint's

178 +Hugging Face metadata rather than model-specific code:

179 +

180 +```bash

181 +vllm serve <model> \

182 + --enable-auto-tool-choice \

183 + --tool-call-parser hf \

184 + --reasoning-parser hf

185 +```

186 +

187 +Either parser can be selected on its own. Startup fails if the tokenizer has no `response_template`, or if it lacks the `thinking` or `tool_calls`

188 +field the selected parser needs. The `hf` parser cannot be combined with other reasoning or tool call parsers.

189 +

190 +When streaming, each tool call is emitted whole once its region parses. A call cut off before its closer, for example by a closer that is also a

191 +stop token, is parsed from the text generated so far; a call that does not parse is dropped.

192 +

193 +Response templates currently provide parsing, not format-specific constrained decoding. The parser therefore rejects requests for strict tools,

194 +required or named tool choice, or `parallel_tool_calls=false`. A custom server or request chat template is allowed, but the parser still expects the

195 +checkpoint's output format.

196 +

197 !!! note

198 With `tool_choice="auto"`, structural-tag constraints require both `VLLM_ENFORCE_STRICT_TOOL_CALLING=true` (the default) and at least one tool with `strict: true`, or a server-side floor set via `--tool-strict-level`. When these conditions are met and the selected parser supports structural tags, vLLM constrains the tool-call envelope and pins the argument schema of each tool that sets `strict: true` (or of every tool under `--tool-strict-level parameter`). Otherwise, vLLM extracts tool calls from raw text, so arguments may occasionally be malformed or violate the function's parameter schema.

199

tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py

+56 -4

@@ -246,11 +246,15 @@ async def test_batched_chat_completions_logprob_token_ids(

246 }

247

248

249 -def _make_request_output(prompt_idx: int, text: str) -> RequestOutput:

249 +def _make_request_output(

250 + prompt_idx: int,

251 + text: str,

252 + prompt_token_ids: list[int] | None = None,

253 +) -> RequestOutput:

254 return RequestOutput(

255 request_id=f"req-{prompt_idx}",

256 prompt=None,

253 - prompt_token_ids=[1, 2, 3],

257 + prompt_token_ids=prompt_token_ids or [1, 2, 3],

258 prompt_logprobs=None,

259 outputs=[

260 CompletionOutput(

@@ -266,8 +270,12 @@ def _make_request_output(prompt_idx: int, text: str) -> RequestOutput:

270 )

271

272

269 -async def _generator(prompt_idx: int, text: str) -> AsyncGenerator[RequestOutput, None]:

270 - yield _make_request_output(prompt_idx, text)

273 +async def _generator(

274 + prompt_idx: int,

275 + text: str,

276 + prompt_token_ids: list[int] | None = None,

277 +) -> AsyncGenerator[RequestOutput, None]:

278 + yield _make_request_output(prompt_idx, text, prompt_token_ids)

279

280

281 @pytest.mark.asyncio

@@ -402,3 +410,47 @@ def test_batch_rejects_kv_transfer_prompt_token_ids():

410 ],

411 kv_transfer_params={"prompt_token_ids": [10, 20, 30]},

412 )

413 +

414 +

415 +@pytest.mark.asyncio

416 +@pytest.mark.skip_global_cleanup

417 +async def test_batched_parser_receives_each_prompt_prefix() -> None:

418 + prefixes = []

419 +

420 + class RecordingParser:

421 + def __init__(self, *args, **kwargs):

422 + self.prompt_token_ids = None

423 +

424 + def set_prompt_token_ids(self, prompt_token_ids):

425 + self.prompt_token_ids = prompt_token_ids

426 +

427 + def parse(self, model_output, **kwargs):

428 + prefixes.append(self.prompt_token_ids)

429 + return None, model_output, None

430 +

431 + serving = OpenAIServingChatBatch.__new__(OpenAIServingChatBatch)

432 + serving.response_role = "assistant"

433 + serving.system_fingerprint = None

434 + serving.chat_template = None

435 + serving.chat_template_content_format = "auto"

436 + serving.default_chat_template_kwargs = {}

437 +

438 + request = BatchChatCompletionRequest(

439 + model="test-model",

440 + messages=[

441 + [{"role": "user", "content": "first"}],

442 + [{"role": "user", "content": "second"}],

443 + ],

444 + )

445 + await serving.chat_completion_full_generator_batch(

446 + request=request,

447 + generators=[_generator(0, "one", [1]), _generator(1, "two", [2])],

448 + request_id="req-prefix",

449 + model_name="test-model",

450 + all_conversations=request.messages,

451 + tokenizer=object(),

452 + request_metadata=RequestResponseMetadata(request_id="req-prefix"),

453 + parser_cls=RecordingParser,

454 + )

455 +

456 + assert prefixes == [[1], [2]]

tests/entrypoints/openai/chat_completion/test_serving_chat.py

+73

@@ -790,6 +790,64 @@ async def test_chat_per_request_metrics_suppressed_for_n_greater_than_one():

790 assert response.metrics is None

791

792

793 +@pytest.mark.asyncio

794 +async def test_chat_full_generator_gives_each_choice_parser_the_prompt():

795 + prompts: list[list[int] | None] = []

796 +

797 + class RecordingParser:

798 + tool_parser_cls = None

799 +

800 + def __init__(self, *args, **kwargs):

801 + self.prompt_token_ids = None

802 +

803 + def set_prompt_token_ids(self, prompt_token_ids):

804 + self.prompt_token_ids = list(prompt_token_ids)

805 +

806 + def parse(self, model_output, request, **kwargs):

807 + prompts.append(self.prompt_token_ids)

808 + return None, model_output, None

809 +

810 + def count_reasoning_tokens(self, token_ids):

811 + return 0

812 +

813 + serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=False)

814 + serving.parser_cls = RecordingParser

815 + serving.model_config = None

816 + serving.chat_template = None

817 + serving.chat_template_content_format = "auto"

818 + serving.default_chat_template_kwargs = {}

819 + request_output = _make_metrics_request_output()

820 + request_output.prompt_token_ids = [7, 8]

821 + request_output.outputs.append(

822 + CompletionOutput(

823 + index=1,

824 + text="Hi",

825 + token_ids=[102],

826 + cumulative_logprob=None,

827 + logprobs=None,

828 + finish_reason="stop",

829 + )

830 + )

831 +

832 + await serving.chat_completion_full_generator(

833 + ChatCompletionRequest(

834 + model="test-model",

835 + messages=[{"role": "user", "content": "Test prompt"}],

836 + max_tokens=10,

837 + n=2,

838 + ),

839 + _single_request_output(request_output),

840 + "chatcmpl-test-id",

841 + "test-model",

842 + conversation=[{"role": "user", "content": "Test"}],

843 + tokenizer=MagicMock(),

844 + request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"),

845 + parser=RecordingParser(),

846 + )

847 +

848 + assert prompts == [[7, 8], [7, 8]]

849 +

850 +

851 @pytest.mark.asyncio

852 async def test_chat_streaming_metrics_ride_on_usage_chunk():

853 serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=True)

@@ -1608,6 +1666,21 @@ async def test_serving_chat_truncation_side_controls_prompt_truncation():

1666 assert left_token_ids == full_token_ids[-4:]

1667

1668

1669 +def test_serving_chat_starts_without_tokenizer():

1670 + """Regression test: building the serving objects must not require a

1671 + tokenizer, which is absent under `skip_tokenizer_init`."""

1672 + mock_engine = _build_mock_engine()

1673 + mock_engine.model_config = MockModelConfig(skip_tokenizer_init=True)

1674 + mock_engine.renderer = HfRenderer(

1675 + MockVllmConfig(mock_engine.model_config, parallel_config=MockParallelConfig()),

1676 + None,

1677 + )

1678 +

1679 + serving_chat = _build_serving_chat(mock_engine)

1680 +

1681 + assert serving_chat.parser_cls is None

1682 +

1683 +

1684 @pytest.mark.asyncio

1685 async def test_serving_chat_mistral_token_ids_prompt_is_validated():

1686 """Regression test: when the Mistral tokenizer path returns token IDs

tests/entrypoints/scale_out/derender/test_derender_stream.py

+39

@@ -296,6 +296,45 @@ def parsed_derenderer(tokenizer):

296 return dr

297

298

299 +def test_non_streaming_derender_initializes_parser_prefix(

300 + derenderer, tokenizer, monkeypatch

301 +):

302 + from unittest.mock import MagicMock

303 +

304 + from vllm.entrypoints.generate.base.protocol import FunctionCall

305 + from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest

306 +

307 + parser = MagicMock()

308 + parser.parse.return_value = (

309 + None,

310 + None,

311 + [FunctionCall(name="tool", arguments="{}")],

312 + )

313 + monkeypatch.setattr(derenderer, "parser", MagicMock(return_value=parser))

314 + generated_ids = tokenizer.encode("answer", add_special_tokens=False)

315 + response = GenerateResponse(

316 + request_id="test",

317 + prompt_token_ids=[11, 12],

318 + choices=[

319 + GenerateResponseChoice(

320 + index=0,

321 + token_ids=generated_ids,

322 + finish_reason="length",

323 + )

324 + ],

325 + )

326 + request = ChatCompletionRequest(

327 + model=MODEL_NAME,

328 + messages=[{"role": "user", "content": "question"}],

329 + )

330 +

331 + choices = derenderer._derender_chat(response, request)

332 +

333 + parser.set_prompt_token_ids.assert_called_once_with([11, 12])

334 + assert choices[0].message.tool_calls[0].function.name == "tool"

335 + assert choices[0].finish_reason == "length"

336 +

337 +

338 class TestDetokenizeDelta:

339 """_detokenize_delta: chunked decode must equal one shot decode."""

340

tests/parser/test_response_template.py new

+1349

@@ -0,0 +1,1349 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

4 +import json

5 +from copy import deepcopy

6 +from types import SimpleNamespace

7 +from typing import Any

8 +

9 +import pytest

10 +from openai.types.responses import NamespaceTool

11 +

12 +from vllm.entrypoints.openai.chat_completion.protocol import (

13 + ChatCompletionNamedToolChoiceParam,

14 + ChatCompletionToolsParam,

15 +)

16 +from vllm.exceptions import VLLMValidationError

17 +from vllm.parser import response_template as response_template_module

18 +from vllm.parser.gemma4 import Gemma4Parser

19 +from vllm.parser.parser_manager import ParserManager

20 +from vllm.parser.response_template import (

21 + ResponseTemplateParser,

22 + ResponseTemplateReasoningParser,

23 + ResponseTemplateToolParser,

24 + resolve_response_template,

25 +)

26 +from vllm.reasoning import ReasoningParserManager

27 +from vllm.tool_parsers import ToolParserManager

28 +

29 +# Public metadata shared by the Gemma 4 instruction checkpoints.

30 +GEMMA4_RESPONSE_TEMPLATE: dict[str, Any] = {

31 + "defaults": {"role": "assistant"},

32 + "fields": {

33 + "content": {

34 + "close": ["<turn|>", "<|tool_response>", "<eos>"],

35 + "content": "text",

36 + },

37 + "thinking": {

38 + "close": "<channel|>",

39 + "content": "text",

40 + "open": "<|channel>thought\n",

41 + },

42 + "tool_calls": {

43 + "close": "<tool_call|>",

44 + "content": "json",

45 + "content_args": {

46 + "string_delims": [['<|"|>', '<|"|>']],

47 + "unquoted_keys": True,

48 + },

49 + "open_pattern": r"<\|tool_call>call:(?P<name>\w+)",

50 + "repeats": True,

51 + "transform": {

52 + "function": {

53 + "arguments": "{content}",

54 + "name": "{name}",

55 + },

56 + "type": "function",

57 + },

58 + },

59 + },

60 + "start_anchor": ["<|turn>model\n", "<tool_response|>"],

61 +}

62 +

63 +PREFIX = "<|turn>model\n"

64 +THINKING = "<|channel>thought\nplan<channel|>"

65 +TOOL_GENERATION = (

66 + '<|tool_call>call:set_alarm{hour:7,label:<|"|>morning<|"|>}<tool_call|>'

67 +)

68 +

69 +TOOLS = [

70 + ChatCompletionToolsParam(

71 + type="function",

72 + function={

73 + "name": "set_alarm",

74 + "parameters": {

75 + "type": "object",

76 + "properties": {

77 + "hour": {

78 + "oneOf": [

79 + {"type": "integer"},

80 + {"type": "null"},

81 + ]

82 + },

83 + "label": {"type": "string"},

84 + },

85 + },

86 + },

87 + ),

88 + ChatCompletionToolsParam(

89 + type="function",

90 + function={

91 + "name": "get_weather",

92 + "parameters": {

93 + "type": "object",

94 + "properties": {

95 + "city": {"type": "string"},

96 + "unit": {"type": "string"},

97 + },

98 + },

99 + },

100 + ),

101 + ChatCompletionToolsParam(

102 + type="function",

103 + function={

104 + "name": "set_status",

105 + "parameters": {

106 + "type": "object",

107 + "properties": {

108 + "is_active": {"type": "boolean"},

109 + "count": {"type": "integer"},

110 + "score": {"type": "number"},

111 + },

112 + },

113 + },

114 + ),

115 + ChatCompletionToolsParam(

116 + type="function",

117 + function={

118 + "name": "complex_function",

119 + "parameters": {

120 + "type": "object",

121 + "properties": {

122 + "nested": {

123 + "type": "object",

124 + "properties": {"inner": {"type": "string"}},

125 + },

126 + "list": {

127 + "type": "array",

128 + "items": {"type": "string"},

129 + },

130 + },

131 + },

132 + },

133 + ),

134 + ChatCompletionToolsParam(

135 + type="function",

136 + function={

137 + "name": "configure",

138 + "parameters": {

139 + "type": "object",

140 + "properties": {

141 + "enabled": {"type": "boolean"},

142 + "ratio": {"type": "number"},

143 + "label": {"type": "string"},

144 + "value": {"type": ["string", "null"]},

145 + },

146 + },

147 + },

148 + ),

149 + ChatCompletionToolsParam(

150 + type="function",

151 + function={

152 + "name": "get_status",

153 + "parameters": {"type": "object", "properties": {}},

154 + },

155 + ),

156 + ChatCompletionToolsParam(

157 + type="function",

158 + function={

159 + "name": "get-weather",

160 + "parameters": {

161 + "type": "object",

162 + "properties": {"city": {"type": "string"}},

163 + },

164 + },

165 + ),

166 + ChatCompletionToolsParam(

167 + type="function",

168 + function={

169 + "name": "search",

170 + "parameters": {

171 + "type": "object",

172 + "properties": {

173 + "query": {"type": "string"},

174 + "filters": {

175 + "type": "object",

176 + "properties": {

177 + "language": {"type": "string"},

178 + "min_stars": {"type": "integer"},

179 + },

180 + },

181 + },

182 + },

183 + },

184 + ),

185 +]

186 +

187 +VOCAB = {

188 + "<|tool_call>": 48,

189 + "<tool_call|>": 49,

190 + "<|channel>": 50,

191 + "<channel|>": 51,

192 + '<|"|>': 52,

193 + "<|turn>": 53,

194 + "<|tool_response>": 54,

195 + "<turn|>": 55,

196 +}

197 +PROMPT_TOKEN_IDS = [1]

198 +OPEN_REASONING_PROMPT_TOKEN_IDS = [53, 100, 50, 101]

199 +PROMPT_TOKEN_TEXT = {

200 + 100: "model\n",

201 + 101: "thought\n",

202 + 102: "plan",

203 + 103: "answer",

204 +}

205 +

206 +

207 +class FakeTokenizer:

208 + def __init__(self, response_template=None, *, prefix=PREFIX):

209 + self.response_template = response_template

210 + self.init_kwargs = {}

211 + self.name_or_path = ""

212 + self.prefix = prefix

213 + self.all_special_tokens = list(VOCAB)

214 + self.all_special_ids = list(VOCAB.values())

215 +

216 + def decode(self, token_ids, skip_special_tokens=False, **_kwargs):

217 + if token_ids == [2]:

218 + return PREFIX + "answer"

219 + if token_ids == OPEN_REASONING_PROMPT_TOKEN_IDS:

220 + return PREFIX + "<|channel>thought\n"

221 + if token_ids != PROMPT_TOKEN_IDS:

222 + inverse_vocab = {token_id: token for token, token_id in VOCAB.items()}

223 + return "".join(

224 + PROMPT_TOKEN_TEXT.get(token_id, inverse_vocab.get(token_id, ""))

225 + for token_id in token_ids

226 + )

227 + return self.prefix if token_ids else ""

228 +

229 + def get_vocab(self):

230 + return VOCAB

231 +

232 + def encode(self, text, add_special_tokens=False):

233 + del add_special_tokens

234 + if text == "plan":

235 + return [102]

236 + return list(text.encode())

237 +

238 +

239 +def request(tool_choice="auto", *, include_reasoning=True, **overrides):

240 + values = dict(

241 + tool_choice=tool_choice,

242 + include_reasoning=include_reasoning,

243 + tools=TOOLS,

244 + skip_special_tokens=True,

245 + spaces_between_special_tokens=True,

246 + include_stop_str_in_output=False,

247 + parallel_tool_calls=True,

248 + )

249 + values.update(overrides)

250 + return SimpleNamespace(**values)

251 +

252 +

253 +def normalize_calls(calls):

254 + return [(call.name, json.loads(call.arguments)) for call in calls or []]

255 +

256 +

257 +def feed_chunks(

258 + parser,

259 + chunks,

260 + req,

261 + *,

262 + prompt_token_ids=PROMPT_TOKEN_IDS,

263 +):

264 + deltas = []

265 + for index, chunk in enumerate(chunks):

266 + delta = parser.parse_delta(

267 + chunk,

268 + [],

269 + req,

270 + prompt_token_ids=prompt_token_ids if index == 0 else None,

271 + finished=index == len(chunks) - 1,

272 + )

273 + if delta is not None:

274 + deltas.append(delta)

275 + return deltas

276 +

277 +

278 +def collect_stream(

279 + parser,

280 + text,

281 + req,

282 + chunk_size,

283 + *,

284 + prompt_token_ids=PROMPT_TOKEN_IDS,

285 +):

286 + reasoning = []

287 + content = []

288 + calls: dict[int, dict[str, str]] = {}

289 + for start in range(0, len(text), chunk_size):

290 + end = min(start + chunk_size, len(text))

291 + delta = parser.parse_delta(

292 + text[start:end],

293 + [],

294 + req,

295 + prompt_token_ids=prompt_token_ids if start == 0 else None,

296 + finished=end == len(text),

297 + )

298 + if delta is None:

299 + continue

300 + if delta.reasoning:

301 + reasoning.append(delta.reasoning)

302 + if delta.content:

303 + content.append(delta.content)

304 + for call in delta.tool_calls:

305 + function = call.function

306 + assert function is not None

307 + slot = calls.setdefault(

308 + call.index,

309 + {"name": "", "arguments": ""},

310 + )

311 + slot["name"] += function.name or ""

312 + slot["arguments"] += function.arguments or ""

313 + normalized_calls = [

314 + (call["name"], json.loads(call["arguments"]))

315 + for _, call in sorted(calls.items())

316 + ]

317 + return "".join(reasoning), "".join(content), normalized_calls

318 +

319 +

320 +def test_resolve_response_template_from_tokenizer_configuration():

321 + tokenizer = FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE)

322 + assert resolve_response_template(tokenizer) == GEMMA4_RESPONSE_TEMPLATE

323 +

324 + tokenizer = FakeTokenizer()

325 + del tokenizer.response_template

326 + tokenizer.init_kwargs["response_template"] = GEMMA4_RESPONSE_TEMPLATE

327 + assert resolve_response_template(tokenizer) == GEMMA4_RESPONSE_TEMPLATE

328 +

329 + explicit = deepcopy(GEMMA4_RESPONSE_TEMPLATE)

330 + explicit["defaults"]["role"] = "custom"

331 + assert resolve_response_template(tokenizer, explicit) is explicit

332 +

333 +

334 +def test_response_template_parsers_are_registered():

335 + assert ToolParserManager.get_tool_parser("hf") is ResponseTemplateToolParser

336 + assert (

337 + ReasoningParserManager.get_reasoning_parser("hf")

338 + is ResponseTemplateReasoningParser

339 + )

340 +

341 +

342 +@pytest.mark.parametrize(

343 + ("reasoning", "tools"),

344 + [(True, True), (True, False), (False, True)],

345 + ids=["both", "reasoning", "tools"],

346 +)

347 +def test_parser_manager_selects_response_template_by_name(reasoning, tools):

348 + parser_cls = ParserManager.get_parser(

349 + tool_parser_name="hf" if tools else None,

350 + reasoning_parser_name="hf" if reasoning else None,

351 + enable_auto_tools=tools,

352 + tokenizer=FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE),

353 + )

354 +

355 + assert parser_cls is not None

356 + assert issubclass(parser_cls, ResponseTemplateParser)

357 + assert (parser_cls.reasoning_parser_cls is not None) is reasoning

358 + assert (parser_cls.tool_parser_cls is not None) is tools

359 +

360 +

361 +def test_parser_manager_validates_checkpoint_metadata_at_startup():

362 + with pytest.raises(TypeError, match="requires `response_template` metadata"):

363 + ParserManager.get_parser(

364 + reasoning_parser_name="hf",

365 + tokenizer=FakeTokenizer(),

366 + )

367 + with pytest.raises(TypeError, match="Invalid response_template"):

368 + ParserManager.get_parser(

369 + reasoning_parser_name="hf",

370 + tokenizer=FakeTokenizer({"fields": {}}),

371 + )

372 +

373 + template = deepcopy(GEMMA4_RESPONSE_TEMPLATE)

374 + del template["fields"]["tool_calls"]

375 + with pytest.raises(TypeError, match="tool_calls field"):

376 + ParserManager.get_parser(

377 + tool_parser_name="hf",

378 + enable_auto_tools=True,

379 + tokenizer=FakeTokenizer(template),

380 + )

381 + assert (

382 + ParserManager.get_parser(

383 + reasoning_parser_name="hf",

384 + tokenizer=FakeTokenizer(template),

385 + )

386 + is not None

387 + )

388 +

389 +

390 +def test_response_template_parser_rejects_other_parsers():

391 + with pytest.raises(TypeError, match="cannot be combined"):

392 + ParserManager.get_parser(

393 + tool_parser_name="hermes",

394 + reasoning_parser_name="hf",

395 + enable_auto_tools=True,

396 + tokenizer=FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE),

397 + )

398 +

399 +

400 +def test_request_chat_template_is_honored_with_warning(monkeypatch):

401 + warnings: list[str] = []

402 + monkeypatch.setattr(

403 + response_template_module.logger, "warning_once", warnings.append

404 + )

405 + parser = ResponseTemplateParser(FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), TOOLS)

406 +

407 + parser.adjust_request(request(chat_template="{{ messages }}"))

408 +

409 + assert len(warnings) == 1

410 + assert parser.response_template.fields

411 +

412 +

413 +def test_reasoning_gate_waits_for_content_region():

414 + parser = ResponseTemplateParser(

415 + FakeTokenizer(

416 + GEMMA4_RESPONSE_TEMPLATE,

417 + prefix=PREFIX + "<|channel>thought\n",

418 + ),

419 + TOOLS,

420 + )

421 +

422 + assert parser.is_reasoning_end([1]) is False

423 + assert parser.is_reasoning_end([2]) is True

424 +

425 +

426 +def test_reasoning_gate_ends_at_content_opener_held_by_prompt():

427 + template = deepcopy(GEMMA4_RESPONSE_TEMPLATE)

428 + template["fields"]["content"]["open_pattern"] = r"<\|channel>final\n"

429 + parser = ResponseTemplateParser(

430 + FakeTokenizer(template, prefix=PREFIX + "<|channel>final\n"),

431 + TOOLS,

432 + )

433 +

434 + assert parser.is_reasoning_end([1]) is True

435 +

436 +

437 +def streamed_reasoning_end(reasoner, output, prompt=PROMPT_TOKEN_IDS):

438 + """Index of the output token where the per-step gate reports the end."""

439 + for index, token in enumerate(output):

440 + if reasoner.is_reasoning_end_streaming(prompt + output[: index + 1], [token]):

441 + return index

442 + return None

443 +

444 +

445 +def test_streaming_reasoning_gate_matches_thinking_closer_without_decoding():

446 + tokenizer = FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE)

447 + reasoner = ResponseTemplateReasoningParser(tokenizer)

448 + tokenizer.decode = lambda *_args, **_kwargs: pytest.fail("decoded") # type: ignore[method-assign]

449 + output = [50, 101] + [102] * 100 + [51, 103]

450 +

451 + assert streamed_reasoning_end(reasoner, output) == len(output) - 2

452 + assert reasoner.is_reasoning_end_streaming(PROMPT_TOKEN_IDS + output, output[-3:])

453 +

454 +

455 +def test_reasoning_gate_honors_disabled_and_repeated_thinking():

456 + disabled = ResponseTemplateParser(

457 + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE),

458 + TOOLS,

459 + chat_template_kwargs={"enable_thinking": False},

460 + )

461 + assert disabled.is_reasoning_end([1]) is True

462 +

463 + repeated_template = deepcopy(GEMMA4_RESPONSE_TEMPLATE)

464 + repeated_template["fields"]["thinking"]["repeats"] = True

465 + repeated = ResponseTemplateParser(

466 + FakeTokenizer(repeated_template),

467 + TOOLS,

468 + response_template=repeated_template,

469 + )

470 + assert repeated.is_reasoning_end(OPEN_REASONING_PROMPT_TOKEN_IDS) is False

471 + assert repeated.is_reasoning_end([2]) is False

472 +

473 +

474 +@pytest.mark.parametrize(

475 + "text",

476 + [

477 + "Hello<turn|>",

478 + THINKING + "The answer is 42.<turn|>",

479 + THINKING + TOOL_GENERATION,

480 + "Let me check." + TOOL_GENERATION,

481 + (

482 + '<|tool_call>call:get_weather{city:<|"|>Paris<|"|>,'

483 + 'unit:<|"|>celsius<|"|>}<tool_call|>' + TOOL_GENERATION

484 + ),

485 + (

486 + "<|tool_call>call:complex_function{"

487 + 'nested:{inner:<|"|>value<|"|>},'

488 + 'list:[<|"|>a<|"|>,<|"|>b<|"|>]}'

489 + "<tool_call|>"

490 + ),

491 + ],

492 + ids=[

493 + "content",

494 + "reasoning",

495 + "tool",

496 + "content_then_tool",

497 + "multiple_tools",

498 + "nested_arguments",

499 + ],

500 +)

501 +def test_gemma4_response_template_matches_registered_parser(text):

502 + tokenizer = FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE)

503 + req = request()

504 + registered = Gemma4Parser(tokenizer, TOOLS)

505 + metadata = ResponseTemplateParser(

506 + tokenizer,

507 + TOOLS,

508 + response_template=GEMMA4_RESPONSE_TEMPLATE,

509 + )

510 + metadata.set_prompt_token_ids([1])

511 +

512 + expected_reasoning, expected_content, expected_calls = registered.parse(

513 + text,

514 + req,

515 + enable_auto_tools=True,

516 + )

517 + reasoning, content, calls = metadata.parse(

518 + text,

519 + req,

520 + enable_auto_tools=True,

521 + )

522 +

523 + assert reasoning == expected_reasoning

524 + assert content == expected_content

525 + assert normalize_calls(calls) == normalize_calls(expected_calls)

526 +

527 +

528 +def test_metadata_parser_keeps_already_decoded_nullable_values():

529 + text = (

530 + "<|tool_call>call:configure{"

531 + "enabled:false,ratio:5.0,label:null,value:null}"

532 + "<tool_call|>"

533 + )

534 + parser = ResponseTemplateParser(

535 + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE),

536 + TOOLS,

537 + response_template=GEMMA4_RESPONSE_TEMPLATE,

538 + )

539 + parser.set_prompt_token_ids([1])

540 +

541 + _, content, calls = parser.parse(

542 + text,

543 + request(),

544 + enable_auto_tools=True,

545 + )

546 +

547 + assert content is None

548 + assert normalize_calls(calls) == [

549 + (

550 + "configure",

551 + {

552 + "enabled": False,

553 + "ratio": 5.0,

554 + "label": None,

555 + "value": None,

556 + },

557 + )

558 + ]

559 +

560 +

561 +@pytest.mark.parametrize(

562 + "text",

563 + [

564 + THINKING + TOOL_GENERATION,

565 + THINKING + "The answer is 42.<turn|>",

566 + "Let me check. " + TOOL_GENERATION,

567 + (

568 + '<|tool_call>call:get_weather{city:<|"|>Paris<|"|>}<tool_call|>'

569 + + TOOL_GENERATION

570 + ),

571 + ],

572 + ids=[

573 + "reasoning_then_tool",

574 + "reasoning_then_content",

575 + "content_then_tool",

576 + "multiple_tools",

577 + ],

578 +)

579 +@pytest.mark.parametrize("chunk_size", [1, 7, 10_000])

580 +def test_gemma4_response_template_streaming_matches_registered_parser(

581 + text,

582 + chunk_size,

583 +):

584 + req = request()

585 + registered = Gemma4Parser(

586 + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE),

587 + TOOLS,

588 + )

589 + metadata = ResponseTemplateParser(

590 + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE),

591 + TOOLS,

592 + enable_auto_tools=True,

593 + )

594 +

595 + assert collect_stream(metadata, text, req, chunk_size) == collect_stream(

596 + registered,

597 + text,

598 + req,

599 + chunk_size,

600 + )

601 +

602 +

603 +@pytest.mark.parametrize(

604 + ("prompt_token_ids", "text"),

605 + [

606 + (

607 + OPEN_REASONING_PROMPT_TOKEN_IDS,

608 + "continued reasoning<channel|>Final answer<turn|>",

609 + ),

610 + (

611 + PROMPT_TOKEN_IDS,

612 + "Direct answer without reasoning.<turn|>",

613 + ),

614 + ],

615 + ids=["prefilled_reasoning", "direct_answer"],

616 +)

617 +def test_gemma4_response_template_prompt_state_matches_registered_parser(

618 + prompt_token_ids,

619 + text,

620 +):

621 + req = request()

622 + registered = Gemma4Parser(

623 + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE),

624 + TOOLS,

625 + chat_template_kwargs={"enable_thinking": True},

626 + )

627 + metadata = ResponseTemplateParser(

628 + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE),

629 + TOOLS,

630 + enable_auto_tools=True,

631 + )

632 +

633 + assert collect_stream(

634 + metadata,

635 + text,

636 + req,

637 + 1,

638 + prompt_token_ids=prompt_token_ids,

639 + ) == collect_stream(

640 + registered,

641 + text,

642 + req,

643 + 1,

644 + prompt_token_ids=prompt_token_ids,

645 + )

646 +

647 +

648 +def test_gemma4_response_template_reasoning_suppression_matches_registered_parser():

649 + req = request(include_reasoning=False)

650 + text = THINKING + "Final answer<turn|>"

651 +

652 + assert collect_stream(

653 + ResponseTemplateParser(

654 + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE),

655 + TOOLS,

656 + enable_auto_tools=True,

657 + ),

658 + text,

659 + req,

660 + 1,

661 + ) == collect_stream(

662 + Gemma4Parser(FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), TOOLS),

663 + text,

664 + req,

665 + 1,

666 + )

667 +

668 +

669 +def test_streamed_tool_call_is_emitted_when_its_region_parses():

670 + chunks = [

671 + "<|tool_call>call:set_alarm{",

672 + "hour:7,",

673 + 'label:<|"|>morning<|"|>}',

674 + "<tool_call|>",

675 + ]

676 + deltas = feed_chunks(

677 + ResponseTemplateParser(

678 + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE),

679 + TOOLS,

680 + enable_auto_tools=True,

681 + ),

682 + chunks,

683 + request(),

684 + )

685 +

686 + assert all(not delta.tool_calls for delta in deltas[:-1])

687 + calls = deltas[-1].tool_calls

688 + assert len(calls) == 1

689 + assert calls[0].id is not None

690 + assert calls[0].index == 0

691 + assert calls[0].function is not None

692 + assert calls[0].function.name == "set_alarm"

693 + assert json.loads(calls[0].function.arguments) == {

694 + "hour": 7,

695 + "label": "morning",

696 + }

697 +

698 +

699 +def test_response_template_emits_whole_call_from_one_chunk():

700 + parser = ResponseTemplateParser(

701 + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE),

702 + TOOLS,

703 + enable_auto_tools=True,

704 + )

705 +

706 + delta = parser.parse_delta(

707 + TOOL_GENERATION,

708 + [],

709 + request(),

710 + prompt_token_ids=PROMPT_TOKEN_IDS,

711 + finished=True,

712 + )

713 +

714 + assert delta is not None

715 + assert len(delta.tool_calls) == 1

716 + call = delta.tool_calls[0]

717 + assert call.id is not None

718 + assert call.function is not None

719 + assert call.function.name == "set_alarm"

720 + assert json.loads(call.function.arguments) == {"hour": 7, "label": "morning"}

721 +

722 +

723 +def test_response_template_takes_name_from_region_content():

724 + template = {

725 + "defaults": {"role": "assistant"},

726 + "start_anchor": PREFIX,

727 + "fields": {

728 + "tool_calls": {

729 + "open": "<tool>",

730 + "close": "</tool>",

731 + "content": "json",

732 + "transform": {

733 + "type": "function",

734 + "function": {

735 + "name": "{content.name}",

736 + "arguments": "{content.arguments}",

737 + },

738 + },

739 + }

740 + },

741 + }

742 + chunks = [

743 + "<tool>",

744 + '{"name":"set_',

745 + 'alarm","arguments":{"hour":7}}',

746 + "</tool>",

747 + ]

748 + parser = ResponseTemplateParser(

749 + FakeTokenizer(template),

750 + TOOLS,

751 + response_template=template,

752 + enable_auto_tools=True,

753 + )

754 +

755 + deltas = feed_chunks(parser, chunks, request())

756 +

757 + assert all(not delta.tool_calls for delta in deltas[:-1])

758 + assert len(deltas[-1].tool_calls) == 1

759 + call = deltas[-1].tool_calls[0]

760 + assert call.function is not None

761 + assert call.function.name == "set_alarm"

762 + assert json.loads(call.function.arguments) == {"hour": 7}

763 +

764 +

765 +def test_response_template_applies_literal_transformed_name():

766 + template = {

767 + "defaults": {"role": "assistant"},

768 + "start_anchor": PREFIX,

769 + "fields": {

770 + "tool_calls": {

771 + "open": "<call>",

772 + "close": "</call>",

773 + "content": "json",

774 + "transform": {

775 + "type": "function",

776 + "function": {

777 + "name": "set_alarm",

778 + "arguments": "{content}",

779 + },

780 + },

781 + }

782 + },

783 + }

784 + parser = ResponseTemplateParser(

785 + FakeTokenizer(template),

786 + TOOLS,

787 + response_template=template,

788 + enable_auto_tools=True,

789 + )

790 +

791 + deltas = feed_chunks(

792 + parser,

793 + ["<call>", '{"hour":7}', "</call>"],

794 + request(),

795 + )

796 +

797 + assert len(deltas) == 1

798 + assert deltas[0].tool_calls[0].function.name == "set_alarm"

799 + assert json.loads(deltas[0].tool_calls[0].function.arguments) == {"hour": 7}

800 +

801 +

802 +def test_gemma4_response_template_drops_incomplete_call():

803 + text = '<|tool_call>call:get_weather{city:<|"|>London'

804 + req = request()

805 + registered = Gemma4Parser(FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), TOOLS)

806 + metadata = ResponseTemplateParser(

807 + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE),

808 + TOOLS,

809 + )

810 + metadata.set_prompt_token_ids(PROMPT_TOKEN_IDS)

811 +

812 + _, _, registered_calls = registered.parse(

813 + text,

814 + req,

815 + enable_auto_tools=True,

816 + )

817 + _, metadata_content, metadata_calls = metadata.parse(

818 + text,

819 + req,

820 + enable_auto_tools=True,

821 + )

822 +

823 + assert normalize_calls(registered_calls) == [("get_weather", {"city": "London"})]

824 + assert metadata_content is None

825 + assert metadata_calls is None

826 +

827 +

828 +def test_gemma4_response_template_preserves_content_after_tool_call():

829 + text = TOOL_GENERATION + "Done.<turn|>"

830 + req = request()

831 + registered = Gemma4Parser(FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), TOOLS)

832 + metadata = ResponseTemplateParser(

833 + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE),

834 + TOOLS,

835 + )

836 + metadata.set_prompt_token_ids(PROMPT_TOKEN_IDS)

837 +

838 + _, registered_content, registered_calls = registered.parse(

839 + text,

840 + req,

841 + enable_auto_tools=True,

842 + )

843 + _, metadata_content, metadata_calls = metadata.parse(

844 + text,

845 + req,

846 + enable_auto_tools=True,

847 + )

848 +

849 + assert registered_content is None

850 + assert metadata_content == "Done."

851 + assert normalize_calls(metadata_calls) == normalize_calls(registered_calls)

852 +

853 +

854 +def test_gemma4_checkpoint_grammar_excludes_hyphenated_name():

855 + text = '<|tool_call>call:get-weather{city:<|"|>London<|"|>}<tool_call|>'

856 + req = request()

857 + registered = Gemma4Parser(FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), TOOLS)

858 + metadata = ResponseTemplateParser(

859 + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE),

860 + TOOLS,

861 + )

862 + metadata.set_prompt_token_ids(PROMPT_TOKEN_IDS)

863 +

864 + _, _, registered_calls = registered.parse(

865 + text,

866 + req,

867 + enable_auto_tools=True,

868 + )

869 + _, metadata_content, metadata_calls = metadata.parse(

870 + text,

871 + req,

872 + enable_auto_tools=True,

873 + )

874 +

875 + assert normalize_calls(registered_calls) == [("get-weather", {"city": "London"})]

876 + assert metadata_content is None

877 + assert metadata_calls is None

878 +

879 +

880 +@pytest.mark.parametrize(

881 + (

882 + "prefix_suffix",

883 + "generated",

884 + "enable_auto_tools",

885 + "expected_reasoning",

886 + "expected_content",

887 + ),

888 + [

889 + (

890 + "<|channel>thought\nprefill",

891 + " generated<channel|>",

892 + False,

893 + " generated",

894 + None,

895 + ),

896 + ("prefill ", "generated<turn|>", False, None, "generated"),

897 + ("prefill", "", False, None, None),

898 + (TOOL_GENERATION.split("{", 1)[0], "", True, None, None),

899 + ("<|chan", "", False, None, None),

900 + ("<|chan", "nel", False, None, "nel"),

901 + (

902 + "<|chan",

903 + "nel>thought\nreason<channel|>",

904 + False,

905 + "reason",

906 + None,

907 + ),

908 + ],

909 + ids=[

910 + "reasoning",

911 + "content",

912 + "empty_content",

913 + "empty_tool_open",

914 + "empty_partial_delimiter",

915 + "unresolved_partial_delimiter",

916 + "completed_partial_delimiter",

917 + ],

918 +)

919 +def test_streaming_prefill_is_state_only(

920 + prefix_suffix,

921 + generated,

922 + enable_auto_tools,

923 + expected_reasoning,

924 + expected_content,

925 +):

926 + tokenizer = FakeTokenizer(

927 + GEMMA4_RESPONSE_TEMPLATE,

928 + prefix=PREFIX + prefix_suffix,

929 + )

930 + parser = ResponseTemplateParser(

931 + tokenizer,

932 + TOOLS,

933 + enable_auto_tools=enable_auto_tools,

934 + )

935 +

936 + delta = parser.parse_delta(

937 + generated,

938 + [],

939 + request(),

940 + prompt_token_ids=[1],

941 + finished=True,

942 + )

943 +

944 + if expected_reasoning is None and expected_content is None:

945 + assert delta is None

946 + else:

947 + assert delta is not None

948 + assert delta.reasoning == expected_reasoning

949 + assert delta.content == expected_content

950 +

951 +

952 +def test_malformed_call_opened_across_prompt_boundary_is_dropped():

953 + parser = ResponseTemplateParser(

954 + FakeTokenizer(

955 + GEMMA4_RESPONSE_TEMPLATE,

956 + prefix=PREFIX + "<|tool_call>call:set_",

957 + ),

958 + TOOLS,

959 + enable_auto_tools=True,

960 + )

961 +

962 + opened = parser.parse_delta(

963 + "alarm{",

964 + [],

965 + request(),

966 + prompt_token_ids=[1],

967 + finished=False,

968 + )

969 + delta = parser.parse_delta(

970 + "hour:7}unexpected<tool_call|>",

971 + [],

972 + request(),

973 + finished=True,

974 + )

975 +

976 + assert opened is None

977 + assert delta is None

978 +

979 +

980 +def test_responses_namespace_tools_use_vllm_flattened_names():

981 + namespace = NamespaceTool(

982 + type="namespace",

983 + name="calendar",

984 + description="Calendar tools",

985 + tools=[

986 + {

987 + "type": "function",

988 + "name": "set_alarm",

989 + "description": "Set an alarm",

990 + "parameters": TOOLS[0].function.parameters,

991 + }

992 + ],

993 + )

994 + parser = ResponseTemplateParser(

995 + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE),

996 + [namespace],

997 + )

998 + parser.set_prompt_token_ids([1])

999 + output = TOOL_GENERATION.replace(

1000 + "call:set_alarm",

1001 + "call:calendar__set_alarm",

1002 + )

1003 +

1004 + _, content, calls = parser.parse(

1005 + output,

1006 + request(),

1007 + enable_auto_tools=True,

1008 + )

1009 +

1010 + assert content is None

1011 + assert normalize_calls(calls) == [

1012 + ("calendar__set_alarm", {"hour": 7, "label": "morning"})

1013 + ]

1014 +

1015 +

1016 +def test_streaming_unknown_call_is_forwarded():

1017 + text = TOOL_GENERATION.replace("call:set_alarm", "call:unknown")

1018 + parser = ResponseTemplateParser(

1019 + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE),

1020 + TOOLS,

1021 + enable_auto_tools=True,

1022 + )

1023 + _, content, calls = collect_stream(parser, text, request(), 7)

1024 +

1025 + assert content == ""

1026 + assert calls == [("unknown", {"hour": 7, "label": "morning"})]

1027 +

1028 +

1029 +@pytest.mark.parametrize(

1030 + "text",

1031 + [

1032 + TOOL_GENERATION.replace(

1033 + "<tool_call|>",

1034 + "unexpected<tool_call|>",

1035 + ),

1036 + TOOL_GENERATION.removesuffix("<tool_call|>") + "<tool_",

1037 + ],

1038 + ids=["malformed", "truncated"],

1039 +)

1040 +def test_streaming_invalid_call_is_dropped(text):

1041 + parser = ResponseTemplateParser(

1042 + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE),

1043 + TOOLS,

1044 + enable_auto_tools=True,

1045 + )

1046 + deltas = feed_chunks(

1047 + parser,

1048 + [text[start : start + 7] for start in range(0, len(text), 7)],

1049 + request(),

1050 + )

1051 +

1052 + assert deltas == []

1053 +

1054 +

1055 +def test_malformed_call_does_not_block_following_call():

1056 + parser = ResponseTemplateParser(

1057 + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE),

1058 + TOOLS,

1059 + enable_auto_tools=True,

1060 + )

1061 + malformed = TOOL_GENERATION.replace("<tool_call|>", "unexpected<tool_call|>")

1062 +

1063 + _, content, calls = collect_stream(

1064 + parser, malformed + TOOL_GENERATION, request(), 7

1065 + )

1066 +

1067 + assert content == ""

1068 + assert calls == [("set_alarm", {"hour": 7, "label": "morning"})]

1069 +

1070 +

1071 +def test_non_streaming_malformed_call_is_dropped():

1072 + text = TOOL_GENERATION.replace(

1073 + "<tool_call|>",

1074 + "unexpected<tool_call|>",

1075 + )

1076 + parser = ResponseTemplateParser(

1077 + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE),

1078 + TOOLS,

1079 + )

1080 + parser.set_prompt_token_ids([1])

1081 +

1082 + reasoning, content, calls = parser.parse(

1083 + text,

1084 + request(),

1085 + enable_auto_tools=True,

1086 + )

1087 +

1088 + assert reasoning is None

1089 + assert content is None

1090 + assert calls is None

1091 +

1092 +

1093 +def test_streaming_malformed_call_is_dropped_before_emission():

1094 + parser = ResponseTemplateParser(

1095 + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE),

1096 + TOOLS,

1097 + enable_auto_tools=True,

1098 + )

1099 + first = parser.parse_delta(

1100 + "hello<turn|>",

1101 + [],

1102 + request(),

1103 + prompt_token_ids=[1],

1104 + finished=False,

1105 + )

1106 + malformed = TOOL_GENERATION.replace(

1107 + "<tool_call|>",

1108 + "unexpected<tool_call|>",

1109 + )

1110 + failed = parser.parse_delta(

1111 + malformed,

1112 + [],

1113 + request(),

1114 + finished=False,

1115 + )

1116 +

1117 + assert first is not None

1118 + assert first.content == "hello"

1119 + assert failed is None

1120 +

1121 +

1122 +def test_prefilled_malformed_call_is_dropped():

1123 + opening, body = TOOL_GENERATION.split("{", 1)

1124 + malformed_body = "{" + body.replace(

1125 + "<tool_call|>",

1126 + "unexpected<tool_call|>",

1127 + )

1128 + tokenizer = FakeTokenizer(

1129 + GEMMA4_RESPONSE_TEMPLATE,

1130 + prefix=PREFIX + opening,

1131 + )

1132 + parser = ResponseTemplateParser(

1133 + tokenizer,

1134 + TOOLS,

1135 + enable_auto_tools=True,

1136 + )

1137 +

1138 + delta = parser.parse_delta(

1139 + malformed_body,

1140 + [],

1141 + request(),

1142 + prompt_token_ids=[1],

1143 + finished=True,

1144 + )

1145 +

1146 + assert delta is None

1147 +

1148 +

1149 +def test_tool_choice_none_preserves_raw_call_as_content():

1150 + parser = ResponseTemplateParser(

1151 + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE),

1152 + TOOLS,

1153 + enable_auto_tools=True,

1154 + )

1155 + _, content, calls = collect_stream(

1156 + parser,

1157 + TOOL_GENERATION,

1158 + request("none"),

1159 + 5,

1160 + )

1161 +

1162 + assert content == TOOL_GENERATION

1163 + assert calls == []

1164 +

1165 +

1166 +def test_adjust_request_preserves_parser_delimiters_without_forcing_stop_text():

1167 + req = request()

1168 + parser = ResponseTemplateParser(

1169 + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE),

1170 + TOOLS,

1171 + )

1172 +

1173 + assert parser.adjust_request(req) is req

1174 + assert req.skip_special_tokens is False

1175 + assert req.spaces_between_special_tokens is False

1176 + assert req.include_stop_str_in_output is False

1177 +

1178 +

1179 +def test_call_cut_off_before_its_closer_is_parsed():

1180 + """A stop-token closer, such as a tool closer that ends generation, is

1181 + absent from the text; the open region is parsed at the end of the stream."""

1182 + trimmed = TOOL_GENERATION.removesuffix("<tool_call|>")

1183 + parser = ResponseTemplateParser(FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), TOOLS)

1184 + streaming = ResponseTemplateParser(

1185 + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE),

1186 + TOOLS,

1187 + enable_auto_tools=True,

1188 + )

1189 +

1190 + _, content, calls = parser.parse(trimmed, request(), enable_auto_tools=True)

1191 + streamed = streaming.parse_delta(

1192 + trimmed,

1193 + [],

1194 + request(),

1195 + prompt_token_ids=[1],

1196 + finished=True,

1197 + )

1198 +

1199 + assert content is None

1200 + assert normalize_calls(calls) == [("set_alarm", {"hour": 7, "label": "morning"})]

1201 + assert streamed is not None

1202 + assert [call.function.name for call in streamed.tool_calls] == ["set_alarm"]

1203 +

1204 +

1205 +@pytest.mark.parametrize(

1206 + ("req", "match"),

1207 + [

1208 + (request("required"), "required tool choice"),

1209 + (

1210 + request(

1211 + ChatCompletionNamedToolChoiceParam(

1212 + type="function",

1213 + function={"name": "set_alarm"},

1214 + )

1215 + ),

1216 + "named tool choice",

1217 + ),

1218 + (request("auto", parallel_tool_calls=False), "parallel_tool_calls=False"),

1219 + (

1220 + request(

1221 + "auto",

1222 + tools=[

1223 + {

1224 + "type": "function",

1225 + "name": "set_alarm",

1226 + "parameters": {"type": "object"},

1227 + "strict": True,

1228 + }

1229 + ],

1230 + ),

1231 + "strict tools",

1232 + ),

1233 + ],

1234 +)

1235 +def test_unconstrained_tool_guarantees_are_rejected(req, match):

1236 + parser = ResponseTemplateParser(

1237 + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE),

1238 + req.tools,

1239 + enable_auto_tools=True,

1240 + )

1241 +

1242 + with pytest.raises(VLLMValidationError, match=match):

1243 + parser.adjust_request(req)

1244 +

1245 +

1246 +def test_tool_choice_none_does_not_require_structural_guarantees():

1247 + req = request(

1248 + "none",

1249 + tools=[

1250 + {

1251 + "type": "function",

1252 + "name": "set_alarm",

1253 + "parameters": {"type": "object"},

1254 + "strict": True,

1255 + }

1256 + ],

1257 + parallel_tool_calls=False,

1258 + )

1259 + parser = ResponseTemplateParser(

1260 + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE),

1261 + req.tools,

1262 + enable_auto_tools=True,

1263 + )

1264 +

1265 + assert parser.adjust_request(req) is req

1266 +

1267 +

1268 +def test_unsupported_semantic_field_is_rejected():

1269 + template = deepcopy(GEMMA4_RESPONSE_TEMPLATE)

1270 + template["fields"]["citations"] = {

1271 + "open": "<citation>",

1272 + "close": "</citation>",

1273 + }

1274 +

1275 + with pytest.raises(ValueError, match="citations"):

1276 + ResponseTemplateParser(

1277 + FakeTokenizer(template),

1278 + TOOLS,

1279 + response_template=template,

1280 + )

1281 +

1282 +

1283 +@pytest.mark.parametrize(

1284 + ("field_name", "field_update"),

1285 + [

1286 + ("content", {"content": "json"}),

1287 + ("thinking", {"transform": "{content}"}),

1288 + ],

1289 +)

1290 +def test_unsupported_content_field_semantics_are_rejected(

1291 + field_name,

1292 + field_update,

1293 +):

1294 + template = deepcopy(GEMMA4_RESPONSE_TEMPLATE)

1295 + template["fields"][field_name].update(field_update)

1296 +

1297 + with pytest.raises(ValueError, match="cannot be streamed"):

1298 + ResponseTemplateParser(

1299 + FakeTokenizer(template),

1300 + TOOLS,

1301 + response_template=template,

1302 + )

1303 +

1304 +

1305 +def test_delimited_template_keeps_plain_content_and_tolerates_required_fields():

1306 + template = {

1307 + "start_anchor": "<assistant>",

1308 + "fields": {

1309 + "thinking": {"open": "<think>", "close": "</think>"},

1310 + "tool_calls": {

1311 + "open": "<call>",

1312 + "close": "</call>",

1313 + "content": "json",

1314 + "optional": False,

1315 + },

1316 + },

1317 + }

1318 + parser = ResponseTemplateParser(

1319 + FakeTokenizer(template, prefix="<assistant>"),

1320 + TOOLS,

1321 + response_template=template,

1322 + )

1323 +

1324 + assert parser.parse("<think>plan</think>answer", request()) == (

1325 + "plan",

1326 + "answer",

1327 + None,

1328 + )

1329 +

1330 +

1331 +def test_separate_parse_calls_use_distinct_tool_call_ids():

1332 + parser = ResponseTemplateParser(

1333 + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE),

1334 + TOOLS,

1335 + enable_auto_tools=True,

1336 + )

1337 + parsed_calls = []

1338 + for _ in range(2):

1339 + _, _, calls = parser.parse(

1340 + TOOL_GENERATION,

1341 + request(),

1342 + enable_auto_tools=True,

1343 + )

1344 + assert calls is not None

1345 + parsed_calls.append(calls[0])

1346 +

1347 + assert parsed_calls[0].id is not None

1348 + assert parsed_calls[1].id is not None

1349 + assert parsed_calls[0].id != parsed_calls[1].id

vllm/entrypoints/launchers/cli_args.py

+11

@@ -22,6 +22,7 @@ from vllm.entrypoints.chat_utils import (

22 validate_chat_template,

23 )

24 from vllm.entrypoints.openai.models.protocol import LoRAModulePath

25 +from vllm.logger import init_logger

26 from vllm.tool_parsers import ToolParserManager

27 from vllm.tool_parsers.tool_strict_level import ToolStrictLevelName

28 from vllm.utils.argparse_utils import FlexibleArgumentParser

@@ -31,6 +32,8 @@ from .utils.constants import (

32 H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT,

33 )

35 +logger = init_logger(__name__)

36 +

38 class LoRAParserAction(argparse.Action):

39 def __call__(

@@ -466,6 +469,14 @@ def validate_parsed_serve_args(args: argparse.Namespace):

469

470 # Ensure that the chat template is valid; raises if it likely isn't

471 validate_chat_template(args.chat_template)

472 + if args.chat_template is not None and "hf" in (

473 + args.tool_call_parser,

474 + getattr(args, "reasoning_parser", None),

475 + ):

476 + logger.warning(

477 + "--chat-template is set; the hf parser still expects "

478 + "the checkpoint's output format."

479 + )

480

481 # Enable auto tool needs a tool call parser to be valid

482 if args.enable_auto_tool_choice and not args.tool_call_parser:

vllm/entrypoints/openai/chat_completion/batch_serving.py

+15 -15

@@ -129,17 +129,6 @@ class OpenAIServingChatBatch(OpenAIServingChat):

129 return render_result

130 single_requests, all_conversations, engine_prompts = render_result

131

132 - parser: Parser | None = None

133 - if self.parser_cls is not None:

134 - chat_template_kwargs = self._effective_chat_template_kwargs(

135 - single_requests[0]

136 - )

137 - parser = self.parser_cls(

138 - tokenizer,

139 - None, # tools

140 - chat_template_kwargs=chat_template_kwargs,

141 - )

142 -

132 request_id = (

133 f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"

134 )

@@ -202,7 +191,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):

191 all_conversations,

192 tokenizer,

193 request_metadata,

205 - parser,

194 + self.parser_cls,

195 )

196

197 async def chat_completion_full_generator_batch(

@@ -214,7 +203,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):

203 all_conversations: list[list[ConversationMessage]],

204 tokenizer: TokenizerLike,

205 request_metadata: RequestResponseMetadata,

217 - parser: Parser | None = None,

206 + parser_cls: type[Parser] | None = None,

207 ) -> ErrorResponse | ChatCompletionResponse:

208 """Handle batched (non-streaming) chat completions.

209

@@ -273,10 +262,21 @@ class OpenAIServingChatBatch(OpenAIServingChat):

262 else:

263 logprobs = None

264

276 - if parser is not None:

265 + if parser_cls is not None:

266 + single_request = request.to_chat_completion_request(

267 + request.messages[prompt_idx]

268 + )

269 + parser = parser_cls(

270 + tokenizer,

271 + None,

272 + chat_template_kwargs=self._effective_chat_template_kwargs(

273 + single_request

274 + ),

275 + )

276 + parser.set_prompt_token_ids(final_res.prompt_token_ids)

277 reasoning, content, _ = parser.parse(

278 output.text,

279 - request=request, # type: ignore[arg-type]

279 + request=single_request,

280 model_output_token_ids=output.token_ids,

281 )

282 if not request.include_reasoning:

vllm/entrypoints/openai/chat_completion/serving.py

@@ -165,6 +165,7 @@ class OpenAIServingChat(GenerateBaseServing):

165 tool_strict_level=tool_strict_level,

166 model_name=self.model_config.model,

167 is_harmony=self.model_config.hf_config.model_type == "gpt_oss",

168 + tokenizer=self.renderer.tokenizer,

169 )

170 self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none

171

@@ -974,6 +975,8 @@ class OpenAIServingChat(GenerateBaseServing):

975 parser = self._make_parser(

976 request, tokenizer, self._effective_chat_template_kwargs(request)

977 )

978 + if parser is not None and final_res.prompt_token_ids is not None:

979 + parser.set_prompt_token_ids(final_res.prompt_token_ids)

980 token_ids = output.token_ids

981 out_logprobs = output.logprobs

982

vllm/entrypoints/openai/responses/serving.py

+16 -1

@@ -141,6 +141,7 @@ class OpenAIServingResponses(GenerateBaseServing):

141 tool_strict_level=tool_strict_level,

142 model_name=self.model_config.model,

143 is_harmony=self.model_config.hf_config.model_type == "gpt_oss",

144 + tokenizer=self.renderer.tokenizer,

145 )

146 self.enable_prompt_tokens_details = enable_prompt_tokens_details

147 self.enable_force_include_usage = enable_force_include_usage

@@ -446,6 +447,12 @@ class OpenAIServingResponses(GenerateBaseServing):

447 response_parser = self._make_response_parser(

448 request, tokenizer, chat_template_kwargs

449 )

450 + if response_parser is not None:

451 + prompt_token_ids = self._extract_prompt_components(

452 + engine_input

453 + ).token_ids

454 + if prompt_token_ids is not None:

455 + response_parser.set_prompt_token_ids(prompt_token_ids)

456

457 context: ConversationContext

458 function_tool_names = extract_function_tool_names(request.tools)

@@ -703,6 +710,12 @@ class OpenAIServingResponses(GenerateBaseServing):

710 context.request,

711 context.response_messages,

712 )

713 + if context.response_parser is not None:

714 + prompt_token_ids = self._extract_prompt_components(

715 + engine_input

716 + ).token_ids

717 + if prompt_token_ids is not None:

718 + context.response_parser.set_prompt_token_ids(prompt_token_ids)

719

720 sampling_params.max_tokens = get_max_tokens(

721 max_model_len,

@@ -1187,7 +1200,9 @@ class OpenAIServingResponses(GenerateBaseServing):

1200 ) -> AsyncGenerator[StreamingResponsesResponse, None]:

1201 processor = SimpleStreamingEventProcessor(tools=request.tools)

1202

1190 - hide_stream_metadata = not request.include_reasoning and self.parser is not None

1203 + hide_stream_metadata = (

1204 + not request.include_reasoning and context.response_parser is not None

1205 + )

1206

1207 def _get_logprobs(output: CompletionOutput) -> list[Logprob]:

1208 if not request.is_include_output_logprobs():

vllm/parser/abstract_parser.py

@@ -115,6 +115,7 @@ class Parser:

115 tool_parser_cls: type[ToolParser] | None = None

116 # Server-side floor for tool-call structural tags (--tool-strict-level).

117 tool_strict_level: ToolStrictLevel = ToolStrictLevel.AUTO

118 + always_adjust_request: bool = False

119

120 def __init__(

121 self,

@@ -206,6 +207,10 @@ class Parser:

207 """

208 return request

209

210 + def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None:

211 + """Provide the exact rendered prompt to parsers that need prefix state."""

212 + return

213 +

214 @abstractmethod

215 def is_reasoning_end(self, input_ids: list[int]) -> bool:

216 """Check if the reasoning content ends in the input_ids.

vllm/parser/parser_manager.py

+40

@@ -11,10 +11,13 @@ from vllm.tool_parsers.tool_strict_level import ToolStrictLevel

11 if TYPE_CHECKING:

12 from vllm.parser.abstract_parser import Parser

13 from vllm.reasoning import ReasoningParser

14 + from vllm.tokenizers import TokenizerLike

15 from vllm.tool_parsers import ToolParser

17 logger = init_logger(__name__)

19 +HF_PARSER = "hf"

20 +

22 class ParserManager:

23 """Provides a unified Parser by composing reasoning and tool parser adapters."""

@@ -79,6 +82,7 @@ class ParserManager:

82 model_name: str | None = None,

83 is_harmony: bool = False,

84 tool_strict_level: str = "auto",

85 + tokenizer: TokenizerLike | None = None,

86 ) -> type[Parser] | None:

87 """Get a Parser that handles both reasoning and tool parsing.

@@ -93,6 +97,8 @@ class ParserManager:

97 If True, HarmonyParser is always returned.

98 tool_strict_level: Server-side floor for tool-call structural

99 tags (``--tool-strict-level``).

100 + tokenizer: Tokenizer whose `response_template` metadata is

101 + validated when the `hf` parser is selected.

102

103 Returns:

104 A Parser class, or None if neither parser is specified.

@@ -119,6 +125,40 @@ class ParserManager:

125 HarmonyParser.tool_strict_level = strict_level

126 return HarmonyParser

127

128 + if HF_PARSER in (reasoning_parser_name, tool_parser_name):

129 + if {reasoning_parser_name, tool_parser_name} - {

130 + HF_PARSER,

131 + None,

132 + "",

133 + }:

134 + raise TypeError(

135 + "The hf parser cannot be combined with other "

136 + "reasoning or tool call parsers"

137 + )

138 + from vllm.parser.response_template import (

139 + ResponseTemplateParser,

140 + validate_tokenizer_response_template,

141 + )

142 +

143 + if tokenizer is not None:

144 + validate_tokenizer_response_template(

145 + tokenizer,

146 + reasoning=reasoning_parser_cls is not None,

147 + tools=tool_parser_cls is not None,

148 + )

149 +

150 + r_cls = reasoning_parser_cls

151 + t_cls = tool_parser_cls

152 + auto_tools = enable_auto_tools

153 +

154 + class _ResponseTemplateParser(ResponseTemplateParser):

155 + reasoning_parser_cls = r_cls

156 + tool_parser_cls = t_cls

157 + tool_strict_level = strict_level

158 + _enable_auto_tools = auto_tools

159 +

160 + return _ResponseTemplateParser

161 +

162 if reasoning_parser_name == "kimi_k3" or tool_parser_name == "kimi_k3":

163 from vllm.parser.kimi_k3 import KimiK3Parser

164

vllm/parser/response_template.py new

+642

@@ -0,0 +1,642 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +"""Unified vLLM parser backed by checkpoint `response_template` metadata."""

4 +

5 +from __future__ import annotations

6 +

7 +import copy

8 +import json

9 +from collections.abc import Callable, Iterable, Sequence

10 +from typing import TYPE_CHECKING, Any, cast

11 +

12 +from openai.types.responses import ToolChoiceFunction

13 +

14 +from vllm.entrypoints.chat_utils import make_tool_call_id

15 +from vllm.entrypoints.generate.base.protocol import (

16 + DeltaFunctionCall,

17 + DeltaMessage,

18 + DeltaToolCall,

19 + FunctionCall,

20 +)

21 +from vllm.entrypoints.openai.chat_completion.protocol import (

22 + ChatCompletionNamedToolChoiceParam,

23 +)

24 +from vllm.exceptions import VLLMValidationError

25 +from vllm.logger import init_logger

26 +from vllm.parser.abstract_parser import Parser

27 +from vllm.parser.chat_parsing import ResponseParser

28 +from vllm.parser.chat_parsing.response_templates import (

29 + ResponseTemplate,

30 + load_response_template,

31 +)

32 +from vllm.reasoning.abs_reasoning_parsers import ReasoningParser

33 +from vllm.tool_parsers.abstract_tool_parser import ToolParser

34 +from vllm.tool_parsers.utils import iter_response_function_tool_dicts

35 +

36 +if TYPE_CHECKING:

37 + from vllm.entrypoints.openai.chat_completion.protocol import (

38 + ChatCompletionRequest,

39 + )

40 + from vllm.entrypoints.openai.responses.protocol import ResponsesRequest

41 + from vllm.tokenizers import TokenizerLike

42 + from vllm.tool_parsers.abstract_tool_parser import Tool

43 +

44 +logger = init_logger(__name__)

45 +

46 +THINKING_FIELD = "thinking"

47 +CONTENT_FIELD = "content"

48 +TOOL_FIELD = "tool_calls"

49 +_SUPPORTED_FIELDS = frozenset({THINKING_FIELD, CONTENT_FIELD, TOOL_FIELD})

50 +

51 +

52 +def _tool_dict(tool: Any) -> dict[str, Any]:

53 + if hasattr(tool, "model_dump"):

54 + tool = tool.model_dump(exclude_none=True)

55 + return tool if isinstance(tool, dict) else {}

56 +

57 +

58 +def _as_tool_dicts(tools: Sequence[Any] | None) -> list[dict[str, Any]]:

59 + """Responses function tools with flattened namespaces, plus chat tools."""

60 + tools = list(tools or ())

61 + result = iter_response_function_tool_dicts(tools)

62 + result.extend(

63 + tool_dict

64 + for tool_dict in map(_tool_dict, tools)

65 + if isinstance(tool_dict.get("function"), dict)

66 + )

67 + return result

68 +

69 +

70 +def _tool_is_strict(tool: Any) -> bool:

71 + tool_dict = _tool_dict(tool)

72 + function = tool_dict.get("function")

73 + return bool(

74 + tool_dict.get("strict")

75 + or (isinstance(function, dict) and function.get("strict"))

76 + )

77 +

78 +

79 +def _decode(tokenizer: TokenizerLike, token_ids: Sequence[int]) -> str:

80 + try:

81 + return cast(Any, tokenizer.decode)(

82 + token_ids,

83 + skip_special_tokens=False,

84 + clean_up_tokenization_spaces=False,

85 + spaces_between_special_tokens=False,

86 + )

87 + except TypeError:

88 + return tokenizer.decode(token_ids, skip_special_tokens=False)

89 +

90 +

91 +def resolve_response_template(

92 + tokenizer: Any | None,

93 + template: dict[str, Any] | None = None,

94 +) -> dict[str, Any] | None:

95 + """Return `template` if given, otherwise the tokenizer's metadata."""

96 + if template is not None or tokenizer is None:

97 + return template

98 + template = getattr(tokenizer, "response_template", None)

99 + if not isinstance(template, dict):

100 + init_kwargs = getattr(tokenizer, "init_kwargs", None) or {}

101 + template = init_kwargs.get("response_template")

102 + return template if isinstance(template, dict) else None

103 +

104 +

105 +def validate_response_template_for_serving(

106 + template: dict[str, Any],

107 +) -> ResponseTemplate:

108 + """Validate the template and reject semantic fields serving cannot route."""

109 + loaded = load_response_template(template)

110 + unsupported = (set(loaded.fields) - _SUPPORTED_FIELDS) | (

111 + set(loaded.defaults) - {"role"}

112 + )

113 + if unsupported:

114 + raise ValueError(

115 + "response_template contains unsupported semantic fields: "

116 + f"{sorted(unsupported)}. Supported fields are: "

117 + f"{sorted(_SUPPORTED_FIELDS)}"

118 + )

119 + for name in (THINKING_FIELD, CONTENT_FIELD):

120 + field = loaded.fields.get(name)

121 + if field is None:

122 + continue

123 + if (

124 + field.content != "text"

125 + or field.transform is not None

126 + or field.join is not None

127 + ):

128 + raise ValueError(

129 + f"response_template field {name!r} uses semantics that cannot "

130 + "be streamed by the OpenAI serving adapter"

131 + )

132 + return loaded

133 +

134 +

135 +def validate_tokenizer_response_template(

136 + tokenizer: Any,

137 + *,

138 + reasoning: bool,

139 + tools: bool,

140 +) -> None:

141 + """Fail at startup if the tokenizer cannot serve the requested parsing."""

142 + template = resolve_response_template(tokenizer)

143 + if template is None:

144 + raise TypeError(

145 + "The hf parser requires `response_template` metadata "

146 + "in the tokenizer configuration"

147 + )

148 + try:

149 + fields = validate_response_template_for_serving(template).fields

150 + except (TypeError, ValueError) as exc:

151 + raise TypeError(f"Invalid response_template metadata: {exc}") from exc

152 + if reasoning and THINKING_FIELD not in fields:

153 + raise TypeError(

154 + "--reasoning-parser hf requires a response_template with a thinking field"

155 + )

156 + if tools and TOOL_FIELD not in fields:

157 + raise TypeError(

158 + "--tool-call-parser hf requires a response_template with a tool_calls field"

159 + )

160 +

161 +

162 +def _streaming_template(template: dict[str, Any]) -> ResponseTemplate:

163 + result = copy.deepcopy(template)

164 + fields = result["fields"]

165 + if CONTENT_FIELD not in fields and all(

166 + "open" in field or "open_pattern" in field for field in fields.values()

167 + ):

168 + fields[CONTENT_FIELD] = {}

169 + for field in fields.values():

170 + field["optional"] = True

171 + return load_response_template(result)

172 +

173 +

174 +def _is_named_tool_choice(request: ChatCompletionRequest | ResponsesRequest) -> bool:

175 + return isinstance(

176 + request.tool_choice, (ToolChoiceFunction, ChatCompletionNamedToolChoiceParam)

177 + )

178 +

179 +

180 +def _tool_parsing_enabled(

181 + request: ChatCompletionRequest | ResponsesRequest,

182 + *,

183 + enable_auto_tools: bool,

184 +) -> bool:

185 + if request.tool_choice == "none":

186 + return False

187 + if request.tool_choice == "required" or _is_named_tool_choice(request):

188 + return True

189 + return enable_auto_tools and request.tool_choice in (None, "auto")

190 +

191 +

192 +def _unsupported_tool_guarantee(

193 + request: ChatCompletionRequest | ResponsesRequest,

194 +) -> tuple[str, str] | None:

195 + """The first requested tool guarantee this parser cannot enforce, with the

196 + request parameter that asked for it."""

197 + if any(_tool_is_strict(tool) for tool in request.tools or ()):

198 + return "strict tools", "tools"

199 + if request.tool_choice == "required":

200 + return "required tool choice", "tool_choice"

201 + if _is_named_tool_choice(request):

202 + return "named tool choice", "tool_choice"

203 + if getattr(request, "parallel_tool_calls", True) is False:

204 + return "parallel_tool_calls=False", "parallel_tool_calls"

205 + return None

206 +

207 +

208 +def _reasoning_has_ended(

209 + parser: ResponseParser,

210 + template: ResponseTemplate,

211 + *,

212 + thinking_disabled: bool = False,

213 +) -> bool:

214 + thinking = template.fields.get(THINKING_FIELD)

215 + if thinking is None:

216 + return True

217 + if thinking.repeats:

218 + return False

219 +

220 + active_field: str | None = None

221 + thinking_closed = False

222 + for event in parser.initial_events:

223 + if event["type"] == "region_open":

224 + active_field = event["field"]

225 + elif event["type"] in {"region_close", "region_malformed"}:

226 + thinking_closed |= event["field"] == THINKING_FIELD

227 + active_field = None

228 + if thinking_closed:

229 + return True

230 + if active_field in (CONTENT_FIELD, TOOL_FIELD):

231 + return True

232 +

233 + # Resolve delimiters held at the prompt boundary. A NUL cannot occur in

234 + # the model wire format and any implicit chunk containing it is a probe,

235 + # not evidence that content generation has started.

236 + for event in parser.feed("\0"):

237 + if (

238 + event["type"] == "region_open"

239 + and event["field"] in (CONTENT_FIELD, TOOL_FIELD)

240 + and event["end"] > event["start"]

241 + ):

242 + return True

243 + if event["field"] == THINKING_FIELD:

244 + if event["type"] == "region_open":

245 + return False

246 + if event["type"] in {"region_close", "region_malformed"}:

247 + return True

248 + if active_field == THINKING_FIELD:

249 + return False

250 + return thinking_disabled

251 +

252 +

253 +def _literal_ids(

254 + tokenizer: TokenizerLike, vocab: dict[str, int], literals: Sequence[str]

255 +) -> list[list[int]] | None:

256 + """Token ids of each literal, or `None` when one encodes to nothing."""

257 + ids = [

258 + [vocab[literal]]

259 + if literal in vocab

260 + else tokenizer.encode(literal, add_special_tokens=False)

261 + for literal in literals

262 + ]

263 + return ids if all(ids) else None

264 +

265 +

266 +def _reasoning_end_ids(

267 + tokenizer: TokenizerLike, vocab: dict[str, int], template: ResponseTemplate

268 +) -> list[list[int]] | None:

269 + """Token ids of each thinking closer, or `None` when they are not all

270 + literal."""

271 + thinking = template.fields.get(THINKING_FIELD)

272 + if thinking is None or thinking.repeats or not thinking.close_literals:

273 + return None

274 + return _literal_ids(tokenizer, vocab, thinking.close_literals)

275 +

276 +

277 +def _ends_within(token_ids: Sequence[int], end_ids: list[int], num_new: int) -> bool:

278 + """Whether `end_ids` completes within the last `num_new` of `token_ids`."""

279 + window = list(token_ids[max(0, len(token_ids) - num_new - len(end_ids) + 1) :])

280 + return any(

281 + window[index : index + len(end_ids)] == end_ids

282 + for index in range(len(window) - len(end_ids) + 1)

283 + )

284 +

285 +

286 +def _serving_template(

287 + tokenizer: Any,

288 + template: dict[str, Any] | None,

289 +) -> ResponseTemplate:

290 + """The validated streaming form of `template` or the tokenizer's metadata."""

291 + template = resolve_response_template(tokenizer, template)

292 + if template is None:

293 + raise ValueError(

294 + "The hf parser requires `response_template` "

295 + "metadata in the tokenizer configuration"

296 + )

297 + validate_response_template_for_serving(template)

298 + return _streaming_template(template)

299 +

300 +

301 +def _to_tool_calls(value: Any) -> list[tuple[str, str]]:

302 + """`(name, arguments)` of each call in a tool region value, or nothing if

303 + any call is incomplete."""

304 + values = value if isinstance(value, list) else [value]

305 + calls: list[tuple[str, str]] = []

306 + for item in values:

307 + function = item.get("function") if isinstance(item, dict) else None

308 + if not isinstance(function, dict):

309 + return []

310 + name = function.get("name")

311 + arguments = function.get("arguments")

312 + if not isinstance(name, str) or not name or arguments is None:

313 + return []

314 + if not isinstance(arguments, str):

315 + arguments = json.dumps(arguments, ensure_ascii=False)

316 + calls.append((name, arguments))

317 + return calls

318 +

319 +

320 +class ResponseTemplateReasoningParser(ReasoningParser):

321 + """Reasoning end detection for `--reasoning-parser hf`.

322 +

323 + `ResponseTemplateParser` parses the output; this parser tells it and the

324 + structured-output engine where reasoning ends.

325 + """

326 +

327 + def __init__(

328 + self,

329 + tokenizer: TokenizerLike,

330 + *args,

331 + response_template: dict[str, Any] | None = None,

332 + **kwargs,

333 + ) -> None:

334 + super().__init__(tokenizer, *args, **kwargs)

335 + self.response_template = _serving_template(tokenizer, response_template)

336 + chat_template_kwargs = kwargs.get("chat_template_kwargs") or {}

337 + self._thinking_disabled = not chat_template_kwargs.get("enable_thinking", True)

338 + self._reasoning_end_ids = _reasoning_end_ids(

339 + tokenizer, self.vocab, self.response_template

340 + )

341 +

342 + @property

343 + def reasoning_start_str(self) -> str | None:

344 + thinking = self.response_template.fields.get(THINKING_FIELD)

345 + literals = thinking.open_literals if thinking else None

346 + return literals[0] if literals else None

347 +

348 + @property

349 + def reasoning_end_str(self) -> str | None:

350 + thinking = self.response_template.fields.get(THINKING_FIELD)

351 + literals = thinking.close_literals if thinking else None

352 + return literals[0] if literals else None

353 +

354 + def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:

355 + try:

356 + parser = ResponseParser(

357 + self.response_template,

358 + prefix=_decode(self.model_tokenizer, input_ids),

359 + )

360 + except Exception:

361 + return False

362 + return _reasoning_has_ended(

363 + parser,

364 + self.response_template,

365 + thinking_disabled=self._thinking_disabled,

366 + )

367 +

368 + def is_reasoning_end_streaming(

369 + self, input_ids: Sequence[int], delta_ids: Iterable[int]

370 + ) -> bool:

371 + if self._reasoning_end_ids is None:

372 + return self.is_reasoning_end(input_ids)

373 + num_new = len(list(delta_ids))

374 + return any(

375 + _ends_within(input_ids, end_ids, num_new)

376 + for end_ids in self._reasoning_end_ids

377 + )

378 +

379 + def extract_content_ids(self, input_ids: list[int]) -> list[int]:

380 + raise NotImplementedError("ResponseTemplateParser extracts content")

381 +

382 + def extract_reasoning(

383 + self,

384 + model_output: str,

385 + request: ChatCompletionRequest | ResponsesRequest,

386 + ) -> tuple[str | None, str | None]:

387 + raise NotImplementedError("ResponseTemplateParser extracts reasoning")

388 +

389 + def extract_reasoning_streaming(

390 + self,

391 + previous_text: str,

392 + current_text: str,

393 + delta_text: str,

394 + previous_token_ids: Sequence[int],

395 + current_token_ids: Sequence[int],

396 + delta_token_ids: Sequence[int],

397 + ) -> DeltaMessage | None:

398 + raise NotImplementedError("ResponseTemplateParser extracts reasoning")

399 +

400 +

401 +class ResponseTemplateToolParser(ToolParser):

402 + """Registers `--tool-call-parser hf`; `ResponseTemplateParser` extracts the

403 + tool calls."""

404 +

405 +

406 +class _DeltaBuilder:

407 + """Accumulates one `DeltaMessage`."""

408 +

409 + def __init__(self) -> None:

410 + self.reasoning: list[str] = []

411 + self.content: list[str] = []

412 + self.tool_calls: list[DeltaToolCall] = []

413 +

414 + def build(self) -> DeltaMessage | None:

415 + fields: dict[str, Any] = {}

416 + if reasoning := "".join(self.reasoning):

417 + fields["reasoning"] = reasoning

418 + if content := "".join(self.content):

419 + fields["content"] = content

420 + if self.tool_calls:

421 + fields["tool_calls"] = self.tool_calls

422 + return DeltaMessage(**fields) if fields else None

423 +

424 +

425 +class _ResponseStream:

426 + """One generation fed through `ResponseParser`, routed to message deltas."""

427 +

428 + def __init__(

429 + self,

430 + template: ResponseTemplate,

431 + *,

432 + prefix: str,

433 + tools: list[dict[str, Any]],

434 + parse_reasoning: bool,

435 + include_reasoning: bool,

436 + parse_tools: bool,

437 + new_tool_call_id: Callable[[str], str],

438 + ) -> None:

439 + self.parse_reasoning = parse_reasoning

440 + self.include_reasoning = include_reasoning

441 + self.parse_tools = parse_tools

442 + self.new_tool_call_id = new_tool_call_id

443 + self.parser = ResponseParser(template, prefix=prefix, tools=tools)

444 + self._prefix_end = len(self.parser.input_text)

445 + self._next_tool_index = 0

446 + self._finalized = False

447 +

448 + def feed(self, text: str, *, finished: bool) -> DeltaMessage | None:

449 + if self._finalized:

450 + return None

451 + delta = _DeltaBuilder()

452 + self._route(self.parser.feed(text), delta)

453 + if finished:

454 + self._finalized = True

455 + if len(self.parser.input_text) > self._prefix_end:

456 + _, events = self.parser.finalize()

457 + self._route(events, delta)

458 + return delta.build()

459 +

460 + def _route(self, events: Sequence[dict[str, Any]], delta: _DeltaBuilder) -> None:

461 + for event in events:

462 + field = event.get("field")

463 + if field == TOOL_FIELD:

464 + self._route_tool_event(event, delta)

465 + elif event["type"] == "region_chunk":

466 + if field == THINKING_FIELD and self.parse_reasoning:

467 + if self.include_reasoning:

468 + delta.reasoning.append(event["text"])

469 + elif field in (THINKING_FIELD, CONTENT_FIELD):

470 + delta.content.append(event["text"])

471 +

472 + def _route_tool_event(self, event: dict[str, Any], delta: _DeltaBuilder) -> None:

473 + """Emit the calls of a tool region once it ends and parses; a region cut

474 + off at the end of the stream is parsed as generated. With tool parsing

475 + disabled, the region passes through as content."""

476 + if not self.parse_tools:

477 + if event["type"] == "region_chunk":

478 + delta.content.append(event["text"])

479 + else:

480 + start = max(event["start"], self._prefix_end)

481 + delta.content.append(self.parser.input_text[start : event["end"]])

482 + return

483 + if event["type"] == "region_malformed":

484 + logger.warning("response_template: dropping a malformed tool call")

485 + elif event["type"] == "region_close":

486 + calls = _to_tool_calls(event.get("value"))

487 + if not calls:

488 + logger.warning("response_template: dropping an incomplete tool call")

489 + for name, arguments in calls:

490 + delta.tool_calls.append(

491 + DeltaToolCall(

492 + index=self._next_tool_index,

493 + id=self.new_tool_call_id(name),

494 + type="function",

495 + function=DeltaFunctionCall(name=name, arguments=arguments),

496 + )

497 + )

498 + self._next_tool_index += 1

499 +

500 +

501 +class ResponseTemplateParser(Parser):

502 + """Parse reasoning, content, and tool calls with one response template."""

503 +

504 + reasoning_parser_cls: type[ReasoningParser] | None = ResponseTemplateReasoningParser

505 + tool_parser_cls: type[ToolParser] | None = ResponseTemplateToolParser

506 + always_adjust_request = True

507 + _enable_auto_tools = False

508 +

509 + def __init__(

510 + self,

511 + tokenizer: TokenizerLike,

512 + tools: list[Tool] | None = None,

513 + *,

514 + response_template: dict[str, Any] | None = None,

515 + enable_auto_tools: bool | None = None,

516 + **kwargs,

517 + ) -> None:

518 + self.response_template = _serving_template(tokenizer, response_template)

519 + super().__init__(

520 + tokenizer, tools, response_template=response_template, **kwargs

521 + )

522 + if enable_auto_tools is not None:

523 + self._enable_auto_tools = enable_auto_tools

524 + self._tools = _as_tool_dicts(tools)

525 + self._prompt_token_ids: list[int] | None = None

526 + self._prefix = ""

527 + self._stream: _ResponseStream | None = None

528 +

529 + def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None:

530 + prompt_token_ids = list(prompt_token_ids)

531 + if prompt_token_ids == self._prompt_token_ids:

532 + return

533 + self._prompt_token_ids = prompt_token_ids

534 + self._prefix = _decode(self.model_tokenizer, prompt_token_ids)

535 + self._stream = None

536 +

537 + def adjust_request(

538 + self,

539 + request: ChatCompletionRequest | ResponsesRequest,

540 + ) -> ChatCompletionRequest | ResponsesRequest:

541 + if getattr(request, "chat_template", None) is not None:

542 + logger.warning_once(

543 + "A request chat template is set; response_template parsing still "

544 + "expects the checkpoint's output format."

545 + )

546 + request.skip_special_tokens = False

547 + if hasattr(request, "spaces_between_special_tokens"):

548 + request.spaces_between_special_tokens = False

549 + if (

550 + self._tool_parser is not None

551 + and request.tools

552 + and request.tool_choice != "none"

553 + and (unsupported := _unsupported_tool_guarantee(request)) is not None

554 + ):

555 + guarantee, parameter = unsupported

556 + raise VLLMValidationError(

557 + f"response_template parsing cannot guarantee {guarantee} "

558 + "without a format-compatible structural grammar",

559 + parameter=parameter,

560 + )

561 + return request

562 +

563 + def is_reasoning_end(self, input_ids: list[int]) -> bool:

564 + return (

565 + self._reasoning_parser is None

566 + or self._reasoning_parser.is_reasoning_end(input_ids)

567 + )

568 +

569 + def parse(

570 + self,

571 + model_output: str,

572 + request: ChatCompletionRequest | ResponsesRequest,

573 + enable_auto_tools: bool = False,

574 + model_output_token_ids: Sequence[int] = (),

575 + ) -> tuple[str | None, str | None, list[FunctionCall] | None]:

576 + self._initialize_history_tool_call_cnt(request)

577 + stream = self._new_stream(

578 + request,

579 + enable_auto_tools=enable_auto_tools,

580 + include_reasoning=True,

581 + )

582 + delta = stream.feed(model_output, finished=True)

583 + if delta is None:

584 + return None, None, None

585 + tool_calls = [

586 + FunctionCall(

587 + id=call.id,

588 + name=call.function.name or "",

589 + arguments=call.function.arguments or "{}",

590 + )

591 + for call in delta.tool_calls

592 + if call.function is not None

593 + ]

594 + return delta.reasoning, delta.content, tool_calls or None

595 +

596 + def parse_delta(

597 + self,

598 + delta_text: str,

599 + delta_token_ids: list[int],

600 + request: ChatCompletionRequest | ResponsesRequest,

601 + prompt_token_ids: list[int] | None = None,

602 + *,

603 + finished: bool,

604 + ) -> DeltaMessage | None:

605 + self._initialize_history_tool_call_cnt(request)

606 + if self._prompt_token_ids is None and prompt_token_ids is not None:

607 + self.set_prompt_token_ids(prompt_token_ids)

608 + if self._stream is None:

609 + self._stream = self._new_stream(

610 + request,

611 + enable_auto_tools=self._enable_auto_tools,

612 + include_reasoning=request.include_reasoning,

613 + )

614 + return self._stream.feed(delta_text, finished=finished)

615 +

616 + def _new_stream(

617 + self,

618 + request: ChatCompletionRequest | ResponsesRequest,

619 + *,

620 + enable_auto_tools: bool,

621 + include_reasoning: bool,

622 + ) -> _ResponseStream:

623 + return _ResponseStream(

624 + self.response_template,

625 + prefix=self._prefix,

626 + tools=self._tools,

627 + parse_reasoning=self._reasoning_parser is not None,

628 + include_reasoning=include_reasoning,

629 + parse_tools=self._tool_parser is not None

630 + and _tool_parsing_enabled(request, enable_auto_tools=enable_auto_tools),

631 + new_tool_call_id=self._new_tool_call_id,

632 + )

633 +

634 + def _new_tool_call_id(self, name: str) -> str:

635 + state = self._stream_state

636 + tool_call_id = make_tool_call_id(

637 + id_type=state.tool_call_id_type,

638 + func_name=name,

639 + idx=state.history_tool_call_cnt,

640 + )

641 + state.history_tool_call_cnt += 1

642 + return tool_call_id

vllm/reasoning/__init__.py

@@ -80,6 +80,10 @@ _REASONING_PARSERS_TO_REGISTER = {

80 "granite_thinking_engine_reasoning_parser",

81 "GraniteThinkingParserReasoningAdapter",

82 ),

83 + "hf": (

84 + "response_template_reasoning_parser",

85 + "ResponseTemplateReasoningParser",

86 + ),

87 "holo2": (

88 "deepseek_v3_reasoning_parser",

89 "DeepSeekV3ReasoningWithThinkingParser",

vllm/reasoning/response_template_reasoning_parser.py new

@@ -0,0 +1,6 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

4 +from vllm.parser.response_template import ResponseTemplateReasoningParser

5 +

6 +__all__ = ["ResponseTemplateReasoningParser"]

vllm/renderers/online_derenderer.py

@@ -79,6 +79,7 @@ class OnlineDerenderer:

79 tool_strict_level=tool_strict_level,

80 model_name=model_config.model,

81 is_harmony=self.use_harmony,

82 + tokenizer=renderer.tokenizer,

83 )

85 self.chat_template = chat_template

@@ -160,6 +161,8 @@ class OnlineDerenderer:

161 chat_template_kwargs=chat_template_kwargs,

162 model_config=self.model_config,

163 )

164 + if generate_response.prompt_token_ids is not None:

165 + parser.set_prompt_token_ids(generate_response.prompt_token_ids)

166 reasoning, content, tool_calls = parser.parse(

167 decoded_text,

168 chat_request,

vllm/renderers/online_renderer.py

+3 -1

@@ -183,6 +183,7 @@ class OnlineRenderer:

183 tool_strict_level=tool_strict_level,

184 model_name=model_config.model,

185 is_harmony=self.use_harmony,

186 + tokenizer=renderer.tokenizer,

187 )

188

189 self.chat_template = chat_template

@@ -825,7 +826,8 @@ class OnlineRenderer:

826 and tokenizer.supports_grammar

827 )

828 should_adjust_request = (

828 - parser.reasoning_parser_cls is not None

829 + parser.always_adjust_request

830 + or parser.reasoning_parser_cls is not None

831 or tool_choice != "none"

832 or is_mistral_grammar_eligible

833 )

vllm/tool_parsers/__init__.py

@@ -86,6 +86,10 @@ _TOOL_PARSERS_TO_REGISTER = {

86 "hermes_tool_parser",

87 "Hermes2ProToolParser",

88 ),

89 + "hf": (

90 + "response_template_tool_parser",

91 + "ResponseTemplateToolParser",

92 + ),

93 "poolside_v1": (

94 "poolside_v1_tool_parser",

95 "PoolsideV1ToolParser",

vllm/tool_parsers/response_template_tool_parser.py new

@@ -0,0 +1,6 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

4 +from vllm.parser.response_template import ResponseTemplateToolParser

5 +

6 +__all__ = ["ResponseTemplateToolParser"]