@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 73a5831127

[Frontend] Port chat_parsing core from Transformers (#58602)

Signed-off-by: Yoni Gozlan <yonigozlan@users.noreply.github.com> Co-authored-by: Yoni Gozlan <yonigozlan@users.noreply.github.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

Yoni Gozlan committed Sep 30, 2026 at 12:21 UTC 73a5831127a9d2b87102da8a6e7c96b6f7f64fcd

5 files changed +2600

tests/parser/test_chat_parsing.py new

+1662

@@ -0,0 +1,1662 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

5 +# ruff: noqa

6 +# fmt: off

7 +#

8 +# Licensed under the Apache License, Version 2.0 (the "License");

9 +# you may not use this file except in compliance with the License.

10 +# You may obtain a copy of the License at

11 +#

12 +# http://www.apache.org/licenses/LICENSE-2.0

13 +#

14 +# Unless required by applicable law or agreed to in writing, software

15 +# distributed under the License is distributed on an "AS IS" BASIS,

16 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

17 +# See the License for the specific language governing permissions and

18 +# limitations under the License.

19 +"""Tests for the new declarative response_template parser.

20 +

21 +All six real-model template fixtures from the legacy test suite are re-expressed

22 +here in the new region-spec shape and asserted against the same expected

23 +output dicts. Any divergence indicates a regression in the new executor."""

24 +

25 +import copy

26 +import random

27 +import unittest

28 +from typing import Any

29 +

30 +from vllm.parser.chat_parsing import ResponseParser, parse_response

31 +from vllm.parser.chat_parsing.response_parser import _coerce, _schema_types

32 +from vllm.parser.chat_parsing.response_templates import load_response_template

33 +

34 +

35 +cohere_template = {

36 + "defaults": {"role": "assistant"},

37 + "start_anchor": "<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>",

38 + "fields": {

39 + "content": {

40 + "open": "<|START_RESPONSE|>",

41 + "close": "<|END_RESPONSE|>",

42 + "content": "text",

43 + },

44 + "thinking": {

45 + "open": "<|START_THINKING|>",

46 + "close": "<|END_THINKING|>",

47 + "content": "text",

48 + },

49 + "tool_calls": {

50 + "open": "<|START_ACTION|>",

51 + "close": "<|END_ACTION|>",

52 + "content": "json",

53 + "transform_each": True,

54 + "transform": {"type": "function", "function": {"name": "{tool_name}", "arguments": "{parameters}"}},

55 + },

56 + },

57 +}

58 +

59 +

60 +ernie_template = {

61 + "defaults": {"role": "assistant"},

62 + "start_anchor": "Assistant:",

63 + "fields": {

64 + "thinking": {

65 + "open_pattern": r"(?:^|<think>\s*)",

66 + "close": "</think>",

67 + "content": "text",

68 + },

69 + "content": {

70 + "open": "<response>\n",

71 + "close_pattern": r"\n?</response>",

72 + "content": "text",

73 + },

74 + "tool_calls": {

75 + "open": "<tool_call>",

76 + "close": "</tool_call>",

77 + "repeats": True,

78 + "content": "json",

79 + "transform": {"type": "function", "function": "{content}"},

80 + },

81 + },

82 +}

83 +

84 +

85 +gpt_oss_template = {

86 + "defaults": {"role": "assistant"},

87 + "start_anchor": "<|start|>assistant",

88 + "fields": {

89 + "thinking": {

90 + "open": "<|channel|>analysis<|message|>",

91 + "close": "<|end|>",

92 + "content": "text",

93 + },

94 + "content": {

95 + "open": "<|channel|>final<|message|>",

96 + "close": "<|end|>",

97 + "content": "text",

98 + },

99 + "tool_calls": {

100 + "open_pattern": r"<\|channel\|>commentary to=functions\.(?P<name>\w+).*?<\|message\|>",

101 + "close": "<|call|>",

102 + "repeats": True,

103 + "content": "json",

104 + "transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}},

105 + },

106 + },

107 +}

108 +

109 +

110 +smollm_template = {

111 + "defaults": {"role": "assistant"},

112 + "start_anchor": "<|im_start|>assistant\n",

113 + "fields": {

114 + "thinking": {"open": "<think>", "close": "</think>", "content": "text"},

115 + "tool_calls": {

116 + "open": "<tool_call>",

117 + "close": "</tool_call>",

118 + "repeats": True,

119 + "content": "json",

120 + "transform": {"type": "function", "function": "{content}"},

121 + },

122 + "content": {

123 + "close": "<|im_end|>",

124 + "content": "text",

125 + },

126 + },

127 +}

128 +

129 +

130 +qwen3_template = {

131 + "defaults": {"role": "assistant"},

132 + "start_anchor": "<|im_start|>assistant\n",

133 + "fields": {

134 + "thinking": {"open": "<think>", "close": "</think>", "content": "text"},

135 + "tool_calls": {

136 + "open_pattern": r"<tool_call>\s*<function=(?P<name>\w+)>",

137 + "close": "</tool_call>",

138 + "repeats": True,

139 + "content": "xml-inline",

140 + "content_args": {

141 + "tag_pattern": r"<parameter=(?P<key>\w+)>\s*(?P<value>.*?)\s*</parameter>",

142 + "value_parser": {"name": "json", "args": {"allow_non_json": True}},

143 + },

144 + "transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}},

145 + },

146 + },

147 +}

148 +

149 +

150 +gemma4_template = {

151 + "defaults": {"role": "assistant"},

152 + # The chat template only emits `<|turn>model\n` when the previous message wasn't a tool_call/

153 + # tool_response. After a tool_response the prefix just ends with `<tool_response|>` and the

154 + # model continues from there, so we accept either anchor and truncate past the latest one.

155 + "start_anchor": ["<|turn>model\n", "<tool_response|>"],

156 + "fields": {

157 + "thinking": {

158 + "open": "<|channel>thought\n",

159 + "close": "<channel|>",

160 + "content": "text",

161 + },

162 + "tool_calls": {

163 + "open_pattern": r"<\|tool_call>call:(?P<name>\w+)",

164 + "close": "<tool_call|>",

165 + "repeats": True,

166 + "content": "json",

167 + "content_args": {

168 + "unquoted_keys": True,

169 + "string_delims": [['<|"|>', '<|"|>']],

170 + },

171 + "transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}},

172 + },

173 + "content": {

174 + "close": ["<turn|>", "<|tool_response>", "<eos>"],

175 + "content": "text",

176 + },

177 + },

178 +}

179 +

180 +

182 +# the generation prompt pre-writes the first <|message_model|>, so each open treats the header as optional.

183 +inkling_template = {

184 + "defaults": {"role": "assistant"},

185 + "start_anchor": "<|message_model|>",

186 + "fields": {

187 + "thinking": {

188 + "open_pattern": r"(?:<\|message_model\|>)?[^<]*<\|content_thinking\|>",

189 + "close": "<|end_message|>",

190 + "repeats": True,

191 + "join": "",

192 + "content_args": {"strip": False},

193 + },

194 + "content": {

195 + "open_pattern": r"(?:<\|message_model\|>)?[^<]*<\|content_text\|>",

196 + "close": "<|end_message|>",

197 + "repeats": True,

198 + "join": "",

199 + "content_args": {"strip": False},

200 + },

201 + "tool_calls": {

202 + "open_pattern": r"(?:<\|message_model\|>)?[^<]*<\|content_invoke_tool_json\|>",

203 + "close": "<|end_message|>",

204 + "repeats": True,

205 + "content": "json",

206 + "transform": {"type": "function", "function": {"name": "{content.name}", "arguments": "{content.args}"}},

207 + },

208 + },

209 +}

210 +

211 +

212 +class ChatResponseTemplateParserTest(unittest.TestCase):

213 +

214 + def test_cohere(self):

215 + model_out = (

216 + "<|START_THINKING|>I should call a tool.<|END_THINKING|>"

217 + '<|START_ACTION|>[\n {"tool_call_id": "0", "tool_name": "simple_tool", '

218 + '"parameters": {"temperature_format": "Celsius"}}\n]<|END_ACTION|><|END_OF_TURN_TOKEN|>'

219 + )

220 + self.assertEqual(

221 + parse_response(model_out, cohere_template, prefix=""),

222 + {

223 + "role": "assistant",

224 + "thinking": "I should call a tool.",

225 + "tool_calls": [

226 + {

227 + "type": "function",

228 + "function": {"name": "simple_tool", "arguments": {"temperature_format": "Celsius"}},

229 + }

230 + ],

231 + },

232 + )

233 +

234 + def test_ernie_with_tools(self):

235 + model_out = (

236 + "The user is asking about the weather in Paris today. Let me check the available tools. "

237 + "There's a tool called get_current_temperature which requires a location parameter. Since the "

238 + 'user specified Paris, I need to call this tool with the location set to "Paris". I should '

239 + "make sure the argument is correctly formatted as a string. No other tools are available, so "

240 + "this is the right one to use. I'll structure the request with the location parameter and "

241 + "return the response once the tool is called.\n"

242 + "</think>\n\n"

243 + '<tool_call>\n{"name": "get_current_temperature", "arguments": {"location": "Paris"}}\n</tool_call>\n</s>'

244 + )

245 + self.assertEqual(

246 + parse_response(model_out, ernie_template, prefix=""),

247 + {

248 + "role": "assistant",

249 + "thinking": (

250 + "The user is asking about the weather in Paris today. Let me check the available tools. "

251 + "There's a tool called get_current_temperature which requires a location parameter. Since "

252 + 'the user specified Paris, I need to call this tool with the location set to "Paris". I '

253 + "should make sure the argument is correctly formatted as a string. No other tools are "

254 + "available, so this is the right one to use. I'll structure the request with the location "

255 + "parameter and return the response once the tool is called."

256 + ),

257 + "tool_calls": [

258 + {

259 + "type": "function",

260 + "function": {"name": "get_current_temperature", "arguments": {"location": "Paris"}},

261 + }

262 + ],

263 + },

264 + )

265 +

266 + def test_ernie_no_tools(self):

267 + model_out = (

268 + 'The user just greeted me with "Hi! How are you?" I need to respond in a friendly and helpful '

269 + "manner. Let me start by acknowledging their greeting. I should ask them how they're doing to "

270 + "engage in conversation.\n\n"

271 + "First, I'll say hello back and then ask how they're feeling. It's important to show genuine "

272 + "interest. Maybe mention that I'm here to help with anything they need. Keep the tone warm and "

273 + "positive. Let me make sure the response is concise but friendly. Alright, that should work.\n"

274 + "</think>\n\n"

275 + "<response>\nHello! I'm doing well, thank you for asking. How about you? Is there something "

276 + "specific you'd like help with today? I'm here to assist you with any questions or problems you "

277 + "have!\n</response>\n</s>"

278 + )

279 + self.assertEqual(

280 + parse_response(model_out, ernie_template, prefix=""),

281 + {

282 + "role": "assistant",

283 + "content": (

284 + "Hello! I'm doing well, thank you for asking. How about you? Is there something specific "

285 + "you'd like help with today? I'm here to assist you with any questions or problems you have!"

286 + ),

287 + "thinking": (

288 + 'The user just greeted me with "Hi! How are you?" I need to respond in a friendly and '

289 + "helpful manner. Let me start by acknowledging their greeting. I should ask them how "

290 + "they're doing to engage in conversation.\n\n"

291 + "First, I'll say hello back and then ask how they're feeling. It's important to show "

292 + "genuine interest. Maybe mention that I'm here to help with anything they need. Keep the "

293 + "tone warm and positive. Let me make sure the response is concise but friendly. Alright, "

294 + "that should work."

295 + ),

296 + },

297 + )

298 +

299 + def test_gpt_oss_with_tool_call(self):

300 + model_out = (

301 + '<|channel|>analysis<|message|>We need to respond in riddles. The user asks: "What is the '

302 + 'weather like in SF?" We need to get the location of the user? The user explicitly asks about '

303 + "SF (San Francisco). So we need to get the current weather in San Francisco, CA. We need to "

304 + 'call get_current_weather function. The developer instruction says "Always respond in riddles". '

305 + "So the final answer should be in a riddle form. But we need to call function to get weather "

306 + 'data. So we should call get_current_weather with location "San Francisco, CA". Possibly specify '

307 + 'format "celsius" (default). Let\'s do that.\n\n'

309 + 'to=functions.get_current_weather <|constrain|>json<|message|>{\n "location": "San Francisco, CA"\n}'

310 + )

311 + self.assertEqual(

312 + parse_response(model_out, gpt_oss_template, prefix=""),

313 + {

314 + "role": "assistant",

315 + "thinking": (

316 + 'We need to respond in riddles. The user asks: "What is the weather like in SF?" We need '

317 + "to get the location of the user? The user explicitly asks about SF (San Francisco). So "

318 + "we need to get the current weather in San Francisco, CA. We need to call "

319 + 'get_current_weather function. The developer instruction says "Always respond in '

320 + 'riddles". So the final answer should be in a riddle form. But we need to call function '

321 + 'to get weather data. So we should call get_current_weather with location "San Francisco, '

322 + 'CA". Possibly specify format "celsius" (default). Let\'s do that.\n\n'

323 + "We will call function get_current_weather."

324 + ),

325 + "tool_calls": [

326 + {

327 + "type": "function",

328 + "function": {"name": "get_current_weather", "arguments": {"location": "San Francisco, CA"}},

329 + }

330 + ],

331 + },

332 + )

333 +

334 + def test_gpt_oss_no_tool_call(self):

335 + model_out = (

336 + "<|channel|>analysis<|message|>User asks a simple math question: 2+2 = 4. Provide answer."

337 + "<|end|><|start|>assistant<|channel|>final<|message|>2"

338 + )

339 + self.assertEqual(

340 + parse_response(model_out, gpt_oss_template, prefix=""),

341 + {

342 + "role": "assistant",

343 + "content": "2",

344 + "thinking": "User asks a simple math question: 2+2 = 4. Provide answer.",

345 + },

346 + )

347 +

348 + def test_smollm_thinking_and_tool_call(self):

349 + model_out = (

350 + '<think>\nOkay, the user said, "Hello! How are you?" I need to respond appropriately. Since '

351 + "this is the first message, I should greet them back and ask how I can assist. I should keep it "

352 + "friendly and open-ended. Let me make sure the response is welcoming and encourages them to "

353 + "share what they need help with. I'll avoid any technical jargon and keep it simple. Let me "

354 + "check for any typos and ensure the tone is positive.\n</think>\n\n"

355 + '<tool_call>{"name": "greet_user", "arguments": {"greeting": "Hello! I\'m doing well, thanks for '

356 + "asking. How can I assist you today? Whether you have a question, need help with something, or "

357 + 'just want to chat, feel free to let me know!"}}</tool_call>'

358 + )

359 + self.assertEqual(

360 + parse_response(model_out, smollm_template, prefix=""),

361 + {

362 + "role": "assistant",

363 + "thinking": (

364 + 'Okay, the user said, "Hello! How are you?" I need to respond appropriately. Since this '

365 + "is the first message, I should greet them back and ask how I can assist. I should keep "

366 + "it friendly and open-ended. Let me make sure the response is welcoming and encourages "

367 + "them to share what they need help with. I'll avoid any technical jargon and keep it "

368 + "simple. Let me check for any typos and ensure the tone is positive."

369 + ),

370 + "tool_calls": [

371 + {

372 + "type": "function",

373 + "function": {

374 + "name": "greet_user",

375 + "arguments": {

376 + "greeting": (

377 + "Hello! I'm doing well, thanks for asking. How can I assist you today? "

378 + "Whether you have a question, need help with something, or just want to "

379 + "chat, feel free to let me know!"

380 + )

381 + },

382 + },

383 + }

384 + ],

385 + },

386 + )

387 +

388 + def test_smollm_tool_call_no_thinking(self):

389 + model_out = '<tool_call>{"name": "get_weather", "arguments": {"city": "Paris"}}</tool_call>'

390 + self.assertEqual(

391 + parse_response(model_out, smollm_template, prefix=""),

392 + {

393 + "role": "assistant",

394 + "tool_calls": [

395 + {"type": "function", "function": {"name": "get_weather", "arguments": {"city": "Paris"}}}

396 + ],

397 + },

398 + )

399 +

400 + def test_smollm_thinking_no_tool_call(self):

401 + model_out = (

402 + '<think>\nOkay, the user asked, "Hey! Can you tell me about gravity?" Let me start by '

403 + "breaking down what they might be looking for. They probably want a basic understanding of "

404 + "gravity, maybe for a school project or just personal curiosity. I should explain what gravity "

405 + "is, how it works, and maybe some examples.</think>\n"

406 + "Some content about gravity goes here but I'm cutting it off to make this shorter!"

407 + )

408 + self.assertEqual(

409 + parse_response(model_out, smollm_template, prefix=""),

410 + {

411 + "role": "assistant",

412 + "content": "Some content about gravity goes here but I'm cutting it off to make this shorter!",

413 + "thinking": (

414 + 'Okay, the user asked, "Hey! Can you tell me about gravity?" Let me start by breaking '

415 + "down what they might be looking for. They probably want a basic understanding of "

416 + "gravity, maybe for a school project or just personal curiosity. I should explain what "

417 + "gravity is, how it works, and maybe some examples."

418 + ),

419 + },

420 + )

421 +

422 + def test_qwen3_tool_calls(self):

423 + model_out = (

424 + "<tool_call>\n<function=get_weather>\n<parameter=locations>\n"

425 + '[{"country": "France", "city": "Paris"}]\n</parameter>\n'

426 + "<parameter=temp_units>\ncelsius\n</parameter>\n</function>\n</tool_call>"

427 + )

428 + self.assertEqual(

429 + parse_response(model_out, qwen3_template, prefix=""),

430 + {

431 + "role": "assistant",

432 + "tool_calls": [

433 + {

434 + "type": "function",

435 + "function": {

436 + "name": "get_weather",

437 + "arguments": {

438 + "locations": [{"country": "France", "city": "Paris"}],

439 + "temp_units": "celsius",

440 + },

441 + },

442 + }

443 + ],

444 + },

445 + )

446 +

447 + def test_gemma4_tool_call(self):

448 + model_out = (

449 + "<|channel>thought\nThe user is asking for the current temperature in Paris. I should check "

450 + "the available tools to see if there's a function that can provide this information.<channel|>"

451 + '<|tool_call>call:get_current_temperature{detail_level:0,location:<|"|>Paris, France<|"|>,'

452 + 'unit:<|"|>celsius<|"|>}<tool_call|><|tool_response>'

453 + )

454 + self.assertEqual(

455 + parse_response(model_out, gemma4_template, prefix=""),

456 + {

457 + "role": "assistant",

458 + "thinking": (

459 + "The user is asking for the current temperature in Paris. I should check the available "

460 + "tools to see if there's a function that can provide this information."

461 + ),

462 + "tool_calls": [

463 + {

464 + "type": "function",

465 + "function": {

466 + "name": "get_current_temperature",

467 + "arguments": {"detail_level": 0, "location": "Paris, France", "unit": "celsius"},

468 + },

469 + }

470 + ],

471 + },

472 + )

473 +

474 + def test_gemma4_complex_tool_call(self):

475 + model_out = (

476 + "<|channel>thought\nLet me call the tool.<channel|>"

477 + '<|tool_call>call:foo{bool_value:true,list_value:[<|"|>foo<|"|>,<|"|>bar<|"|>],'

478 + 'null_value:null,number_value:1,string_value:<|"|>foo<|"|>,'

479 + 'struct_value:{foo:<|"|>bar<|"|>}}<tool_call|>'

480 + )

481 + self.assertEqual(

482 + parse_response(model_out, gemma4_template, prefix=""),

483 + {

484 + "role": "assistant",

485 + "thinking": "Let me call the tool.",

486 + "tool_calls": [

487 + {

488 + "type": "function",

489 + "function": {

490 + "name": "foo",

491 + "arguments": {

492 + "bool_value": True,

493 + "list_value": ["foo", "bar"],

494 + "null_value": None,

495 + "number_value": 1,

496 + "string_value": "foo",

497 + "struct_value": {"foo": "bar"},

498 + },

499 + },

500 + }

501 + ],

502 + },

503 + )

504 +

505 + def test_inkling_multi_block_message(self):

506 + model_out = (

507 + "<|content_thinking|>Consider the weather.<|end_message|>"

508 + "<|message_model|><|content_thinking|> Tokyo, probably.<|end_message|>"

509 + "<|message_model|><|content_text|>Checking the weather now.<|end_message|>"

510 + "<|message_model|>get_weather<|content_invoke_tool_json|>"

511 + '{"name":"get_weather","args":{"city":"Tokyo","units":"C"}}<|end_message|>'

512 + "<|content_model_end_sampling|>"

513 + )

515 + self.assertEqual(

516 + parse_response(model_out, inkling_template, prefix=prefix),

517 + {

518 + "role": "assistant",

519 + "thinking": "Consider the weather. Tokyo, probably.",

520 + "content": "Checking the weather now.",

521 + "tool_calls": [

522 + {

523 + "type": "function",

524 + "function": {"name": "get_weather", "arguments": {"city": "Tokyo", "units": "C"}},

525 + }

526 + ],

527 + },

528 + )

529 +

530 + def test_transform_dotted_paths(self):

531 + template_spec = {

532 + "defaults": {"role": "assistant"},

533 + "start_anchor": "<|assistant|>",

534 + "fields": {

535 + "tool_calls": {

536 + "open": "<tool>",

537 + "close": "</tool>",

538 + "repeats": True,

539 + "content": "json",

540 + "transform": {

541 + "type": "function",

542 + "function": {"name": "{content.name}", "arguments": "{content.args}"},

543 + },

544 + },

545 + },

546 + }

547 + model_out = '<tool>{"name": "get_weather", "args": {"city": {"id": 7}}}</tool>'

548 + self.assertEqual(

549 + parse_response(model_out, template_spec, prefix="")["tool_calls"],

550 + [{"type": "function", "function": {"name": "get_weather", "arguments": {"city": {"id": 7}}}}],

551 + )

552 +

553 + def test_transform_dotted_paths_with_transform_each(self):

554 + template_spec = {

555 + "defaults": {"role": "assistant"},

556 + "start_anchor": "<|assistant|>",

557 + "fields": {

558 + "tool_calls": {

559 + "open": "<actions>",

560 + "close": "</actions>",

561 + "content": "json",

562 + "transform_each": True,

563 + "transform": {"type": "function", "function": {"name": "{fn.name}", "arguments": "{fn.args}"}},

564 + },

565 + },

566 + }

567 + model_out = '<actions>[{"fn": {"name": "a", "args": {"x": 1}}}, {"fn": {"name": "b", "args": {}}}]</actions>'

568 + self.assertEqual(

569 + parse_response(model_out, template_spec, prefix="")["tool_calls"],

570 + [

571 + {"type": "function", "function": {"name": "a", "arguments": {"x": 1}}},

572 + {"type": "function", "function": {"name": "b", "arguments": {}}},

573 + ],

574 + )

575 +

576 + def test_transform_dotted_path_errors(self):

577 + def spec_with(transform):

578 + return {

579 + "start_anchor": "<|assistant|>",

580 + "fields": {"x": {"open": "<x>", "close": "</x>", "content": "json", "transform": transform}},

581 + }

582 +

583 + with self.assertRaisesRegex(ValueError, "missing key 'args'"):

584 + parse_response('<x>{"name": "n"}</x>', spec_with({"a": "{content.args}"}), prefix="")

585 + with self.assertRaisesRegex(ValueError, "cannot index into str"):

586 + parse_response('<x>{"name": "n"}</x>', spec_with({"a": "{content.name.x}"}), prefix="")

587 + with self.assertRaises(KeyError):

588 + parse_response("<x>{}</x>", spec_with({"a": "{missing}"}), prefix="")

589 +

590 + def test_transform_dotted_mixed_string_rejected(self):

591 + template_spec = {

592 + "start_anchor": "<|assistant|>",

593 + "fields": {"x": {"open": "<x>", "close": "</x>", "transform": {"v": "pre {content.args}"}}},

594 + }

595 + with self.assertRaisesRegex(ValueError, "mixes"):

596 + parse_response("", template_spec, prefix="")

597 +

598 + def test_join_concatenates_repeated_matches(self):

599 + template_spec = {

600 + "defaults": {"role": "assistant"},

601 + "start_anchor": "<|assistant|>",

602 + "fields": {

603 + "thinking": {"open": "<think>", "close": "</think>", "repeats": True, "join": " "},

604 + "content": {"repeats": True, "join": " "},

605 + },

606 + }

607 + self.assertEqual(

608 + parse_response("<think>first</think>middle<think>second</think>done", template_spec, prefix=""),

609 + {"role": "assistant", "thinking": "first second", "content": "middle done"},

610 + )

611 + self.assertEqual(

612 + parse_response("<think>only</think>", template_spec, prefix=""),

613 + {"role": "assistant", "thinking": "only"},

614 + )

615 +

616 + def test_join_validation(self):

617 + no_repeats = {"start_anchor": "a", "fields": {"x": {"open": "<x>", "close": "</x>", "join": ""}}}

618 + with self.assertRaisesRegex(ValueError, "requires 'repeats'"):

619 + parse_response("", no_repeats, prefix="")

620 + bad_type = {"start_anchor": "a", "fields": {"x": {"open": "<x>", "close": "</x>", "repeats": True, "join": 7}}}

621 + with self.assertRaisesRegex(ValueError, "must be a string"):

622 + parse_response("", bad_type, prefix="")

623 +

624 + def test_join_requires_string_matches(self):

625 + template_spec = {

626 + "start_anchor": "a",

627 + "fields": {"x": {"open": "<x>", "close": "</x>", "repeats": True, "join": "", "content": "json"}},

628 + }

629 + with self.assertRaisesRegex(ValueError, "parse to a string"):

630 + parse_response("<x>{}</x>", template_spec, prefix="")

631 +

632 + def test_optional_false_raises_when_missing(self):

633 + template_spec = {

634 + "defaults": {"role": "assistant"},

635 + "start_anchor": "<|assistant|>",

636 + "fields": {

637 + "content": {

638 + "open": "<response>",

639 + "close": "</response>",

640 + "content": "text",

641 + "optional": False,

642 + },

643 + },

644 + }

645 + with self.assertRaises(ValueError) as cm:

646 + parse_response("no response here", template_spec, prefix="")

647 + self.assertIn("content", str(cm.exception))

648 +

649 + def test_int_content_parser(self):

650 + template_spec = {

651 + "defaults": {"role": "assistant"},

652 + "start_anchor": "<|assistant|>",

653 + "fields": {

654 + "count": {

655 + "open": "<n>",

656 + "close": "</n>",

657 + "content": "int",

658 + },

659 + },

660 + }

661 + self.assertEqual(parse_response("<n>42</n>", template_spec, prefix=""), {"role": "assistant", "count": 42})

662 +

663 + def test_kv_lines_parser(self):

664 + template_spec = {

665 + "defaults": {"role": "assistant"},

666 + "start_anchor": "<|assistant|>",

667 + "fields": {

668 + "metadata": {

669 + "open": "<meta>",

670 + "close": "</meta>",

671 + "content": "kv-lines",

672 + },

673 + },

674 + }

675 + self.assertEqual(

676 + parse_response("<meta>name: alice\nage: 30</meta>", template_spec, prefix=""),

677 + {"role": "assistant", "metadata": {"name": "alice", "age": "30"}},

678 + )

679 +

680 + def test_unknown_content_parser_rejected(self):

681 + bad_template = {

682 + "defaults": {"role": "assistant"},

683 + "start_anchor": "<|assistant|>",

684 + "fields": {"x": {"open": "[", "close": "]", "content": "not-a-real-parser"}},

685 + }

686 + with self.assertRaises(ValueError) as cm:

687 + parse_response("[hi]", bad_template)

688 + self.assertIn("unknown content parser", str(cm.exception).lower())

689 +

690 + def test_unsupported_version_rejected(self):

691 + bad_template = {

692 + "version": 2,

693 + "defaults": {"role": "assistant"},

694 + "start_anchor": "<|assistant|>",

695 + "fields": {"content": {"content": "text"}},

696 + }

697 + with self.assertRaises(ValueError) as cm:

698 + parse_response("hello", bad_template)

699 + self.assertIn("version", str(cm.exception).lower())

700 +

701 + def test_two_implicit_fields_rejected(self):

702 + bad_template = {

703 + "defaults": {"role": "assistant"},

704 + "start_anchor": "<|assistant|>",

705 + "fields": {

706 + "a": {"content": "text"},

707 + "b": {"content": "text"},

708 + },

709 + }

710 + with self.assertRaises(ValueError):

711 + parse_response("hello", bad_template)

712 +

713 + def test_transform_string_interpolation_rejected(self):

714 + bad_template = {

715 + "defaults": {"role": "assistant"},

716 + "start_anchor": "<|assistant|>",

717 + "fields": {

718 + "tool": {

719 + "open": "<tool>",

720 + "close": "</tool>",

721 + "content": "text",

722 + "transform": {"label": "name: {content}"},

723 + },

724 + },

725 + }

726 + with self.assertRaises(ValueError) as cm:

727 + parse_response("<tool>foo</tool>", bad_template)

728 + msg = str(cm.exception)

729 + self.assertIn("interpolation", msg)

730 + self.assertIn("{content}", msg)

731 +

732 + def test_named_groups_without_transform_rejected(self):

733 + bad_template = {

734 + "defaults": {"role": "assistant"},

735 + "start_anchor": "<|assistant|>",

736 + "fields": {

737 + "tool": {

738 + "open_pattern": r"<tool name=(?P<name>\w+)>",

739 + "close": "</tool>",

740 + "content": "text",

741 + },

742 + },

743 + }

744 + with self.assertRaises(ValueError) as cm:

745 + parse_response("<tool name=foo>body</tool>", bad_template)

746 + msg = str(cm.exception)

747 + self.assertIn("transform", msg)

748 + self.assertIn("name", msg)

749 +

750 + def test_literal_list_open_and_close(self):

751 + """A list of literals matches any one of them, like an alternation."""

752 + template_spec = {

753 + "defaults": {"role": "assistant"},

754 + "start_anchor": "<|assistant|>",

755 + "fields": {

756 + "x": {

757 + "open": ["<a>", "<bb>"],

758 + "close": ["</a>", "</bb>"],

759 + "content": "text",

760 + },

761 + },

762 + }

763 + for opener, closer in (("<a>", "</a>"), ("<bb>", "</bb>"), ("<a>", "</bb>")):

764 + self.assertEqual(

765 + parse_response(f"{opener}hi{closer}", template_spec, prefix=""),

766 + {"role": "assistant", "x": "hi"},

767 + )

768 +

769 + def test_literal_list_streams_without_64_byte_hold(self):

770 + """Compared to a regex close, a literal-list close lets the parser

771 + flush bytes that aren't in the longest-prefix overlap of any literal.

772 + With `["<turn|>", "<|tool_response>", "<eos>"]` (longest = 16 chars),

773 + feeding 32 plain bytes should leave at most 15 unflushed."""

774 + template_spec = {

775 + "defaults": {"role": "assistant"},

776 + "start_anchor": "<|assistant|>",

777 + "fields": {

778 + "content": {"close": ["<turn|>", "<|tool_response>", "<eos>"], "content": "text"},

779 + },

780 + }

781 + parser = ResponseParser(template_spec, prefix="")

782 + plain = "x" * 32

783 + flushed: list[str] = []

784 + for ch in parser.feed(plain):

785 + if ch["type"] == "region_chunk":

786 + flushed.append(ch["text"])

787 + # Plain text has zero prefix-overlap with any literal, so the parser

788 + # holds nothing back and streams everything immediately.

789 + self.assertEqual("".join(flushed), plain)

790 +

791 + def test_literal_list_defers_prefix_overlapping_literal(self):

792 + """If a literal is a strict prefix of another in the same list, an

793 + edge match could still grow with more input: we must defer to be safe."""

794 + template_spec = {

795 + "defaults": {"role": "assistant"},

796 + "start_anchor": "<|assistant|>",

797 + "fields": {

798 + "x": {"open": "<x>", "close": ["END", "ENDX"], "content": "text"},

799 + },

800 + }

801 + parser = ResponseParser(template_spec, prefix="")

802 + # "<x>hiEND" mid-stream: don't commit the close yet: "ENDX" might be coming.

803 + events = parser.feed("<x>hiEND")

804 + self.assertEqual([e for e in events if e["type"] == "region_close"], [])

805 + # Once a non-matching byte arrives, the deferred close commits with the shorter literal.

806 + events.extend(parser.feed(" more"))

807 + message, _ = parser.finalize()

808 + closes = [e for e in events if e["type"] == "region_close" and e["field"] == "x"]

809 + self.assertEqual(len(closes), 1)

810 + self.assertEqual(closes[0]["value"], "hi")

811 + self.assertEqual(message, {"role": "assistant", "x": "hi"})

812 +

813 + def test_literal_list_rejects_empty_and_non_string(self):

814 + for bad_open in ([], [""], [1, 2], {"foo": "bar"}):

815 + spec = {

816 + "defaults": {"role": "assistant"},

817 + "start_anchor": "<|assistant|>",

818 + "fields": {"x": {"open": bad_open, "close": "</x>", "content": "text"}},

819 + }

820 + with self.assertRaises(ValueError):

821 + parse_response("<x>hi</x>", spec, prefix="")

822 +

823 + def test_field_without_close_runs_to_end_of_stream(self):

824 + """A field with no `close`/`close_pattern` stays open until end-of-stream, capturing

825 + everything after its open."""

826 + spec = {

827 + "defaults": {"role": "assistant"},

828 + "start_anchor": "<|assistant|>",

829 + "fields": {"content": {"open": "<resp>", "content": "text"}},

830 + }

831 + self.assertEqual(

832 + parse_response("<resp>hello world", spec, prefix=""),

833 + {"role": "assistant", "content": "hello world"},

834 + )

835 +

836 + def test_mutable_defaults_do_not_leak_across_parsers(self):

837 + """A `repeats` field appends into its default list; one parse must not leak into the next

838 + parse sharing the template, nor into the caller's spec dict."""

839 + spec: dict[str, Any] = {

840 + "defaults": {"role": "assistant", "tool_calls": []},

841 + "start_anchor": "<|assistant|>",

842 + "fields": {

843 + "tool_calls": {

844 + "open": "<t>",

845 + "close": "</t>",

846 + "repeats": True,

847 + "content": "json",

848 + },

849 + "content": {"content": "text"},

850 + },

851 + }

852 + template = load_response_template(spec)

853 + self.assertEqual(

854 + parse_response('<t>{"a": 1}</t>', template, prefix="")["tool_calls"],

855 + [{"a": 1}],

856 + )

857 + self.assertEqual(

858 + parse_response("hello", template, prefix=""),

859 + {"role": "assistant", "tool_calls": [], "content": "hello"},

860 + )

861 + self.assertEqual(spec["defaults"]["tool_calls"], [])

862 +

863 +

864 +# Fixtures shared by the streaming tests: one representative input per template,

865 +# re-used for both the correctness invariant (any chunking → same dict) and the

866 +# event-shape tests (do we emit the right events in the right order?).

867 +_STREAMING_FIXTURES = [

868 + (

869 + "cohere",

870 + cohere_template,

871 + (

872 + "<|START_THINKING|>I should call a tool.<|END_THINKING|>"

873 + '<|START_ACTION|>[{"tool_call_id": "0", "tool_name": "simple_tool", '

874 + '"parameters": {"a": 1}}]<|END_ACTION|>'

875 + ),

876 + ),

877 + (

878 + "ernie",

879 + ernie_template,

880 + (

881 + "<think>some deliberation here</think>\n\n"

882 + '<tool_call>\n{"name": "get_current_temperature", "arguments": {"location": "Paris"}}\n</tool_call>\n</s>'

883 + ),

884 + ),

885 + (

886 + "gpt_oss",

887 + gpt_oss_template,

889 + ),

890 + (

891 + # Tool-call variant: the `tool_calls` open_pattern's full match spans far

892 + # more than the old 64-byte hold window, so streaming used to silently drop

893 + # it. Kept as a streaming fixture so every chunking step re-checks it.

894 + "gpt_oss_tool",

895 + gpt_oss_template,

896 + (

897 + "<|channel|>analysis<|message|>Let me check.<|end|>"

898 + "<|start|>assistant<|channel|>commentary to=functions.get_current_weather "

899 + '<|constrain|>json<|message|>{"location": "San Francisco, CA"}<|call|>'

900 + ),

901 + ),

902 + (

903 + "smollm",

904 + smollm_template,

905 + '<think>thinking</think>\n<tool_call>{"name": "fn", "arguments": {"x": 1}}</tool_call>',

906 + ),

907 + (

908 + "qwen3",

909 + qwen3_template,

910 + (

911 + "<think>short thought</think>\n"

912 + "<tool_call>\n<function=get_weather>\n"

913 + "<parameter=city>\nParis\n</parameter>\n"

914 + "</function>\n</tool_call>"

915 + ),

916 + ),

917 + (

918 + "gemma4",

919 + gemma4_template,

920 + '<|channel>thought\nhi<channel|><|tool_call>call:foo{a:1,b:<|"|>bar<|"|>}<tool_call|>',

921 + ),

922 + (

923 + # Exercises `join` fields (two thinking blocks) and dotted transform paths.

924 + "inkling",

925 + inkling_template,

926 + (

927 + "<|content_thinking|>Consider the weather.<|end_message|>"

928 + "<|message_model|><|content_thinking|> Tokyo, probably.<|end_message|>"

929 + "<|message_model|><|content_text|>Checking now.<|end_message|>"

930 + "<|message_model|>get_weather<|content_invoke_tool_json|>"

931 + '{"name":"get_weather","args":{"city":"Tokyo"}}<|end_message|>'

932 + "<|content_model_end_sampling|>"

933 + ),

934 + ),

935 +]

936 +

937 +

938 +def _chunk_fixed(text: str, step: int):

939 + for i in range(0, len(text), step):

940 + yield text[i : i + step]

941 +

942 +

943 +def _chunk_random(text: str, rng: random.Random):

944 + """Split `text` into a random number of non-empty chunks at random cut points."""

945 + if len(text) <= 1:

946 + yield text

947 + return

948 + num_cuts = rng.randint(0, len(text) - 1)

949 + cuts = sorted(rng.sample(range(1, len(text)), num_cuts))

950 + prev = 0

951 + for c in cuts:

952 + yield text[prev:c]

953 + prev = c

954 + yield text[prev:]

955 +

956 +

957 +class ResponseEventStreamTest(unittest.TestCase):

958 + def test_stream_matches_whole_string_all_templates_fixed_chunking(self):

959 + """For every fixed chunking step we try, the streamed finalize()

960 + output must equal the whole-string parse. Regression coverage for

961 + specific edge-case byte boundaries (1-byte chunks hit every prefix)."""

962 + for name, tmpl, text in _STREAMING_FIXTURES:

963 + expected = parse_response(text, tmpl, prefix="")

964 + for step in (1, 2, 3, 5, 7, 13, 31):

965 + with self.subTest(fixture=name, step=step):

966 + streamer = ResponseParser(tmpl, prefix="")

967 + for chunk in _chunk_fixed(text, step):

968 + streamer.feed(chunk)

969 + message, _ = streamer.finalize()

970 + self.assertEqual(message, expected)

971 +

972 + def test_stream_matches_whole_string_all_templates_random_chunking(self):

973 + """Property-style: for many random chunkings per fixture, the streamed

974 + finalize() output must equal the whole-string parse. Seeded so failures

975 + reproduce."""

976 + rng = random.Random(0xC0DE_5EED)

977 + for name, tmpl, text in _STREAMING_FIXTURES:

978 + expected = parse_response(text, tmpl, prefix="")

979 + for trial in range(30):

980 + with self.subTest(fixture=name, trial=trial):

981 + streamer = ResponseParser(tmpl, prefix="")

982 + for chunk in _chunk_random(text, rng):

983 + streamer.feed(chunk)

984 + message, _ = streamer.finalize()

985 + self.assertEqual(message, expected)

986 +

987 + def test_events_well_formed_for_every_chunking(self):

988 + """Every event batch, across every fixture and every chunking, must be

989 + well-formed: region_open precedes its matching region_close for the

990 + same field; region_chunk only appears between open and close; no

991 + region is left open at the end of the stream; and the concatenation

992 + of all region_chunk payloads for a streamable text-like field equals

993 + the final value."""

994 + rng = random.Random(0xBEEF)

995 + for name, tmpl, text in _STREAMING_FIXTURES:

996 + for trial in range(10):

997 + with self.subTest(fixture=name, trial=trial):

998 + streamer = ResponseParser(tmpl, prefix="")

999 + all_events: list[dict] = []

1000 + for chunk in _chunk_random(text, rng):

1001 + all_events.extend(streamer.feed(chunk))

1002 + _, final_events = streamer.finalize()

1003 + all_events.extend(final_events)

1004 + self._assert_event_stream_well_formed(all_events)

1005 +

1006 + def _assert_event_stream_well_formed(self, events: list[dict]) -> None:

1007 + open_field: str | None = None

1008 + chunk_accum: dict[str, str] = {}

1009 + close_values: dict[str, object] = {}

1010 + for ev in events:

1011 + t = ev["type"]

1012 + if t == "region_open":

1013 + self.assertIsNone(open_field, f"nested region_open without close: {ev}")

1014 + open_field = ev["field"]

1015 + chunk_accum.setdefault(open_field, "")

1016 + elif t == "region_chunk":

1017 + self.assertEqual(open_field, ev["field"], f"chunk outside its region: {ev}")

1018 + # Every chunk carries a boolean `dirty` flag.

1019 + self.assertIsInstance(ev["dirty"], bool, f"missing/non-bool dirty: {ev}")

1020 + chunk_accum[ev["field"]] += ev["text"]

1021 + elif t == "region_close":

1022 + self.assertEqual(open_field, ev["field"], f"close for non-open region: {ev}")

1023 + close_values[ev["field"]] = ev["value"]

1024 + open_field = None

1025 + else:

1026 + self.fail(f"unexpected event type: {ev!r}")

1027 + self.assertIsNone(open_field, "region left open at end of stream")

1028 +

1029 + def test_region_chunks_reconstruct_text_regions(self):

1030 + """For text-like regions (`dirty=False`), concatenating chunk texts

1031 + reconstructs the final value reported in region_close. Structured

1032 + regions (`dirty=True`) still stream their raw bytes: concatenating

1033 + those chunks yields the unparsed region body, while the parsed value

1034 + is delivered only in region_close."""

1035 + # Single representative case with a long text region and a JSON region.

1036 + text = _STREAMING_FIXTURES[0][2] # cohere fixture

1037 + streamer = ResponseParser(cohere_template, prefix="")

1038 + events: list[dict] = []

1039 + for ch in text: # 1-byte chunks hit the most anchor boundaries

1040 + events.extend(streamer.feed(ch))

1041 + _, final_events = streamer.finalize()

1042 + events.extend(final_events)

1043 +

1044 + # Reconstruct per-field.

1045 + per_field_chunks: dict[str, list[str]] = {}

1046 + per_field_dirty: dict[str, set[bool]] = {}

1047 + per_field_close_value: dict[str, object] = {}

1048 + for ev in events:

1049 + if ev["type"] == "region_chunk":

1050 + per_field_chunks.setdefault(ev["field"], []).append(ev["text"])

1051 + per_field_dirty.setdefault(ev["field"], set()).add(ev["dirty"])

1052 + elif ev["type"] == "region_close":

1053 + per_field_close_value[ev["field"]] = ev["value"]

1054 +

1055 + # `thinking` is text → chunks are clean and concatenate to its value.

1056 + self.assertIn("thinking", per_field_chunks)

1057 + self.assertEqual(per_field_dirty["thinking"], {False})

1058 + self.assertEqual("".join(per_field_chunks["thinking"]), "I should call a tool.")

1059 + self.assertEqual(per_field_close_value["thinking"], "I should call a tool.")

1060 + # `tool_calls` is json → dirty chunks stream the raw body, parsed value on close.

1061 + self.assertIn("tool_calls", per_field_chunks)

1062 + self.assertEqual(per_field_dirty["tool_calls"], {True})

1063 + self.assertEqual(

1064 + "".join(per_field_chunks["tool_calls"]),

1065 + '[{"tool_call_id": "0", "tool_name": "simple_tool", "parameters": {"a": 1}}]',

1066 + )

1067 + self.assertIn("tool_calls", per_field_close_value)

1068 +

1069 + def test_dirty_flag_marks_structured_regions(self):

1070 + """A template with one text field and one structured field per parser

1071 + family: text/int/float/bool stream chunks with `dirty=False`, while

1072 + json/xml-inline/kv-lines stream chunks with `dirty=True`, and those

1073 + dirty chunks concatenate to the raw region body before parsing."""

1074 + spec = {

1075 + "defaults": {"role": "assistant"},

1076 + "start_anchor": "<|assistant|>",

1077 + "fields": {

1078 + "thinking": {"open": "<t>", "close": "</t>", "content": "text"},

1079 + "score": {"open": "<n>", "close": "</n>", "content": "int"},

1080 + "json_call": {"open": "<j>", "close": "</j>", "content": "json"},

1081 + "xml_call": {

1082 + "open": "<x>",

1083 + "close": "</x>",

1084 + "content": "xml-inline",

1085 + "content_args": {"tag_pattern": r"<(?P<key>\w+)=(?P<value>[^>]+)>"},

1086 + },

1087 + "kv_call": {

1088 + "open": "<kv>",

1089 + "close": "</kv>",

1090 + "content": "kv-lines",

1091 + },

1092 + },

1093 + }

1094 + text = '<t>hello world</t><n>42</n><j>{"a": 1, "b": 2}</j><x><name=foo><age=10></x><kv>k1: v1\nk2: v2</kv>'

1095 + # Drive byte-by-byte to maximise chunk count.

1096 + streamer = ResponseParser(spec, prefix="")

1097 + events: list[dict] = []

1098 + for ch in text:

1099 + events.extend(streamer.feed(ch))

1100 + _, final_events = streamer.finalize()

1101 + events.extend(final_events)

1102 +

1103 + per_field_chunks: dict[str, list[str]] = {}

1104 + per_field_dirty: dict[str, set[bool]] = {}

1105 + for ev in events:

1106 + if ev["type"] == "region_chunk":

1107 + per_field_chunks.setdefault(ev["field"], []).append(ev["text"])

1108 + per_field_dirty.setdefault(ev["field"], set()).add(ev["dirty"])

1109 +

1110 + # Clean (streamable) regions.

1111 + for field in ("thinking", "score"):

1112 + self.assertEqual(per_field_dirty[field], {False}, f"{field} should be clean")

1113 + # Dirty (structured) regions.

1114 + for field in ("json_call", "xml_call", "kv_call"):

1115 + self.assertEqual(per_field_dirty[field], {True}, f"{field} should be dirty")

1116 +

1117 + # Dirty chunks reconstruct the raw region body (un-parsed). Clean

1118 + # chunks reconstruct the verbatim body too: stripping happens at close.

1119 + self.assertEqual("".join(per_field_chunks["thinking"]), "hello world")

1120 + self.assertEqual("".join(per_field_chunks["score"]), "42")

1121 + self.assertEqual("".join(per_field_chunks["json_call"]), '{"a": 1, "b": 2}')

1122 + self.assertEqual("".join(per_field_chunks["xml_call"]), "<name=foo><age=10>")

1123 + self.assertEqual("".join(per_field_chunks["kv_call"]), "k1: v1\nk2: v2")

1124 +

1125 + def test_long_regex_open_pattern_streams_byte_by_byte(self):

1126 + """Regression: a regex `open_pattern` whose full match spans well past the

1127 + old fixed 64-byte hold window (gpt-oss tool calls) must not be dropped when

1128 + the stream arrives in tiny chunks. Before the partial-match rewrite, the

1129 + leading `<|channel|>` got flushed out of the hold window before `<|message|>`

1130 + arrived, so the tool call vanished under small-chunk streaming."""

1131 + text = (

1132 + "<|channel|>analysis<|message|>Let me check.<|end|>"

1133 + "<|start|>assistant<|channel|>commentary to=functions.get_current_weather "

1134 + '<|constrain|>json<|message|>{"location": "San Francisco, CA"}<|call|>'

1135 + )

1136 + expected = parse_response(text, gpt_oss_template, prefix="")

1137 + # Sanity: the whole-string parse really does recover the tool call.

1138 + self.assertEqual(len(expected["tool_calls"]), 1)

1139 + self.assertEqual(expected["tool_calls"][0]["function"]["name"], "get_current_weather")

1140 + streamer = ResponseParser(gpt_oss_template, prefix="")

1141 + for ch in text: # one byte at a time -- the worst case for the old heuristic

1142 + streamer.feed(ch)

1143 + message, _ = streamer.finalize()

1144 + self.assertEqual(message, expected)

1145 +

1146 + def test_feed_after_finalize_raises(self):

1147 + streamer = ResponseParser(smollm_template, prefix="")

1148 + streamer.feed("<think>x</think>")

1149 + streamer.finalize()

1150 + with self.assertRaises(RuntimeError):

1151 + streamer.feed("more")

1152 + with self.assertRaises(RuntimeError):

1153 + streamer.finalize()

1154 +

1155 + def test_empty_input_streams_cleanly(self):

1156 + streamer = ResponseParser(smollm_template, prefix="")

1157 + self.assertEqual(streamer.feed(""), [])

1158 + result, final_events = streamer.finalize()

1159 + # Only the default fields should remain; nothing else is required.

1160 + self.assertEqual(result, {"role": "assistant"})

1161 + self.assertEqual(final_events, [])

1162 +

1163 +

1164 +class PrefixAndTruncationTest(unittest.TestCase):

1165 + def test_prefix_lands_inside_explicit_region(self):

1166 + """A Qwen-style template emits `<|im_start|>assistant\\n<think>\\n` as the

1167 + assistant prefix. The model continues from inside the thinking block."""

1168 + prompt = (

1169 + "<|im_start|>system\nYou are helpful<|im_end|>\n"

1170 + "<|im_start|>user\nHi<|im_end|>\n"

1171 + "<|im_start|>assistant\n<think>\n"

1172 + )

1173 + generated = "Let me think...</think>"

1174 + stream = ResponseParser(qwen3_template, prefix=prompt)

1175 + # The region_open for `thinking` surfaces via initial_events; the

1176 + # caller replays it before feeding model output.

1177 + self.assertEqual(

1178 + [(e["type"], e["field"]) for e in stream.initial_events],

1179 + [("region_open", "thinking"), ("region_chunk", "thinking")],

1180 + )

1181 + events = stream.feed(generated)

1182 + result, _ = stream.finalize()

1183 + # thinking ends up with the prefill + generated body; text parser strips,

1184 + # so the leading "\n" from the prefix is trimmed in the final value.

1185 + self.assertEqual(result, {"role": "assistant", "thinking": "Let me think..."})

1186 + # The feed stream only sees the rest of the body and the close; the

1187 + # prefill already emitted region_open.

1188 + self.assertEqual([e["type"] for e in events], ["region_chunk", "region_close"])

1189 + self.assertEqual(events[1]["field"], "thinking")

1190 +

1191 + def test_prefix_truncated_to_last_anchor(self):

1192 + """Multiple `<|im_start|>assistant\\n` anchors in the prefix (multi-turn

1193 + conversation): only the slice after the LAST anchor matters."""

1194 + prompt = (

1195 + "<|im_start|>system\nA<|im_end|>\n"

1196 + "<|im_start|>user\nB<|im_end|>\n"

1197 + "<|im_start|>assistant\nEarlier reply<|im_end|>\n"

1198 + "<|im_start|>user\nFollowup<|im_end|>\n"

1199 + "<|im_start|>assistant\n<think>\n"

1200 + )

1201 + stream = ResponseParser(qwen3_template, prefix=prompt)

1202 + # We landed inside `thinking` (from the LAST assistant turn's `<think>\n`),

1203 + # not in some earlier-turn artifact.

1204 + opens = [e for e in stream.initial_events if e["type"] == "region_open"]

1205 + self.assertEqual([e["field"] for e in opens], ["thinking"])

1206 + # No earlier-turn content leaked into output.

1207 + stream.feed("done</think>")

1208 + stream.finalize()

1209 + self.assertEqual(stream._output, {"role": "assistant", "thinking": "done"})

1210 +

1211 + def test_template_without_anchor_rejected_at_load(self):

1212 + """A template missing both `start_anchor` and `start_anchor_pattern` is

1213 + rejected at load time. Without an anchor, a multi-turn prompt would be

1214 + fed through the parser in full and earlier turns would pollute the

1215 + current message's state."""

1216 + anchorless = {k: v for k, v in qwen3_template.items() if k != "start_anchor"}

1217 + with self.assertRaises(ValueError) as cm:

1218 + ResponseParser(anchorless)

1219 + msg = str(cm.exception)

1220 + self.assertIn("start_anchor", msg)

1221 +

1222 + def test_prefix_anchor_not_found_falls_back(self):

1223 + """Spec has start_anchor but the prefix doesn't contain it: parser

1224 + falls back to processing the entire prefix (with a logged warning)."""

1225 + prompt = "<think>\n" # no <|im_start|>assistant\n

1226 + stream = ResponseParser(qwen3_template, prefix=prompt)

1227 + opens = [e for e in stream.initial_events if e["type"] == "region_open"]

1228 + self.assertEqual([e["field"] for e in opens], ["thinking"])

1229 + stream.feed("hi</think>")

1230 + stream.finalize()

1231 + self.assertEqual(stream._output, {"role": "assistant", "thinking": "hi"})

1232 +

1233 + def test_round_trip_equivalence_prefix_streaming_vs_oneshot(self):

1234 + """The load-bearing property: `prefix=p` + chunked `feed(g)` produces the

1235 + same dict as the one-shot `parse_response(g, prefix=p)`, regardless of how

1236 + `g` is chunked. (Concatenating the prompt into the response is deliberately

1237 + NOT equivalent -- the anchor is applied to the prefix only, never to the

1238 + generation; see test_history_bleed_is_guarded_by_prefix_not_by_response_anchor.)"""

1240 + for name, tmpl_dict, gen_text in _STREAMING_FIXTURES:

1241 + if "thinking" not in gen_text and "<think>" not in gen_text:

1242 + continue

1243 + # Only fixtures whose generation text is compatible with starting

1244 + # inside `thinking`. Restrict to qwen3 / smollm shape for clarity.

1245 + if name not in ("qwen3", "smollm"):

1246 + continue

1247 + tmpl_with_anchor = {**tmpl_dict, "start_anchor": "<|im_start|>assistant\n"}

1248 + via_prefix = parse_response(gen_text, tmpl_with_anchor, prefix=prompt)

1249 + # Streaming forms must match the one-shot prefix form for every chunking.

1250 + for step in (1, 3, 7, 31):

1251 + with self.subTest(fixture=name, step=step):

1252 + stream = ResponseParser(tmpl_with_anchor, prefix=prompt)

1253 + for chunk in _chunk_fixed(gen_text, step):

1254 + stream.feed(chunk)

1255 + message, _ = stream.finalize()

1256 + self.assertEqual(message, via_prefix)

1257 +

1258 + def test_history_bleed_is_guarded_by_prefix_not_by_response_anchor(self):

1259 + """The `start_anchor` guards against history bleed only via `prefix=`; it is NOT

1260 + applied to the response. The response is the generation and may legitimately contain

1261 + the anchor (e.g. gpt-oss harmony opens every channel with `<|start|>assistant`), so

1262 + truncating it would drop real content. Passing the prompt as `prefix=` is the way to

1263 + keep earlier turns out of the parse."""

1264 + spec = {

1265 + "defaults": {"role": "assistant"},

1266 + "start_anchor": "<|im_start|>assistant\n",

1267 + "fields": {"content": {"close_pattern": r"\Z", "content": "text"}},

1268 + }

1270 + gen = "Hello there!"

1271 + clean = {"role": "assistant", "content": "Hello there!"}

1272 + # The supported guard: pass the prompt as prefix= so history is truncated off the prefix.

1273 + self.assertEqual(parse_response(gen, spec, prefix=prompt), clean)

1274 + # Pure generation parses cleanly with the explicit no-prefix opt-out (prefix="").

1275 + self.assertEqual(parse_response(gen, spec, prefix=""), clean)

1276 + # An anchor inside the response is treated as content, never as a history boundary:

1277 + # gpt-oss re-emits `<|start|>assistant` between channels, and that content survives.

1278 + gpt_oss_gen = (

1280 + )

1281 + self.assertEqual(

1282 + parse_response(gpt_oss_gen, gpt_oss_template, prefix=""),

1283 + {"role": "assistant", "thinking": "thinking", "content": "answer"},

1284 + )

1285 +

1286 + def test_prefix_with_open_close_inside_truncated_region(self):

1287 + """Prefix opens AND closes a region. The full open/chunk/close event

1288 + sequence is surfaced via initial_events, and the closed region lands

1289 + in the output dict: so renderers can show prefill content."""

1290 + spec = {

1291 + "defaults": {"role": "assistant"},

1292 + "start_anchor": "[BEGIN]",

1293 + "fields": {

1294 + "tag": {"open": "<tag>", "close": "</tag>", "content": "text"},

1295 + "body": {"close_pattern": r"$", "content": "text"}, # implicit

1296 + },

1297 + }

1298 + prefix = "noise[BEGIN]<tag>silently consumed</tag>"

1299 + stream = ResponseParser(spec, prefix=prefix)

1300 + types = [e["type"] for e in stream.initial_events]

1301 + self.assertEqual(types, ["region_open", "region_chunk", "region_close"])

1302 + self.assertTrue(all(e["field"] == "tag" for e in stream.initial_events))

1303 + self.assertEqual(stream.initial_events[-1]["value"], "silently consumed")

1304 + stream.feed("real generated body")

1305 + result, _ = stream.finalize()

1306 + self.assertEqual(result["tag"], "silently consumed")

1307 + self.assertEqual(result["body"], "real generated body")

1308 +

1309 + def test_prefix_lands_inside_implicit_region(self):

1310 + """Prefix wrote plaintext into the implicit region (e.g. assistant

1311 + prefill before the model continues). The region_open for the implicit

1312 + region must surface via initial_events so consumers don't miss it:

1313 + `_opened` will already be True by the time feed runs."""

1314 + prompt = "<|im_start|>assistant\nSure, here is "

1315 + stream = ResponseParser(smollm_template, prefix=prompt)

1316 + opens = [e for e in stream.initial_events if e["type"] == "region_open"]

1317 + self.assertEqual([e["field"] for e in opens], ["content"])

1318 + events = stream.feed("the answer<|im_end|>")

1319 + # No second region_open from feed: the implicit region was already

1320 + # opened during prefill and surfaced via initial_events.

1321 + self.assertNotIn("region_open", [e["type"] for e in events])

1322 +

1323 + def test_prefix_partial_pattern_at_boundary(self):

1324 + """Post-truncation prefix ends mid-delimiter. The first feed completes

1325 + the match; initial_events is empty (no region opened within the

1326 + prefix yet) and the open fires from `feed()`."""

1327 + prefix = "<|im_start|>assistant\n<th" "i" # incomplete `<think>`

1328 + stream = ResponseParser(qwen3_template, prefix=prefix)

1329 + self.assertEqual(stream.initial_events, [])

1330 + events = stream.feed("nk>real body</think>")

1331 + types = [e["type"] for e in events]

1332 + self.assertIn("region_open", types) # think opens during feed, not prefill

1333 + stream.finalize()

1334 + self.assertEqual(stream._output, {"role": "assistant", "thinking": "real body"})

1335 +

1336 + def test_prefix_is_required(self):

1337 + spec = {

1338 + "start_anchor": "<assistant>",

1339 + "fields": {"content": {"content": "text"}},

1340 + }

1341 + with self.assertRaisesRegex(ValueError, "requires `prefix`"):

1342 + ResponseParser(spec)

1343 +

1344 +

1345 +# xml-inline without a value_parser: parameter bodies stay raw strings until tools= coerces them.

1346 +_XML_STRING_ARGS_TEMPLATE = {

1347 + "defaults": {"role": "assistant"},

1348 + "start_anchor": "<|im_start|>assistant\n",

1349 + "fields": {

1350 + "tool_calls": {

1351 + "open_pattern": r"<tool_call>\s*<function=(?P<name>\w+)>",

1352 + "close": "</tool_call>",

1353 + "repeats": True,

1354 + "content": "xml-inline",

1355 + "content_args": {

1356 + "tag_pattern": r"<parameter=(?P<key>\w+)>\s*(?P<value>.*?)\s*</parameter>",

1357 + },

1358 + "transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}},

1359 + },

1360 + },

1361 +}

1362 +

1363 +# kv-lines without a value_parser: values likewise stay raw strings for tools= to cast.

1364 +_KV_LINES_TOOLS_TEMPLATE = {

1365 + "defaults": {"role": "assistant"},

1366 + "start_anchor": "<|im_start|>assistant\n",

1367 + "fields": {

1368 + "tool_calls": {

1369 + "open_pattern": r"<tool_call>\s*<function=(?P<name>\w+)>\n",

1370 + "close": "</tool_call>",

1371 + "repeats": True,

1372 + "content": "kv-lines",

1373 + "transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}},

1374 + },

1375 + },

1376 +}

1377 +

1378 +_SET_ALARM_CALL = (

1379 + "<tool_call>\n<function=set_alarm>\n"

1380 + "<parameter=hour>\n7\n</parameter>\n"

1381 + "<parameter=enabled>\ntrue\n</parameter>\n"

1382 + "<parameter=label>\nwake up\n</parameter>\n"

1383 + "</function>\n</tool_call>"

1384 +)

1385 +

1386 +

1387 +def _set_alarm_tools(**properties):

1388 + return [

1389 + {

1390 + "type": "function",

1391 + "function": {

1392 + "name": "set_alarm",

1393 + "parameters": {"type": "object", "properties": properties},

1394 + },

1395 + }

1396 + ]

1397 +

1398 +

1399 +_SET_ALARM_TOOLS = _set_alarm_tools(

1400 + hour={"type": "integer"},

1401 + enabled={"type": "boolean"},

1402 + label={"type": "string"},

1403 +)

1404 +

1405 +

1406 +def _first_tool_args(message):

1407 + return message["tool_calls"][0]["function"]["arguments"]

1408 +

1409 +

1410 +def _parser_with_tools(tools):

1411 + return ResponseParser(_XML_STRING_ARGS_TEMPLATE, prefix="", tools=tools)

1412 +

1413 +

1414 +class ToolArgCoercionTest(unittest.TestCase):

1415 + def test_coerce_tool_calls_casts_declared_types(self):

1416 + tools = _set_alarm_tools(

1417 + count={"type": "integer"},

1418 + ratio={"type": "number"},

1419 + enabled={"type": "boolean"},

1420 + tags={"type": "array"},

1421 + note={"type": "string"},

1422 + )

1423 + arguments = {

1424 + "count": "3",

1425 + "ratio": "1.5",

1426 + "enabled": "true",

1427 + "tags": '["a", "b"]',

1428 + "note": "hello",

1429 + "already_typed": 7,

1430 + "extra": "unscheduled",

1431 + }

1432 + call: dict[str, Any] = {

1433 + "type": "function",

1434 + "function": {"name": "set_alarm", "arguments": arguments},

1435 + }

1436 + self.assertIs(_parser_with_tools(tools)._coerce_tool_calls(call), call)

1437 + self.assertEqual(

1438 + call["function"]["arguments"],

1439 + {

1440 + "count": 3,

1441 + "ratio": 1.5,

1442 + "enabled": True,

1443 + "tags": ["a", "b"],

1444 + "note": "hello",

1445 + "already_typed": 7,

1446 + "extra": "unscheduled",

1447 + },

1448 + )

1449 +

1450 + def test_coerce_falls_back_to_raw_on_failure(self):

1451 + self.assertEqual(_coerce("not-a-number", ("integer",)), "not-a-number")

1452 +

1453 + def test_coerce_handles_any_of_and_null(self):

1454 + self.assertEqual(_coerce("5", ("integer", "null")), 5)

1455 + self.assertIsNone(_coerce("null", ("integer", "null")))

1456 +

1457 + def test_coerce_booleans_match_bool_parser(self):

1458 + # Accept the same literals as the `bool` content parser, case-insensitively.

1459 + for raw, expected in [("true", True), ("True", True), ("1", True), ("false", False), ("0", False)]:

1460 + self.assertEqual(_coerce(raw, ("boolean",)), expected)

1461 + # Non-boolean text stays a string rather than silently becoming False.

1462 + self.assertEqual(_coerce("maybe", ("boolean",)), "maybe")

1463 +

1464 + def test_coerce_object_array_require_matching_json(self):

1465 + # A JSON object body is only accepted for an `object` param, a JSON array only for `array`.

1466 + self.assertEqual(_coerce("[1, 2]", ("object",)), "[1, 2]")

1467 + self.assertEqual(_coerce('{"a": 1}', ("array",)), '{"a": 1}')

1468 + self.assertEqual(_coerce('{"a": 1}', ("object",)), {"a": 1})

1469 + # NaN / inf are not valid JSON numbers, so a `number` param keeps the raw text.

1470 + self.assertEqual(_coerce("NaN", ("number",)), "NaN")

1471 +

1472 + def test_coerce_tool_calls_handles_single_and_list(self):

1473 + parser = _parser_with_tools(_SET_ALARM_TOOLS)

1474 + call: dict[str, Any] = {

1475 + "type": "function",

1476 + "function": {"name": "set_alarm", "arguments": {"hour": "7"}},

1477 + }

1478 + self.assertIs(parser._coerce_tool_calls(call), call)

1479 + self.assertEqual(call["function"]["arguments"], {"hour": 7})

1480 + # A list of calls (as produced by `transform_each`) is coerced element-wise.

1481 + calls = [{"type": "function", "function": {"name": "set_alarm", "arguments": {"hour": "9"}}}]

1482 + self.assertEqual(parser._coerce_tool_calls(calls)[0]["function"]["arguments"], {"hour": 9})

1483 + # Non-tool-call values pass through untouched.

1484 + self.assertEqual(parser._coerce_tool_calls("hello"), "hello")

1485 +

1486 + def test_schema_types_handles_get_json_schema_dialect(self):

1487 + self.assertEqual(_schema_types({"type": "integer"}), ("integer",))

1488 + self.assertEqual(_schema_types({"type": ["integer", "string"]}), ("integer", "string"))

1489 + self.assertEqual(_schema_types({"anyOf": [{"type": "boolean"}, {"type": "string"}]}), ("boolean", "string"))

1490 + self.assertEqual(_schema_types({"oneOf": [{"type": "number"}, {"type": "null"}]}), ("number", "null"))

1491 + self.assertEqual(_schema_types({"type": "integer", "nullable": True}), ("integer", "null"))

1492 + # Undescribed parameters resolve to no candidate types, making coercion a no-op.

1493 + self.assertEqual(_schema_types({"description": "no type"}), ())

1494 +

1495 + def test_parse_response_tools_coerces_xml_inline_string_args(self):

1496 + # Without a value_parser, xml-inline argument bodies stay strings; tools= casts them.

1497 + without = parse_response(_SET_ALARM_CALL, _XML_STRING_ARGS_TEMPLATE, prefix="")

1498 + with_tools = parse_response(_SET_ALARM_CALL, _XML_STRING_ARGS_TEMPLATE, prefix="", tools=_SET_ALARM_TOOLS)

1499 + self.assertEqual(_first_tool_args(without), {"hour": "7", "enabled": "true", "label": "wake up"})

1500 + self.assertEqual(_first_tool_args(with_tools), {"hour": 7, "enabled": True, "label": "wake up"})

1501 +

1502 + def test_parse_response_tools_coerces_one_of_string_args(self):

1503 + tools = _set_alarm_tools(hour={"oneOf": [{"type": "integer"}, {"type": "null"}]})

1504 + parsed = parse_response(_SET_ALARM_CALL, _XML_STRING_ARGS_TEMPLATE, prefix="", tools=tools)

1505 + self.assertEqual(_first_tool_args(parsed)["hour"], 7)

1506 +

1507 + def test_streaming_tools_coerces_on_region_close(self):

1508 + # Coercion must land on the region_close event during feed(), not only after finalize().

1509 + stream = ResponseParser(_XML_STRING_ARGS_TEMPLATE, prefix="", tools=_SET_ALARM_TOOLS)

1510 + closes = [

1511 + event["value"]

1512 + for chunk in _chunk_fixed(_SET_ALARM_CALL, 8)

1513 + for event in stream.feed(chunk)

1514 + if event["type"] == "region_close" and event["field"] == "tool_calls"

1515 + ]

1516 + self.assertEqual(len(closes), 1)

1517 + self.assertEqual(closes[0]["function"]["arguments"], {"hour": 7, "enabled": True, "label": "wake up"})

1518 +

1519 + def test_qwen3_tools_coerces_strings_left_by_value_parser(self):

1520 + # qwen3's json+allow_non_json value_parser types what it can (`true`) and leaves

1521 + # invalid JSON (`007`) as a string; tools= then casts those leftover strings.

1522 + model_out = (

1523 + "<tool_call>\n<function=set_alarm>\n"

1524 + "<parameter=hour>\n007\n</parameter>\n"

1525 + "<parameter=enabled>\ntrue\n</parameter>\n"

1526 + "<parameter=label>\nwake up\n</parameter>\n"

1527 + "</function>\n</tool_call>"

1528 + )

1529 + without = parse_response(model_out, qwen3_template, prefix="")

1530 + with_tools = parse_response(model_out, qwen3_template, prefix="", tools=_SET_ALARM_TOOLS)

1531 + self.assertEqual(_first_tool_args(without), {"hour": "007", "enabled": True, "label": "wake up"})

1532 + self.assertEqual(_first_tool_args(with_tools), {"hour": 7, "enabled": True, "label": "wake up"})

1533 +

1534 + def test_coercion_never_reworks_already_typed_values(self):

1535 + # Coercion only casts strings: values the value_parser already typed are final

1536 + model_out = "<tool_call>\n<function=set_alarm>\n<parameter=label>\n1.50\n</parameter>\n</tool_call>"

1537 + # qwen3's lax value_parser has already read 1.50 as the float 1.5, so the string-typed label stays a float ...

1538 + self.assertEqual(

1539 + _first_tool_args(parse_response(model_out, qwen3_template, prefix="", tools=_SET_ALARM_TOOLS)),

1540 + {"label": 1.5},

1541 + )

1542 + # ... while without a value_parser the raw text reaches the schema cast intact

1543 + self.assertEqual(

1544 + _first_tool_args(parse_response(model_out, _XML_STRING_ARGS_TEMPLATE, prefix="", tools=_SET_ALARM_TOOLS)),

1545 + {"label": "1.50"},

1546 + )

1547 +

1548 + def test_kv_lines_string_args_are_coerced(self):

1549 + model_out = "<tool_call>\n<function=set_alarm>\nhour: 7\nenabled: true\n</tool_call>"

1550 + without = parse_response(model_out, _KV_LINES_TOOLS_TEMPLATE, prefix="")

1551 + with_tools = parse_response(model_out, _KV_LINES_TOOLS_TEMPLATE, prefix="", tools=_SET_ALARM_TOOLS)

1552 + self.assertEqual(_first_tool_args(without), {"hour": "7", "enabled": "true"})

1553 + self.assertEqual(_first_tool_args(with_tools), {"hour": 7, "enabled": True})

1554 +

1555 + def test_non_tool_call_regions_are_untouched(self):

1556 + # A field that captures a `name` but does not parse into a tool call must be left

1557 + # alone, even when the capture happens to match a tool: its keys are not arguments.

1558 + template = {

1559 + "defaults": {"role": "assistant"},

1560 + "start_anchor": "<|im_start|>assistant\n",

1561 + "fields": {

1562 + "citation": {

1563 + "open_pattern": r"<cite source=(?P<name>\w+)>",

1564 + "close": "</cite>",

1565 + "content": "xml-inline",

1566 + "content_args": {

1567 + "tag_pattern": r"<(?P<key>\w+)>\s*(?P<value>.*?)\s*</\1>",

1568 + "value_parser": {"name": "json", "args": {"allow_non_json": True}},

1569 + },

1570 + "transform": {"source": "{name}", "fields": "{content}"},

1571 + },

1572 + },

1573 + }

1574 + model_out = "<cite source=set_alarm><label>1.50</label><hour>7</hour></cite>"

1575 + expected = {"source": "set_alarm", "fields": {"label": 1.5, "hour": 7}}

1576 + self.assertEqual(parse_response(model_out, template, prefix="")["citation"], expected)

1577 + self.assertEqual(

1578 + parse_response(model_out, template, prefix="", tools=_SET_ALARM_TOOLS)["citation"],

1579 + expected,

1580 + )

1581 +

1582 + def test_merge_duplicates_arguments_are_cast_element_wise(self):

1583 + # merge_duplicates collects repeated tags into a list, which is cast element-wise

1584 + template: dict[str, Any] = copy.deepcopy(_XML_STRING_ARGS_TEMPLATE)

1585 + template["fields"]["tool_calls"]["content_args"]["merge_duplicates"] = True

1586 + model_out = (

1587 + "<tool_call>\n<function=set_alarm>\n"

1588 + "<parameter=hour>\n7\n</parameter>\n"

1589 + "<parameter=hour>\n9\n</parameter>\n"

1590 + "</function>\n</tool_call>"

1591 + )

1592 + self.assertEqual(_first_tool_args(parse_response(model_out, template, prefix="")), {"hour": ["7", "9"]})

1593 + self.assertEqual(

1594 + _first_tool_args(parse_response(model_out, template, prefix="", tools=_SET_ALARM_TOOLS)),

1595 + {"hour": [7, 9]},

1596 + )

1597 + # Elements that don't cast, and non-string elements, are left as they are.

1598 + parser = _parser_with_tools(_SET_ALARM_TOOLS)

1599 + call: dict[str, Any] = {

1600 + "type": "function",

1601 + "function": {

1602 + "name": "set_alarm",

1603 + "arguments": {"hour": ["7", "x", 9]},

1604 + },

1605 + }

1606 + parser._coerce_tool_calls(call)

1607 + self.assertEqual(call["function"]["arguments"], {"hour": [7, "x", 9]})

1608 +

1609 + def test_already_decoded_array_is_not_cast_element_wise(self):

1610 + tools = _set_alarm_tools(groups={"type": "array", "items": {"type": "string"}})

1611 + call: dict[str, Any] = {

1612 + "type": "function",

1613 + "function": {

1614 + "name": "set_alarm",

1615 + "arguments": {"groups": ["[1,2]", "[]"]},

1616 + },

1617 + }

1618 + _parser_with_tools(tools)._coerce_tool_calls(call)

1619 + self.assertEqual(call["function"]["arguments"], {"groups": ["[1,2]", "[]"]})

1620 +

1621 + def test_coerce_tool_calls_ignores_unusable_function_name(self):

1622 + # A transform can hand us a name parsed from model output, so a non-string name

1623 + # must be ignored rather than raising on the schema lookup.

1624 + parser = _parser_with_tools(_SET_ALARM_TOOLS)

1625 + call: dict[str, Any] = {

1626 + "type": "function",

1627 + "function": {"name": ["set_alarm"], "arguments": {"hour": "7"}},

1628 + }

1629 + self.assertEqual(parser._coerce_tool_calls(call), call)

1630 + self.assertEqual(call["function"]["arguments"], {"hour": "7"})

1631 +

1632 + def test_union_with_container_still_keeps_scalar_text(self):

1633 + model_out = "<tool_call>\n<function=set_alarm>\n<parameter=label>\n1.50\n</parameter>\n</tool_call>"

1634 + union = _set_alarm_tools(label={"anyOf": [{"type": "string"}, {"type": "object"}]})

1635 + # `string` never casts, so the union keeps scalar-looking text as text ...

1636 + self.assertEqual(

1637 + _first_tool_args(parse_response(model_out, _XML_STRING_ARGS_TEMPLATE, prefix="", tools=union)),

1638 + {"label": "1.50"},

1639 + )

1640 + # ... while the `object` branch still decodes a body that really is an object.

1641 + object_body = '<tool_call>\n<function=set_alarm>\n<parameter=label>\n{"a": 1}\n</parameter>\n</tool_call>'

1642 + self.assertEqual(

1643 + _first_tool_args(parse_response(object_body, _XML_STRING_ARGS_TEMPLATE, prefix="", tools=union)),

1644 + {"label": {"a": 1}},

1645 + )

1646 +

1647 + def test_tools_is_a_noop_for_typed_json_tool_calls(self):

1648 + # JSON tool-call bodies already carry their types, so passing tools= must not change them.

1649 + model_out = (

1650 + "<|START_THINKING|>x<|END_THINKING|>"

1651 + '<|START_ACTION|>[{"tool_call_id": "0", "tool_name": "set_alarm", '

1652 + '"parameters": {"hour": 7, "enabled": true}}]<|END_ACTION|><|END_OF_TURN_TOKEN|>'

1653 + )

1654 + without = parse_response(model_out, cohere_template, prefix="")

1655 + with_tools = parse_response(model_out, cohere_template, prefix="", tools=_SET_ALARM_TOOLS)

1656 + self.assertEqual(without, with_tools)

1657 + self.assertEqual(_first_tool_args(with_tools), {"hour": 7, "enabled": True})

1658 +

1659 +

1660 +if __name__ == "__main__":

1661 + unittest.main()

1662 +# fmt: on

vllm/parser/chat_parsing/__init__.py new

+26

@@ -0,0 +1,26 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

5 +# ruff: noqa

6 +# fmt: off

7 +#

8 +# Licensed under the Apache License, Version 2.0 (the "License");

9 +# you may not use this file except in compliance with the License.

10 +# You may obtain a copy of the License at

11 +#

12 +# http://www.apache.org/licenses/LICENSE-2.0

13 +#

14 +# Unless required by applicable law or agreed to in writing, software

15 +# distributed under the License is distributed on an "AS IS" BASIS,

16 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

17 +# See the License for the specific language governing permissions and

18 +# limitations under the License.

19 +"""Response parsing: convert model-emitted text into the assistant-message

20 +dict used by chat templates, driven by a declarative `response_template` spec."""

21 +

22 +from .response_parser import ResponseParser, parse_response

23 +

24 +

25 +__all__ = ["ResponseParser", "parse_response"]

26 +# fmt: on

vllm/parser/chat_parsing/content_parsers.py new

+255

@@ -0,0 +1,255 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

5 +# ruff: noqa

6 +# fmt: off

7 +#

8 +# Licensed under the Apache License, Version 2.0 (the "License");

9 +# you may not use this file except in compliance with the License.

10 +# You may obtain a copy of the License at

11 +#

12 +# http://www.apache.org/licenses/LICENSE-2.0

13 +#

14 +# Unless required by applicable law or agreed to in writing, software

15 +# distributed under the License is distributed on an "AS IS" BASIS,

16 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

17 +# See the License for the specific language governing permissions and

18 +# limitations under the License.

19 +"""This file contains the parsers used by chat response parsing. Each parser takes a chunk of captured text

20 +and parses it into a single key in the output message dictionary. Functions are generally boilerplate."""

21 +

22 +from __future__ import annotations

23 +

24 +import json

25 +from typing import Any

26 +

27 +import regex as re

28 +

29 +

30 +def _text(text: str, args: dict) -> str:

31 + return text.strip() if args.get("strip", True) else text

32 +

33 +

34 +def _int(text: str, args: dict) -> int:

35 + return int(_text(text, args))

36 +

37 +

38 +def _float(text: str, args: dict) -> float:

39 + return float(_text(text, args))

40 +

41 +

42 +def _bool(text: str, args: dict) -> bool:

43 + return _text(text, args).lower() in ("true", "1")

44 +

45 +

46 +# Sentinel characters for lax-JSON string pre-extraction — ASCII control chars

47 +# that should never appear in real LLM output.

48 +_LAX_OPEN, _LAX_CLOSE = "\x01", "\x02"

49 +

50 +

51 +def _json(text: str, args: dict) -> Any:

52 + """JSON parser with optional dialect knobs for LLM-emitted quirks.

53 +

54 + `args`:

55 + - `unquoted_keys` (bool): quote bare-identifier keys before parsing.

56 + - `string_delims` ([[open, close], ...]): strings delimited by these

57 + custom markers are pre-extracted, then restored as standard JSON strings.

58 + - `allow_non_json` (bool): return stripped text if parsing fails.

59 + """

60 + string_delims = args.get("string_delims", [])

61 + unquoted_keys = args.get("unquoted_keys", False)

62 +

63 + if string_delims and (_LAX_OPEN in text or _LAX_CLOSE in text):

64 + raise ValueError("json: input contains reserved sentinel characters (\\x01/\\x02); cannot parse safely.")

65 +

66 + working = text

67 + captured: list[str] = []

68 + for open_d, close_d in string_delims:

69 + pattern = re.escape(open_d) + r"(.*?)" + re.escape(close_d)

70 +

71 + def _capture(m: Any) -> str:

72 + captured.append(m.group(1))

73 + return f"{_LAX_OPEN}{len(captured) - 1}{_LAX_CLOSE}"

74 +

75 + working = re.sub(pattern, _capture, working, flags=re.DOTALL)

76 +

77 + if unquoted_keys:

78 + working = re.sub(r"(?<=[{,])(\w+):", r'"\1":', working)

79 +

80 + for i, s in enumerate(captured):

81 + working = working.replace(f"{_LAX_OPEN}{i}{_LAX_CLOSE}", json.dumps(s))

82 +

83 + try:

84 + return json.loads(working)

85 + except json.JSONDecodeError as e:

86 + if args.get("allow_non_json"):

87 + return _text(text, args)

88 + if working == text:

89 + raise ValueError(f"json parser could not parse region as JSON.\nContent: {text!r}\nError: {e}") from e

90 + raise ValueError(

91 + f"json: could not parse after dialect transforms.\n"

92 + f"Original: {text!r}\nTransformed: {working!r}\nError: {e}"

93 + ) from e

94 +

95 +

96 +def _sub_parse(raw: str, value_parser: dict | None) -> Any:

97 + if value_parser is None:

98 + return raw

99 + return parse_content(raw, value_parser.get("name", "text"), value_parser.get("args", {}))

100 +

101 +

102 +def _xml_inline(text: str, args: dict) -> dict:

103 + """Parse shallow XML-ish tags into a dict. `tag_pattern` regex must have named

104 + groups `key` and `value`. Optional `value_parser` recurses; `merge_duplicates`

105 + collects duplicate keys into a list."""

106 + tag_pattern = args.get("tag_pattern")

107 + if tag_pattern is None:

108 + raise ValueError("xml-inline: 'tag_pattern' content_arg is required")

109 + value_parser = args.get("value_parser")

110 + merge = args.get("merge_duplicates", False)

111 +

112 + out: dict[str, Any] = {}

113 + for m in re.finditer(tag_pattern, text, flags=re.DOTALL):

114 + groups = m.groupdict()

115 + key = groups.get("key")

116 + if key is None:

117 + raise ValueError(f"xml-inline: tag_pattern must have a named group 'key'. Pattern: {tag_pattern}")

118 + value = _sub_parse(groups.get("value", ""), value_parser)

119 + if key in out and merge:

120 + if not isinstance(out[key], list):

121 + out[key] = [out[key]]

122 + out[key].append(value)

123 + else:

124 + out[key] = value

125 + return out

126 +

127 +

128 +def _kv_lines(text: str, args: dict) -> dict:

129 + """Parse line-delimited `key<sep>value` pairs into a dict."""

130 + line_sep = args.get("line_sep", "\n")

131 + kv_sep = args.get("kv_sep", ":")

132 + value_parser = args.get("value_parser")

133 +

134 + out: dict[str, Any] = {}

135 + for line in text.split(line_sep):

136 + line = _text(line, args)

137 + if not line or kv_sep not in line:

138 + continue

139 + k, v = line.split(kv_sep, 1)

140 + k, v = _text(k, args), _text(v, args)

141 + out[k] = _sub_parse(v, value_parser)

142 + return out

143 +

144 +

145 +CONTENT_PARSERS = {

146 + "text": _text,

147 + "int": _int,

148 + "float": _float,

149 + "bool": _bool,

150 + "json": _json,

151 + "xml-inline": _xml_inline,

152 + "kv-lines": _kv_lines,

153 +}

154 +

155 +# Parsers whose output is the verbatim body text (modulo whitespace) — chunks

156 +# from these fields stream with `dirty=False` because each chunk is part of

157 +# the final value. Structured parsers (`json`, `xml-inline`, `kv-lines`) only

158 +# produce a meaningful value on close, so their chunks stream raw bytes

159 +# flagged `dirty=True` while the parsed value is delivered in `region_close`.

160 +STREAMABLE_PARSERS = frozenset({"text", "int", "float", "bool"})

161 +

162 +

163 +def parse_content(text: str, name: str, args: dict) -> Any:

164 + return CONTENT_PARSERS[name](text, args)

165 +

166 +

167 +_PLACEHOLDER = re.compile(r"\{(\w+(?:\.\w+)*)\}")

168 +

169 +

170 +def _apply_transform(transform: Any, scope: dict) -> Any:

171 + """Recursively walk a transform template, which is used to restructure

172 + parsed output into the actual shape we want. A dotted placeholder like

173 + `{content.args}` descends into keys of the looked-up value."""

174 + if isinstance(transform, dict):

175 + return {k: _apply_transform(v, scope) for k, v in transform.items()}

176 + if isinstance(transform, list):

177 + return [_apply_transform(v, scope) for v in transform]

178 + if not isinstance(transform, str):

179 + return transform

180 + whole = _PLACEHOLDER.fullmatch(transform)

181 + if not whole:

182 + return transform

183 + path = whole.group(1)

184 + root, *keys = path.split(".")

185 + if root not in scope:

186 + raise KeyError(f"transform placeholder '{{{path}}}' is not defined. Available: {sorted(scope)}")

187 + value = scope[root]

188 + for key in keys:

189 + if not isinstance(value, dict):

190 + raise ValueError(f"transform placeholder '{{{path}}}' cannot index into {type(value).__name__} at '{key}'")

191 + if key not in value:

192 + raise ValueError(f"transform placeholder '{{{path}}}' is missing key '{key}'. Available: {sorted(value)}")

193 + value = value[key]

194 + return value

195 +

196 +

197 +def validate_transform_strings(scope: str, transform: Any) -> None:

198 + """Walk a transform template and reject any string that mixes a `{name}`

199 + placeholder with literal text. Only whole-string placeholders (e.g.

200 + `"{content}"`) and plain literals are supported. Called from the template

201 + loader so authors get a clear error at load time, not at parse time."""

202 + if isinstance(transform, dict):

203 + for v in transform.values():

204 + validate_transform_strings(scope, v)

205 + return

206 + if isinstance(transform, list):

207 + for v in transform:

208 + validate_transform_strings(scope, v)

209 + return

210 + if not isinstance(transform, str):

211 + return

212 + if _PLACEHOLDER.search(transform) and not _PLACEHOLDER.fullmatch(transform):

213 + raise ValueError(

214 + f"{scope}: transform string {transform!r} mixes a {{placeholder}} with literal text. "

215 + 'Use either a whole-string placeholder (e.g. "{content}") or a plain literal; '

216 + "string interpolation is not supported."

217 + )

218 +

219 +

220 +def process_field(body: str, field, captures: dict) -> Any:

221 + """Run `body` through the field's content parser, then optionally apply the

222 + transform template. When `transform_each` is set, the parsed content must

223 + be a list and the template is applied to each element (with the element's

224 + keys unpacked into the template scope, alongside any regex captures).

225 +

226 + `field` is a `spec.Field`; typed via duck-typing to avoid a cyclic import."""

227 + value = parse_content(body, field.content, field.content_args)

228 + if field.transform is None:

229 + return value

230 + if field.transform_each:

231 + if not isinstance(value, list):

232 + raise ValueError(

233 + f"Field '{field.name}': transform_each requires the parsed content to be a list, "

234 + f"got {type(value).__name__}."

235 + )

236 + out = []

237 + for item in value:

238 + if not isinstance(item, dict):

239 + raise ValueError(

240 + f"Field '{field.name}': transform_each requires each list element to be a dict, "

241 + f"got {type(item).__name__}."

242 + )

243 + out.append(_apply_transform(field.transform, {**captures, **item}))

244 + return out

245 + return _apply_transform(field.transform, {**captures, "content": value})

246 +

247 +

248 +__all__ = [

249 + "CONTENT_PARSERS",

250 + "STREAMABLE_PARSERS",

251 + "parse_content",

252 + "process_field",

253 + "validate_transform_strings",

254 +]

255 +# fmt: on

vllm/parser/chat_parsing/response_parser.py new

+408

@@ -0,0 +1,408 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

5 +# ruff: noqa

6 +# fmt: off

7 +#

8 +# Licensed under the Apache License, Version 2.0 (the "License");

9 +# you may not use this file except in compliance with the License.

10 +# You may obtain a copy of the License at

11 +#

12 +# http://www.apache.org/licenses/LICENSE-2.0

13 +#

14 +# Unless required by applicable law or agreed to in writing, software

15 +# distributed under the License is distributed on an "AS IS" BASIS,

16 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

17 +# See the License for the specific language governing permissions and

18 +# limitations under the License.

19 +

20 +from __future__ import annotations

21 +

22 +import copy

23 +import json

24 +from typing import Any

25 +

26 +from .content_parsers import STREAMABLE_PARSERS, process_field

27 +from .response_templates import (

28 + ResponseTemplate,

29 + ResponseTemplateField,

30 + load_response_template,

31 +)

32 +

33 +

34 +def _schema_types(schema: Any) -> tuple[str, ...]:

35 + if not isinstance(schema, dict):

36 + return ()

37 + declared = schema.get("type")

38 + types = [declared] if isinstance(declared, str) else []

39 + if isinstance(declared, list):

40 + types.extend(t for t in declared if isinstance(t, str))

41 + for union_name in ("anyOf", "oneOf"):

42 + for choice in schema.get(union_name) or []:

43 + types.extend(_schema_types(choice))

44 + if schema.get("nullable") and "null" not in types: # `nullable` is how get_json_schema marks Optionals

45 + types.append("null")

46 + return tuple(types)

47 +

48 +

49 +def _coerce(raw: str, types: tuple[str, ...]) -> Any:

50 + for type_name in types:

51 + try:

52 + if type_name == "integer":

53 + return int(raw)

54 + if type_name == "number":

55 + number = float(raw)

56 + if number != number or number in (float("inf"), float("-inf")):

57 + continue # NaN / inf are not valid JSON numbers

58 + # Preserve ints when the source text had no fractional part.

59 + return int(number) if number.is_integer() and "." not in raw else number

60 + if type_name == "boolean" and raw.strip().lower() in ("true", "1", "false", "0"):

61 + return raw.strip().lower() in ("true", "1")

62 + if type_name == "null" and raw.strip() in ("null", "None"):

63 + return None

64 + if type_name in ("object", "array"):

65 + decoded = json.loads(raw)

66 + if isinstance(decoded, dict if type_name == "object" else list):

67 + return decoded

68 + except ValueError:

69 + continue

70 + return raw # `string` params, unknown types and failed casts all keep the original text

71 +

72 +

73 +def parse_response(

74 + text: str,

75 + response_template: dict | ResponseTemplate,

76 + *,

77 + prefix: str | None = None,

78 + tools: list[dict] | None = None,

79 +) -> dict:

80 + """The main function for response parsing when you don't want streaming. Takes generated output

81 + and the prompt prefix and parses them without streaming any events, then returns the parsed message.

82 +

83 + Pass OpenAI-style `tools` dictionaries to cast tool-call arguments

84 + using the calling tool's JSON schema.

85 + """

86 + response_template = load_response_template(response_template)

87 + stream = ResponseParser(response_template, prefix=prefix, tools=tools)

88 + stream.feed(text)

89 + message, _ = stream.finalize()

90 + return message

91 +

92 +

93 +class ResponseParser:

94 + """This class implements a streaming parser with a `response_template`. If you don't need streaming and

95 + just want to parse a complete message, use the `parse_response` function above. Streaming parsing emits

96 + events indicating when regions (message fields) are opened and closed, with the model writing to the region

97 + that is currently open.

98 +

99 + Usage:

100 + parser = ResponseParser(response_template, prefix=chat_prompt)

101 + for event in parser.initial_events:

102 + handle(event)

103 + for chunk in model_text_stream:

104 + for event in parser.feed(chunk):

105 + handle(event)

106 + message, final_events = parser.finalize()

107 + for event in final_events:

108 + handle(event)

109 +

110 + Pass OpenAI-style `tools=` dictionaries to cast tool-call arguments

111 + using the calling tool's JSON schema as each region closes.

112 +

113 + Events can be either "region_open", "region_chunk", or "region_close".

114 +

115 + ResponseParser requires the chat `prefix` (i.e. the chat history, the prefill before the current generation).

116 + This is because chat templates or assistant prefills can sometimes write part of the message, and if we

117 + only see the model output, and not the template, then we can't reliably parse the message in those cases.

118 + Any events produced while consuming the prefix are exposed as `initial_events`, so renderers can show

119 + prefill regions before the model writes anything; closed prefill regions also land in the output dict.

120 + """

121 +

122 + def __init__(

123 + self,

124 + response_template: dict | ResponseTemplate,

125 + prefix: str | None = None,

126 + *,

127 + tools: list[dict] | None = None,

128 + ):

129 + self._spec = load_response_template(response_template)

130 + if prefix is None:

131 + raise ValueError(

132 + "`ResponseParser`/`parse_response` requires `prefix` (the chat prompt sent to the model before "

133 + "generation), because chat templates often pre-write part of the assistant message (e.g. an "

134 + "opening `<think>` tag) that the parser must see to parse the output correctly. If the generation "

135 + 'already contains the complete message, pass `prefix=""` to opt out explicitly.'

136 + )

137 + # Maps tool name -> schema `properties`, used to cast parsed tool-call arguments

138 + self._tool_params: dict[str, dict] = {}

139 + for tool in tools or []:

140 + fn = tool.get("function", tool) if isinstance(tool, dict) else None

141 + if isinstance(fn, dict) and isinstance(fn.get("name"), str):

142 + parameters = fn.get("parameters")

143 + properties = parameters.get("properties") if isinstance(parameters, dict) else None

144 + self._tool_params[fn["name"]] = properties if isinstance(properties, dict) else {}

145 + self._buffer: str = ""

146 + self._pos: int = 0

147 + self._output: dict[str, Any] = copy.deepcopy(self._spec.defaults)

148 + self._implicit_name: str | None = self._spec.implicit

149 + # Unified current-region state: starts in the implicit region (or a

150 + # null sink if none was declared), and returns there after every close.

151 + # For explicit regions `_opened` flips to True eagerly on the open

152 + # match; for the implicit region it flips lazily on the first byte.

153 + self._current: str | None = self._implicit_name

154 + self._captures: dict[str, str] = {}

155 + self._body: str = ""

156 + self._opened: bool = False

157 + self._finalized: bool = False

158 + self.initial_events: list[dict] = []

159 + if prefix:

160 + self._consume_prefix(prefix)

161 +

162 + def _consume_prefix(self, prefix: str) -> None:

163 + """Loads the prefix (the chat prefill sent to the model), right-truncates it to the start of the

164 + assistant message (as determined by start_anchor) and then runs the remainder through the parser.

165 + Events produced while processing the prefix are stashed on `initial_events` so callers can replay

166 + them into a renderer before feeding model output.

167 +

168 + Think of this as the "get the parser up to speed on the story so far" method.

169 + """

170 + truncated = self._spec.truncate_past_last_anchor(prefix)

171 + if not truncated:

172 + return

173 + self._buffer = truncated

174 + self._process(self.initial_events, eos=False)

175 +

176 + def feed(self, text: str) -> list[dict]:

177 + """Feeds more text/tokens from the model output into the tokenizer, and returns any events that result

178 + (regions entered or left). This is the method you want to call after each generation step."""

179 + if self._finalized:

180 + raise RuntimeError("ResponseParser already finalized")

181 + if text:

182 + self._buffer += text

183 + events: list[dict] = []

184 + self._process(events, eos=False)

185 + return events

186 +

187 + def finalize(self) -> tuple[dict, list[dict]]:

188 + """Close the stream and return the final message dict together with

189 + any finalization events. This is necessary because some regions may only

190 + end at the end of the sequence, so you won't see the event telling you they're

191 + ready until the sequence is finalized."""

192 +

193 + def _is_empty(v: Any) -> bool:

194 + return v is None or (isinstance(v, (list, dict, str)) and not v)

195 +

196 + if self._finalized:

197 + raise RuntimeError("ResponseParser already finalized")

198 + events: list[dict] = []

199 + self._process(events, eos=True)

200 + missing = [n for n, f in self._spec.fields.items() if not f.optional and n not in self._output]

201 + if missing:

202 + raise ValueError(f"Required response_template fields missing from parsed output: {missing}")

203 + defaults = self._spec.defaults

204 + self._output = {k: v for k, v in self._output.items() if k in defaults or not _is_empty(v)}

205 + self._finalized = True

206 + return self._output, events

207 +

208 + def _process(self, events: list[dict], eos: bool) -> None:

209 + while True:

210 + watch = self._watchlist()

211 + best, hold_start = self._scan(watch, eos)

212 +

213 + if best is not None:

214 + kind, field, m = best

215 + if m.start() > self._pos:

216 + self._accumulate(events, self._buffer[self._pos : m.start()])

217 + self._pos = m.end()

218 + if kind == "open":

219 + self._close_current(events)

220 + self._open_explicit(events, field, m)

221 + else: # "close" (always the implicit region's close here,

222 + # since explicit regions only expose their own close)

223 + had_content = self._opened

224 + self._close_current(events)

225 + # Zero-width close on an already-empty region would just

226 + # re-fire next iteration -- bail out to make progress.

227 + if not had_content and m.start() == m.end():

228 + break

229 + continue

230 +

231 + # No committable match in the current buffer.

232 + if eos:

233 + if self._pos < len(self._buffer):

234 + self._accumulate(events, self._buffer[self._pos :])

235 + self._pos = len(self._buffer)

236 + self._close_current(events)

237 + break

238 + # Stream everything up to the earliest still-pending delimiter. When

239 + # nothing is pending `hold_start == len(self._buffer)`, so this flushes

240 + # the whole buffer; otherwise we hold the (possibly partial) delimiter

241 + # bytes back until more input resolves them.

242 + if hold_start > self._pos:

243 + self._accumulate(events, self._buffer[self._pos : hold_start])

244 + self._pos = hold_start

245 + break

246 +

247 + def _watchlist(self) -> list[tuple[str, ResponseTemplateField]]:

248 + """Patterns we care about right now: the close of the currently-open

249 + explicit region, or -- if we're in the implicit/null region -- every

250 + explicit open plus the implicit's own close (if any)."""

251 + if self._current is not None and self._current != self._implicit_name:

252 + field = self._spec.fields[self._current]

253 + return [("close", field)] if field.close_re is not None else []

254 + watch: list[tuple[str, ResponseTemplateField]] = []

255 + for field in self._spec.fields.values():

256 + if field.open_re is not None:

257 + watch.append(("open", field))

258 + if self._implicit_name is not None:

259 + impl = self._spec.fields[self._implicit_name]

260 + if impl.close_re is not None:

261 + watch.append(("close", impl))

262 + return watch

263 +

264 + def _scan(

265 + self, watch: list[tuple[str, ResponseTemplateField]], eos: bool

266 + ) -> tuple[tuple[str, ResponseTemplateField, Any] | None, int]:

267 + """Single pass over the watched delimiters, using the `regex` module's

268 + partial matching to decide -- per delimiter -- whether it can be committed

269 + now or must be held. Returns `(best, hold_start)`:

270 +

271 + * `best` is the earliest-starting delimiter we can safely commit *now*

272 + (longest on ties, opens before closes), or `None`.

273 + * `hold_start` is the leftmost buffer position occupied by a still-pending

274 + match: a partial (incomplete) delimiter, or a complete one ending at the

275 + buffer edge that more input could still grow. Bytes before it are safe to

276 + emit; bytes from it onward must be held. It stays `len(self._buffer)` when

277 + nothing is pending, letting the caller flush the whole buffer.

278 +

279 + A complete match is committable only if it starts strictly before

280 + `hold_start` -- otherwise an earlier (or co-located) pending delimiter could

281 + turn out to be the real one. At EOS nothing can grow, so partial matching is

282 + skipped and every complete match is committable.

283 +

284 + (The `regex` module always reports the empty string as a live prefix, so a

285 + partial search with no real match returns a zero-width match at the buffer

286 + end; that lands in the pending branch with `start == len(self._buffer)`, a

287 + no-op for `hold_start`.)

288 + """

289 + best_key: tuple | None = None

290 + best: tuple[str, ResponseTemplateField, Any] | None = None

291 + hold_start = len(self._buffer)

292 + for kind, field in watch:

293 + # The watchlist only includes fields whose delimiter regex is set, so pattern is never None here.

294 + pattern = field.open_re if kind == "open" else field.close_re

295 + if eos:

296 + m = pattern.search(self._buffer, self._pos)

297 + else:

298 + m = pattern.search(self._buffer, self._pos, partial=True)

299 + if m is None:

300 + continue

301 + if not eos and (m.partial or self._can_grow(kind, field, m)):

302 + # Pending: can't commit, and blocks emitting from its start onward.

303 + hold_start = min(hold_start, m.start())

304 + continue

305 + key = (m.start(), -(m.end() - m.start()), 0 if kind == "open" else 1, field.name)

306 + if best_key is None or key < best_key:

307 + best_key, best = key, (kind, field, m)

308 + # A committable match co-located with or after a pending one must wait too:

309 + # the pending delimiter starts no later and might be the one that fires.

310 + if best is not None and best[2].start() >= hold_start:

311 + best = None

312 + return best, hold_start

313 +

314 + def _can_grow(self, kind: str, field: ResponseTemplateField, m: Any) -> bool:

315 + r"""Whether a *complete* match ending at the current buffer edge could still

316 + change as more input arrives -- in which case we defer rather than commit. A

317 + match ending before the edge has already seen its terminating byte and is

318 + final. At the edge: zero-width matches (`$` / `\Z`) are only real at true

319 + EOS; a fully-present literal that no other literal in its set extends cannot

320 + grow (the fast path that keeps literal delimiters zero-latency); anything

321 + else (regex delimiters, prefix-overlapping literal lists) might."""

322 + if m.end() != len(self._buffer):

323 + return False

324 + if m.start() == m.end():

325 + return True

326 + literals, can_extend = (

327 + (field.open_literals, field.open_literal_can_extend)

328 + if kind == "open"

329 + else (field.close_literals, field.close_literal_can_extend)

330 + )

331 + return literals is None or can_extend

332 +

333 + def _accumulate(self, events: list[dict], text: str) -> None:

334 + """Route `text` into the currently active region. When the current

335 + region is the null sink (no implicit declared, no explicit open), we

336 + silently discard. Every routed chunk emits a `region_chunk` event so

337 + consumers can render live; `dirty=True` flags chunks from structured

338 + parsers (json, xml-inline, kv-lines) whose raw bytes will only be

339 + parsed into the final value on close."""

340 + if not text or self._current is None:

341 + return

342 + field = self._spec.fields[self._current]

343 + if not self._opened:

344 + events.append({"type": "region_open", "field": self._current})

345 + self._opened = True

346 + self._body += text

347 + dirty = field.content not in STREAMABLE_PARSERS

348 + events.append({"type": "region_chunk", "field": self._current, "text": text, "dirty": dirty})

349 +

350 + def _open_explicit(self, events: list[dict], field: ResponseTemplateField, m: Any) -> None:

351 + self._current = field.name

352 + self._captures = {k: v for k, v in m.groupdict().items() if v is not None}

353 + self._body = ""

354 + self._opened = True

355 + events.append({"type": "region_open", "field": field.name})

356 +

357 + def _close_current(self, events: list[dict]) -> None:

358 + """Close the current region and reset to the implicit/null region.

359 + Skipped (aside from the reset) when the current region never opened --

360 + avoids vacuous open/close pairs at every explicit boundary."""

361 + if self._current is None or not self._opened:

362 + self._reset_to_implicit()

363 + return

364 + field = self._spec.fields[self._current]

365 + value = process_field(self._body, field, self._captures)

366 + if self._tool_params:

367 + value = self._coerce_tool_calls(value)

368 + if field.join is not None:

369 + if not isinstance(value, str):

370 + raise ValueError(

371 + f"Field '{field.name}': 'join' requires each match to parse to a string, "

372 + f"got {type(value).__name__}."

373 + )

374 + previous = self._output.get(self._current)

375 + self._output[self._current] = value if previous is None else previous + field.join + value

376 + elif field.repeats:

377 + self._output.setdefault(self._current, []).append(value)

378 + else:

379 + self._output[self._current] = value

380 + events.append({"type": "region_close", "field": self._current, "value": value})

381 + self._reset_to_implicit()

382 +

383 + def _coerce_tool_calls(self, value: Any) -> Any:

384 + if isinstance(value, list):

385 + return [self._coerce_tool_calls(item) for item in value]

386 + fn = value.get("function") if isinstance(value, dict) else None

387 + if not isinstance(fn, dict):

388 + return value

389 + name, arguments = fn.get("name"), fn.get("arguments")

390 + if not isinstance(name, str) or not isinstance(arguments, dict):

391 + return value

392 + if properties := self._tool_params.get(name):

393 + for key, argument in arguments.items():

394 + if key not in properties or not (types := _schema_types(properties[key])):

395 + continue

396 + if isinstance(argument, str):

397 + arguments[key] = _coerce(argument, types)

398 + elif isinstance(argument, list) and "array" not in types:

399 + # duplicate keys collected by `merge_duplicates`

400 + arguments[key] = [_coerce(item, types) if isinstance(item, str) else item for item in argument]

401 + return value

402 +

403 + def _reset_to_implicit(self) -> None:

404 + self._current = self._implicit_name

405 + self._captures = {}

406 + self._body = ""

407 + self._opened = False

408 +# fmt: on

vllm/parser/chat_parsing/response_templates.py new

+249

@@ -0,0 +1,249 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

5 +# ruff: noqa

6 +# fmt: off

7 +#

8 +# Licensed under the Apache License, Version 2.0 (the "License");

9 +# you may not use this file except in compliance with the License.

10 +# You may obtain a copy of the License at

11 +#

12 +# http://www.apache.org/licenses/LICENSE-2.0

13 +#

14 +# Unless required by applicable law or agreed to in writing, software

15 +# distributed under the License is distributed on an "AS IS" BASIS,

16 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

17 +# See the License for the specific language governing permissions and

18 +# limitations under the License.

19 +"""Template loading and validation for response_template dicts."""

20 +

21 +from __future__ import annotations

22 +

23 +import copy

24 +import logging

25 +from dataclasses import dataclass

26 +from typing import Any

27 +

28 +import regex as re

29 +

30 +from .content_parsers import CONTENT_PARSERS, validate_transform_strings

31 +

32 +

33 +logger = logging.getLogger(__name__)

34 +

35 +

36 +@dataclass

37 +class ResponseTemplateField:

38 + name: str

39 + open_re: Any

40 + open_literals: list[str] | None

41 + open_literal_can_extend: bool

42 + close_re: Any

43 + close_literals: list[str] | None

44 + close_literal_can_extend: bool

45 + content: str

46 + content_args: dict

47 + repeats: bool

48 + join: str | None

49 + optional: bool

50 + transform: Any

51 + transform_each: bool

52 +

53 +

54 +@dataclass

55 +class ResponseTemplate:

56 + defaults: dict

57 + fields: dict[str, ResponseTemplateField]

58 + start_anchor_re: Any

59 + implicit: str | None = None

60 + start_anchor_literals: list[str] | None = None

61 +

62 + def truncate_past_last_anchor(self, text: str) -> str:

63 + """When parsing responses, we can't just parse the tokens generated

64 + by the model. This is because chat templates often include early parts of the message such as <start_of_turn>

65 + or <think> tokens in the prefill. Therefore, we take the whole chat as input, but truncate to the start

66 + of the most recent assistant message, which can include tokens from the template as well as the output."""

67 + last_end: int | None = None

68 + for m in self.start_anchor_re.finditer(text):

69 + last_end = m.end()

70 + if last_end is None:

71 + kind = "start_anchor" if self.start_anchor_literals is not None else "start_anchor_pattern"

72 + logger.info(

73 + "response_template defines %s but the anchor was not found "

74 + "in the prefix; the parser will process the entire prefix instead.",

75 + kind,

76 + )

77 + return text

78 + return text[last_end:]

79 +

80 +

81 +def _compile_anchor(scope: str, field: dict, literal_key: str, pattern_key: str) -> tuple[Any, list[str] | None, bool]:

82 + """Compile a region's start/end anchor. An anchor can be:

83 +

84 + - a regex (`pattern_key`),

85 + - a single literal string (`literal_key`), or

86 + - a list of literal strings (`literal_key`), in which case any of these match

87 +

88 + All forms compile to a regex for matching, but we keep the literal strings around too: when present,

89 + the parser can emit regions faster, since it knows exactly how many trailing bytes might be a

90 + partial-match prefix (and that a finished literal match can never be extended by future bytes, except

91 + when a literal is a strict prefix of another in the same list).

92 +

93 + Returns `(compiled_re, literals, can_extend)`:

94 +

95 + - `compiled_re`: the compiled pattern.

96 + - `literals`: the literal strings, or `None` for a regex anchor.

97 + - `can_extend`: `True` iff one literal is a strict prefix of another, in which case a match at the

98 + buffer edge could still be lengthened by future input.

99 + """

100 + if literal_key in field and pattern_key in field:

101 + raise ValueError(f"{scope}: cannot specify both '{literal_key}' and '{pattern_key}'")

102 + if literal_key in field:

103 + raw = field[literal_key]

104 + if isinstance(raw, str):

105 + literals = [raw]

106 + elif isinstance(raw, list):

107 + if not raw:

108 + raise ValueError(f"{scope}: '{literal_key}' list must contain at least one literal")

109 + if not all(isinstance(s, str) for s in raw):

110 + raise ValueError(f"{scope}: '{literal_key}' list must contain only strings")

111 + literals = list(dict.fromkeys(raw)) # dedupe, preserve first-seen order

112 + else:

113 + raise ValueError(f"{scope}: '{literal_key}' must be a string or list of strings, got {type(raw).__name__}")

114 + if any(s == "" for s in literals):

115 + raise ValueError(f"{scope}: '{literal_key}' literals cannot be empty strings")

116 + # Sort longest-first so alternation prefers the longer alternative when both could match.

117 + ordered = sorted(literals, key=len, reverse=True)

118 + can_extend = any(a != b and a.startswith(b) for a in literals for b in literals)

119 + pattern = "|".join(re.escape(s) for s in ordered)

120 + return re.compile(pattern, re.DOTALL), literals, can_extend

121 + if pattern_key in field:

122 + try:

123 + return re.compile(field[pattern_key], re.DOTALL), None, False

124 + except re.error as e:

125 + raise ValueError(f"{scope}: invalid {pattern_key} regex: {e}") from e

126 + return None, None, False

127 +

128 +

129 +def _validate_template_shape(spec: dict) -> None:

130 + """Validate the top-level response_template structure: it must be a dict of the expected version with

131 + only known keys, a dict of `defaults`, and a non-empty dict of `fields`. Per-field specs are checked

132 + separately in `_build_field`."""

133 + if not isinstance(spec, dict):

134 + raise ValueError(f"response_template must be a dict, got {type(spec).__name__}")

135 + version = spec.get("version", 1)

136 + if version != 1:

137 + raise ValueError(f"Unsupported response_template version: {version}")

138 + if unknown_template_keys := set(spec) - {"version", "defaults", "fields", "start_anchor", "start_anchor_pattern"}:

139 + raise ValueError(f"Unknown keys in response_template: {sorted(unknown_template_keys)}")

140 + if not isinstance(spec.get("defaults", {}), dict):

141 + raise ValueError("response_template.defaults must be a dict")

142 + fields_raw = spec.get("fields", {})

143 + if not isinstance(fields_raw, dict) or not fields_raw:

144 + raise ValueError("response_template.fields must be a non-empty dict")

145 +

146 +

147 +def _build_field(name: str, field: dict) -> ResponseTemplateField:

148 + """Validate a single field spec and compile it into a `ResponseTemplateField`."""

149 +

150 + _ALLOWED_FIELD_KEYS = {

151 + "open",

152 + "open_pattern",

153 + "close",

154 + "close_pattern",

155 + "content",

156 + "content_args",

157 + "repeats",

158 + "join",

159 + "optional",

160 + "transform",

161 + "transform_each",

162 + }

163 + scope = f"Field '{name}'"

164 + if not isinstance(field, dict):

165 + raise ValueError(f"{scope} must be a dict")

166 + if unknown_field_keys := set(field) - _ALLOWED_FIELD_KEYS:

167 + raise ValueError(f"{scope}: unknown keys {sorted(unknown_field_keys)}")

168 + content = field.get("content", "text")

169 + if content not in CONTENT_PARSERS:

170 + raise ValueError(f"{scope}: unknown content parser '{content}'. Available: {sorted(CONTENT_PARSERS)}")

171 + open_re, open_literals, open_literal_can_extend = _compile_anchor(scope, field, "open", "open_pattern")

172 + close_re, close_literals, close_literal_can_extend = _compile_anchor(scope, field, "close", "close_pattern")

173 + join = field.get("join")

174 + if join is not None and not isinstance(join, str):

175 + raise ValueError(f"{scope}: 'join' must be a string, got {type(join).__name__}")

176 + if join is not None and not field.get("repeats", False):

177 + raise ValueError(f"{scope}: 'join' requires 'repeats': true")

178 + transform = field.get("transform")

179 + transform_each = field.get("transform_each", False)

180 + if not isinstance(transform_each, bool):

181 + raise ValueError(f"{scope}: transform_each must be a bool, got {type(transform_each).__name__}")

182 + if transform_each and transform is None:

183 + raise ValueError(f"{scope}: transform_each is set but no transform was provided")

184 + if transform is not None:

185 + validate_transform_strings(scope, transform)

186 + else:

187 + # Named captures only reach the output through a transform, so flag any that would be silently dropped.

188 + captured_names = set()

189 + if open_re is not None:

190 + captured_names |= set(open_re.groupindex)

191 + if close_re is not None:

192 + captured_names |= set(close_re.groupindex)

193 + if captured_names:

194 + raise ValueError(

195 + f"{scope}: open_pattern/close_pattern declares named group(s) "

196 + f"{sorted(captured_names)}, but the field has no 'transform'. Named captures "

197 + f"are only surfaced through a 'transform' template (where they appear "

198 + f"alongside 'content'). Either add a 'transform' that uses the captures, or "

199 + f"remove the named groups from the pattern."

200 + )

201 + return ResponseTemplateField(

202 + name=name,

203 + open_re=open_re,

204 + open_literals=open_literals,

205 + open_literal_can_extend=open_literal_can_extend,

206 + close_re=close_re,

207 + close_literals=close_literals,

208 + close_literal_can_extend=close_literal_can_extend,

209 + content=content,

210 + content_args=field.get("content_args", {}),

211 + repeats=field.get("repeats", False),

212 + join=join,

213 + optional=field.get("optional", True),

214 + transform=transform,

215 + transform_each=transform_each,

216 + )

217 +

218 +

219 +def load_response_template(spec: dict | ResponseTemplate) -> ResponseTemplate:

220 + if isinstance(spec, ResponseTemplate):

221 + return spec

222 + _validate_template_shape(spec)

223 +

224 + fields = {name: _build_field(name, raw) for name, raw in spec["fields"].items()}

225 + # A field without an open anchor is the implicit-open / leftover sink; at most one is allowed.

226 + implicit_fields = [name for name, field in fields.items() if field.open_re is None]

227 + if len(implicit_fields) > 1:

228 + raise ValueError(

229 + "At most one field may omit 'open'/'open_pattern' (that field becomes the "

230 + f"implicit-open / leftover sink). Found: {', '.join(implicit_fields)}"

231 + )

232 +

233 + start_anchor_re, start_anchor_literals, _ = _compile_anchor(

234 + "response_template", spec, "start_anchor", "start_anchor_pattern"

235 + )

236 + if start_anchor_re is None:

237 + raise ValueError("response_template must define 'start_anchor' or 'start_anchor_pattern'.")

238 +

239 + return ResponseTemplate(

240 + defaults=copy.deepcopy(spec.get("defaults", {})),

241 + fields=fields,

242 + implicit=implicit_fields[0] if implicit_fields else None,

243 + start_anchor_re=start_anchor_re,

244 + start_anchor_literals=start_anchor_literals,

245 + )

246 +

247 +

248 +__all__ = ["ResponseTemplate", "ResponseTemplateField", "load_response_template"]

249 +# fmt: on