@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4a39b4f553

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 3, 10:30 UTC

[Model] Add Apertus Tool Parser (#41154)

Signed-off-by: Blanc <swan.blanc@infomaniak.com>

Blanc Swan committed May 18, 2026 at 17:20 UTC 4a39b4f55374d48ebaa2ca02312e24639db8e0b8

5 files changed +1385

docs/features/tool_calling.md

+11

@@ -464,6 +464,17 @@ Supported models:

464

465 Flags: `--tool-call-parser gigachat3`

466

467 +### Apertus Models (`apertus`)

468 +

469 +Use the chat template from the examples folder; it fixes several OpenAI compatibility issues: `--chat-template /vllm-workspace/examples/tool_chat_template_apertus.jinja`

470 +

471 +Supported models:

472 +

473 +* `swiss-ai/Apertus-8B-Instruct-2509`

474 +* `swiss-ai/Apertus-70B-Instruct-2509`

475 +

476 +Flags: `--tool-call-parser apertus`

477 +

478 ### Models with Pythonic Tool Calls (`pythonic`)

479

480 A growing number of models output a python list to represent tool calls instead of using JSON. This has the advantage of inherently supporting parallel tool calls and removing ambiguity around the JSON schema required for tool calls. The `pythonic` tool parser can support such models.

examples/tool_chat_template_apertus.jinja new

+384

@@ -0,0 +1,384 @@

1 +{%- macro render_typescript_type(param_spec, required_params, is_nullable=false) -%}

2 + {%- if param_spec.type == "array" -%}

3 + {%- if param_spec['items'] -%}

4 + {%- if param_spec['items']['type'] == "string" -%}

5 + {{- "string[]" }}

6 + {%- elif param_spec['items']['type'] == "number" -%}

7 + {{- "number[]" }}

8 + {%- elif param_spec['items']['type'] == "integer" -%}

9 + {{- "number[]" }}

10 + {%- elif param_spec['items']['type'] == "boolean" -%}

11 + {{- "boolean[]" }}

12 + {%- else -%}

13 + {%- set inner_type = render_typescript_type(param_spec['items'], required_params) -%}

14 + {%- if inner_type == "object | object" or inner_type|length > 50 -%}

15 + {{- "any[]" }}

16 + {%- else -%}

17 + {{- inner_type + "[]" }}

18 + {%- endif -%}

19 + {%- endif -%}

20 + {%- if param_spec.nullable -%}

21 + {{- " | null" }}

22 + {%- endif -%}

23 + {%- else -%}

24 + {{- "any[]" }}

25 + {%- if param_spec.nullable -%}

26 + {{- " | null" }}

27 + {%- endif -%}

28 + {%- endif -%}

29 + {%- elif param_spec.type is defined and param_spec.type is iterable and param_spec.type is not string and param_spec.type is not mapping and param_spec.type[0] is defined -%}

30 + {#- Handle array of types like ["object", "object"] from Union[dict, list] #}

31 + {%- if param_spec.type | length > 1 -%}

32 + {{- param_spec.type | join(" | ") }}

33 + {%- else -%}

34 + {{- param_spec.type[0] }}

35 + {%- endif -%}

36 + {%- elif param_spec.oneOf -%}

37 + {#- Handle oneOf schemas - check for complex unions and fallback to any #}

38 + {%- set has_object_variants = false -%}

39 + {%- for variant in param_spec.oneOf -%}

40 + {%- if variant.type == "object" -%}

41 + {%- set has_object_variants = true -%}

42 + {%- endif -%}

43 + {%- endfor -%}

44 + {%- if has_object_variants and param_spec.oneOf|length > 1 -%}

45 + {{- "any" }}

46 + {%- else -%}

47 + {%- for variant in param_spec.oneOf -%}

48 + {{- render_typescript_type(variant, required_params) -}}

49 + {%- if variant.description %}

50 + {{- "// " + variant.description }}

51 + {%- endif -%}

52 + {%- if variant.default is defined %}

53 + {{ "// default: " + variant.default|tojson }}

54 + {%- endif -%}

55 + {%- if not loop.last %}

56 + {{- " | " }}

57 + {% endif -%}

58 + {%- endfor -%}

59 + {%- endif -%}

60 + {%- elif param_spec.type == "string" -%}

61 + {%- if param_spec.enum -%}

62 + {{- '"' + param_spec.enum|join('" | "') + '"' -}}

63 + {%- else -%}

64 + {{- "string" }}

65 + {%- if param_spec.nullable %}

66 + {{- " | null" }}

67 + {%- endif -%}

68 + {%- endif -%}

69 + {%- elif param_spec.type == "number" -%}

70 + {{- "number" }}

71 + {%- elif param_spec.type == "integer" -%}

72 + {{- "number" }}

73 + {%- elif param_spec.type == "boolean" -%}

74 + {{- "boolean" }}

75 + {%- elif param_spec.type == "object" -%}

76 + {%- if param_spec.properties -%}

77 + {{- "{\n" }}

78 + {%- for prop_name, prop_spec in param_spec.properties.items() -%}

79 + {{- prop_name -}}

80 + {%- if prop_name not in (param_spec.required or []) -%}

81 + {{- "?" }}

82 + {%- endif -%}

83 + {{- ": " }}

84 + {{ render_typescript_type(prop_spec, param_spec.required or []) }}

85 + {%- if not loop.last -%}

86 + {{-", " }}

87 + {%- endif -%}

88 + {%- endfor -%}

89 + {{- "}" }}

90 + {%- else -%}

91 + {{- "object" }}

92 + {%- endif -%}

93 + {%- else -%}

94 + {{- "any" }}

95 + {%- endif -%}

96 +{%- endmacro -%}

97 +

98 +{%- macro render_tools(tools) -%}

99 + {%- for tool in tools %}

100 + {%- if tool.function is defined -%}

101 + {#- Chat Completions format: {"type": "function", "function": {...}} #}

102 + {%- set func = tool.function -%}

103 + {%- if func.description is defined -%}

104 + {{- "// " + func.description + "\n" }}

105 + {%- endif -%}

106 + {{- "type "+ func.name + " = " }}

107 + {%- if func.parameters and func.parameters.properties %}

108 + {{- "(_: {\n" }}

109 + {%- for param_name, param_spec in func.parameters.properties.items() %}

110 + {%- if param_spec.description is defined %}

111 + {{- "// " + param_spec.description + "\n" }}

112 + {%- endif %}

113 + {{- param_name }}

114 + {%- if param_name not in (func.parameters.required or []) -%}

115 + {{- "?" }}

116 + {%- endif -%}

117 + {{- ": " }}

118 + {{- render_typescript_type(param_spec, func.parameters.required or []) }}

119 + {%- if param_spec.default is defined -%}

120 + {%- if param_spec.enum %}

121 + {{- ", // default: " + param_spec.default }}

122 + {%- elif param_spec.oneOf %}

123 + {{- "// default: " + param_spec.default }}

124 + {%- else %}

125 + {{- ", // default: " + param_spec.default|tojson }}

126 + {%- endif -%}

127 + {%- endif -%}

128 + {%- if not loop.last %}

129 + {{- ",\n" }}

130 + {%- else %}

131 + {{- "\n" }}

132 + {%- endif -%}

133 + {%- endfor %}

134 + {{- "}) => any;" }}

135 + {%- else -%}

136 + {{- "() => any;" }}

137 + {%- endif -%}

138 + {%- else -%}

139 + {#- Responses format: {"type": "function", "name": "...", ...} #}

140 + {%- if tool.description is defined -%}

141 + {{- "// " + tool.description + "\n" }}

142 + {%- endif -%}

143 + {{- "type "+ tool.name + " = " }}

144 + {%- if tool.parameters and tool.parameters.properties %}

145 + {{- "(_: {\n" }}

146 + {%- for param_name, param_spec in tool.parameters.properties.items() %}

147 + {%- if param_spec.description is defined %}

148 + {{- "// " + param_spec.description + "\n" }}

149 + {%- endif %}

150 + {{- param_name }}

151 + {%- if param_name not in (tool.parameters.required or []) -%}

152 + {{- "?" }}

153 + {%- endif -%}

154 + {{- ": " }}

155 + {{- render_typescript_type(param_spec, tool.parameters.required or []) }}

156 + {%- if param_spec.default is defined -%}

157 + {%- if param_spec.enum %}

158 + {{- ", // default: " + param_spec.default }}

159 + {%- elif param_spec.oneOf %}

160 + {{- "// default: " + param_spec.default }}

161 + {%- else %}

162 + {{- ", // default: " + param_spec.default|tojson }}

163 + {%- endif -%}

164 + {%- endif -%}

165 + {%- if not loop.last %}

166 + {{- ",\n" }}

167 + {%- else %}

168 + {{- "\n" }}

169 + {%- endif -%}

170 + {%- endfor %}

171 + {{- "}) => any;" }}

172 + {%- else -%}

173 + {{- "() => any;" }}

174 + {%- endif -%}

175 + {%- endif -%}

176 + {%- if not loop.last -%}

177 + {{- "\n" }}

178 + {%- endif -%}

179 + {%- endfor %}

180 +{%- endmacro -%}

181 +

182 +{{ bos_token }}

183 +

184 +{%- set system_token = '<|system_start|>' -%}

185 +{%- set end_system_token = '<|system_end|>' -%}

186 +{%- set developer_token = '<|developer_start|>' -%}

187 +{%- set end_developer_token = '<|developer_end|>' -%}

188 +{%- set user_token = '<|user_start|>' -%}

189 +{%- set end_user_token = '<|user_end|>' -%}

190 +{%- set assistant_token = '<|assistant_start|>' -%}

191 +{%- set end_assistant_token = '<|assistant_end|>' -%}

192 +{%- set inner_token = '<|inner_prefix|>' -%}

193 +{%- set outer_token = '<|inner_suffix|>' -%}

194 +{%- set tool_calls_token = '<|tools_prefix|>' -%}

195 +{%- set end_tool_calls_token = '<|tools_suffix|>' -%}

196 +

197 +{%- set ns = namespace(in_assistant=false, in_tool=false, in_inner=false, assistant_format=none) -%}

198 +

199 +{%- if messages and messages[0].role == 'system' -%}

200 + {%- if "content" in messages[0] -%}

201 + {%- if messages[0].content is string -%}

202 + {{ system_token + messages[0].content + end_system_token }}

203 + {%- elif messages[0].content is mapping and "text" in messages[0].content -%}

204 + {{ system_token + messages[0].content.text + end_system_token }}

205 + {%- else -%}

206 + {{- raise_exception("Invalid system message") -}}

207 + {%- endif -%}

208 + {%- else -%}

209 + {{- raise_exception("Invalid system message") -}}

210 + {%- endif -%}

211 + {%- set loop_messages = messages[1:] -%}

212 +{%- else -%}

213 + {{ system_token + 'You are Apertus, a helpful assistant created by the SwissAI initiative.\nKnowledge cutoff: 2024-04\nCurrent date: ' + strftime_now('%Y-%m-%d') + end_system_token }}

214 + {%- set loop_messages = messages -%}

215 +{%- endif -%}

216 +

217 +{{ developer_token + 'Deliberation: ' }}

218 +{%- if enable_thinking is defined and enable_thinking -%}

219 + {{ 'enabled\n' }}

220 +{%- else -%}

221 + {{ 'disabled\n' }}

222 +{%- endif -%}

223 +{%- if tools is defined and tools -%}

224 + {{ 'Tool Capabilities:\n' + render_tools(tools) }}

225 +{%- else -%}

226 + {{ 'Tool Capabilities: disabled' }}

227 +{%- endif -%}

228 +{{ end_developer_token }}

229 +

230 +{%- for message in loop_messages -%}

231 + {%- if message.role == 'user' -%}

232 + {%- set ns.in_inner = false -%}

233 + {%- if ns.in_tool -%}

234 + {{ ']' }}

235 + {%- set ns.in_tool = false -%}

236 + {%- endif -%}

237 + {%- if ns.in_assistant -%}

238 + {{ end_assistant_token }}

239 + {%- set ns.in_assistant = false -%}

240 + {%- endif -%}

241 + {%- if "content" in message -%}

242 + {{ user_token }}

243 + {%- if message.content is string -%}

244 + {{ message.content }}

245 + {%- elif message.content is mapping and "parts" in message.content -%}

246 + {%- set parts = message.content.parts -%}

247 + {%- for part in parts -%}

248 + {%- if part.type == "text" -%}

249 + {{ part.text }}

250 + {%- else -%}

251 + {{- raise_exception("Invalid user part: " + part.type) -}}

252 + {%- endif -%}

253 + {%- endfor -%}

254 + {%- else -%}

255 + {{- raise_exception("Invalid user message: " + message.role) -}}

256 + {%- endif -%}

257 + {{ end_user_token }}

258 + {%- endif -%}

259 + {%- elif message.role == 'assistant' -%}

260 + {%- if not ns.in_assistant -%}

261 + {{ assistant_token }}

262 + {%- set ns.in_assistant = true -%}

263 + {%- endif -%}

264 + {%- if "content" in message and message.content is not none -%}

265 + {%- if message.content is string and (ns.assistant_format is none or ns.assistant_format == "string") -%}

266 + {%- if ns.in_tool -%}

267 + {{ ']' }}

268 + {%- set ns.in_tool = false -%}

269 + {%- endif -%}

270 + {%- set ns.assistant_format = "string" -%}

271 + {{ message.content }}

272 + {%- elif message.content is mapping and "blocks" in message.content and (ns.assistant_format is none or ns.assistant_format == "mapping") -%}

273 + {%- set ns.assistant_format = "mapping" -%}

274 + {%- set blocks = message.content.blocks -%}

275 + {%- for block in blocks -%}

276 + {%- if block.type == 'thoughts' -%}

277 + {%- if ns.in_tool -%}

278 + {{ ']' }}

279 + {%- set ns.in_tool = false -%}

280 + {%- endif -%}

281 + {%- if not ns.in_inner -%}

282 + {%- set ns.in_inner = true -%}

283 + {{ inner_token }}

284 + {%- endif -%}

285 + {{ block.text }}

286 + {%- elif block.type == 'tool_calls' -%}

287 + {%- if ns.in_tool -%}

288 + {{ ']' }}

289 + {%- set ns.in_tool = false -%}

290 + {%- endif -%}

291 + {%- if ns.in_inner and not loop.first and block.calls|length == 1 and block.calls[0].name == 'display_answers' -%}

292 + {%- set ns.in_inner = false -%}

293 + {{ outer_token }}

294 + {%- endif -%}

295 + {{ tool_calls_token + '[' }}

296 + {%- for tool_call in block.calls -%}

297 + {%- set args = tool_call.arguments -%}

298 + {%- if args is string -%}

299 + {{- '{"' + tool_call.name + '": ' + args + '}' }}

300 + {%- else -%}

301 + {{- '{"' + tool_call.name + '": ' + args|tojson + '}' }}

302 + {%- endif -%}

303 + {%- if not loop.last -%}

304 + {{- ", " }}

305 + {%- endif -%}

306 + {%- endfor -%}

307 + {{ ']' + end_tool_calls_token }}

308 + {%- elif block.type == 'tool_outputs' -%}

309 + {%- if ns.in_tool -%}

310 + {{- raise_exception("Cannot have both tool outputs as separate messages and tool outputs as blocks") -}}

311 + {%- endif -%}

312 + {{ '[' }}

313 + {%- for tool_output in block.outputs -%}

314 + {{- tool_output.output }}

315 + {%- if not loop.last -%}

316 + {{- ", " }}

317 + {%- endif -%}

318 + {%- endfor -%}

319 + {{- ']' }}

320 + {%- elif block.type == 'response' -%}

321 + {%- if ns.in_tool -%}

322 + {{ ']' }}

323 + {%- set ns.in_tool = false -%}

324 + {%- endif -%}

325 + {%- if (not loop.first and ns.in_inner) or (ns.in_assistant and ns.in_inner) -%}

326 + {%- set ns.in_inner = false -%}

327 + {{ outer_token }}

328 + {%- endif -%}

329 + {{ block.text }}

330 + {%- else -%}

331 + {{- raise_exception("Invalid assistant block type: " + block.type) -}}

332 + {%- endif -%}

333 + {%- endfor -%}

334 + {%- else -%}

335 + {{- raise_exception("Invalid assistant content") -}}

336 + {%- endif -%}

337 + {%- elif not ("tool_calls" in message and message.tool_calls) -%}

338 + {{- raise_exception("Invalid assistant message") -}}

339 + {%- endif -%}

340 + {%- if "tool_calls" in message and message.tool_calls -%}

341 + {{ tool_calls_token + '[' }}

342 + {%- for tool_call in message.tool_calls -%}

343 + {%- if tool_call.type == 'function' -%}

344 + {%- set function = tool_call.function -%}

345 + {%- set args = function.arguments -%}

346 + {%- if args is string -%}

347 + {{- '{"' + function.name + '": ' + args + '}' }}

348 + {%- else -%}

349 + {{- '{"' + function.name + '": ' + args|tojson + '}' }}

350 + {%- endif -%}

351 + {%- if not loop.last -%}

352 + {{- ", " }}

353 + {%- endif -%}

354 + {%- else -%}

355 + {{- raise_exception("Invalid tool call type: " + tool_call.type) -}}

356 + {%- endif -%}

357 + {%- endfor -%}

358 + {{ ']' + end_tool_calls_token }}

359 + {%- endif -%}

360 + {%- elif message.role == 'tool' -%}

361 + {%- if not ns.in_assistant -%}

362 + {{- raise_exception("Tool message outside of assistant") -}}

363 + {%- endif -%}

364 + {%- if not ns.in_tool -%}

365 + {{ '[' }}

366 + {%- set ns.in_tool = true -%}

367 + {%- else -%}

368 + {{ ", "}}

369 + {%- endif -%}

370 + {%- if message.content is string -%}

371 + {{ message.content }}

372 + {%- else -%}

373 + {{ message.content|tojson }}

374 + {%- endif -%}

375 + {%- else -%}

376 + {{- raise_exception("Invalid message role") -}}

377 + {%- endif -%}

378 +{%- endfor -%}

379 +{%- if ns.in_tool -%}

380 + {{ ']' }}

381 +{%- endif -%}

382 +{%- if add_generation_prompt -%}

383 + {{ assistant_token }}

384 +{%- endif -%}

\ No newline at end of file

tests/tool_parsers/test_apertus_tool_parser.py new

+433

@@ -0,0 +1,433 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

4 +import json

5 +from typing import Any

6 +from unittest.mock import MagicMock

7 +

8 +import pytest

9 +

10 +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest

11 +from vllm.tool_parsers.apertus_tool_parser import (

12 + TOOL_CALLS_PREFIX,

13 + TOOL_CALLS_SUFFIX,

14 + ApertusToolParser,

15 +)

16 +

17 +# ---------------------------------------------------------------------------

18 +# Fixtures

19 +# ---------------------------------------------------------------------------

20 +

21 +

22 +@pytest.fixture

23 +def mock_tokenizer():

24 + tokenizer = MagicMock()

25 + tokenizer.encode.return_value = [1, 2, 3]

26 + # Include the tool call tokens in the vocab for the parser

27 + tokenizer.get_vocab.return_value = {TOOL_CALLS_PREFIX: 100, TOOL_CALLS_SUFFIX: 101}

28 + return tokenizer

29 +

30 +

31 +@pytest.fixture

32 +def parser(mock_tokenizer):

33 + return ApertusToolParser(mock_tokenizer)

34 +

35 +

36 +@pytest.fixture

37 +def mock_request():

38 + request = MagicMock(spec=ChatCompletionRequest)

39 + request.tools = []

40 + request.tool_choice = "auto"

41 + return request

42 +

43 +

44 +# ---------------------------------------------------------------------------

45 +# Non-streaming extraction tests

46 +# ---------------------------------------------------------------------------

47 +

48 +

49 +class TestExtractToolCalls:

50 + def test_no_tool_calls(self, parser, mock_request):

51 + model_output = "Hello, how can I help you today?"

52 + result = parser.extract_tool_calls(model_output, mock_request)

53 +

54 + assert result.tools_called is False

55 + assert result.tool_calls == []

56 + assert result.content == model_output

57 +

58 + def test_single_tool_call(self, parser, mock_request):

59 + model_output = (

60 + '<|tools_prefix|>[{"get_weather": {"location": "London"}}]<|tools_suffix|>'

61 + )

62 + result = parser.extract_tool_calls(model_output, mock_request)

63 +

64 + assert result.tools_called is True

65 + assert len(result.tool_calls) == 1

66 + assert result.tool_calls[0].function.name == "get_weather"

67 + args = json.loads(result.tool_calls[0].function.arguments)

68 + assert args == {"location": "London"}

69 +

70 + def test_multiple_arguments(self, parser, mock_request):

71 + model_output = (

72 + '<|tools_prefix|>[{"get_weather": '

73 + '{"location": "San Francisco", '

74 + '"unit": "celsius"}}]<|tools_suffix|>'

75 + )

76 + result = parser.extract_tool_calls(model_output, mock_request)

77 +

78 + assert result.tools_called is True

79 + assert len(result.tool_calls) == 1

80 + assert result.tool_calls[0].function.name == "get_weather"

81 + args = json.loads(result.tool_calls[0].function.arguments)

82 + assert args == {"location": "San Francisco", "unit": "celsius"}

83 +

84 + def test_text_before_tool_call(self, parser, mock_request):

85 + model_output = (

86 + "Let me check the weather for you. "

87 + '<|tools_prefix|>[{"get_weather": {"location": "Paris"}}]<|tools_suffix|>'

88 + )

89 + result = parser.extract_tool_calls(model_output, mock_request)

90 +

91 + assert result.tools_called is True

92 + assert result.content == "Let me check the weather for you."

93 + assert len(result.tool_calls) == 1

94 + assert result.tool_calls[0].function.name == "get_weather"

95 +

96 + def test_multiple_tool_calls(self, parser, mock_request):

97 + model_output = (

98 + '<|tools_prefix|>[{"get_weather": '

99 + '{"location": "London"}}, '

100 + '{"get_time": {"location": "London"}}]<|tools_suffix|>'

101 + )

102 + result = parser.extract_tool_calls(model_output, mock_request)

103 +

104 + assert result.tools_called is True

105 + assert len(result.tool_calls) == 2

106 + assert result.tool_calls[0].function.name == "get_weather"

107 + assert result.tool_calls[1].function.name == "get_time"

108 +

109 + def test_nested_arguments(self, parser, mock_request):

110 + model_output = (

111 + '<|tools_prefix|>[{"complex_function": '

112 + '{"nested": {"inner": "value"}, '

113 + '"list": ["a", "b"]}}]<|tools_suffix|>'

114 + )

115 + result = parser.extract_tool_calls(model_output, mock_request)

116 +

117 + assert result.tools_called is True

118 + assert len(result.tool_calls) == 1

119 + assert result.tool_calls[0].function.name == "complex_function"

120 + args = json.loads(result.tool_calls[0].function.arguments)

121 + assert args == {"nested": {"inner": "value"}, "list": ["a", "b"]}

122 +

123 + def test_incomplete_tool_call(self, parser, mock_request):

124 + model_output = '<|tools_prefix|>[{"get_weather": {"location": "London"}'

125 + result = parser.extract_tool_calls(model_output, mock_request)

126 +

127 + assert result.tools_called is True

128 + assert len(result.tool_calls) == 1

129 + assert result.tool_calls[0].function.name == "get_weather"

130 + args = json.loads(result.tool_calls[0].function.arguments)

131 + assert args == {"location": "London"}

132 +

133 + def test_missing_tool_suffix(self, parser, mock_request):

134 + model_output = (

135 + '<|tools_prefix|>[{"get_weather": '

136 + '{"location": "San Francisco", "unit": "celsius"}}]'

137 + )

138 + result = parser.extract_tool_calls(model_output, mock_request)

139 +

140 + assert result.tools_called is True

141 + assert len(result.tool_calls) == 1

142 + assert result.tool_calls[0].function.name == "get_weather"

143 + args = json.loads(result.tool_calls[0].function.arguments)

144 + assert args == {"location": "San Francisco", "unit": "celsius"}

145 +

146 +

147 +# ---------------------------------------------------------------------------

148 +# Streaming extraction tests

149 +# ---------------------------------------------------------------------------

150 +

151 +

152 +class TestStreamingExtraction:

153 + def _simulate_streaming(

154 + self, parser: ApertusToolParser, mock_request: Any, chunks: list[str]

155 + ) -> list[tuple[Any, str]]:

156 + results: list[tuple[Any, str]] = []

157 + previous_text: str = ""

158 + previous_token_ids: list[int] = []

159 +

160 + for chunk in chunks:

161 + current_text = previous_text + chunk

162 + # Simulate a token ID sequence matching the chunk progression

163 + delta_token_ids: list[int] = [0] * max(1, len(chunk) // 4)

164 + current_token_ids = previous_token_ids + delta_token_ids

165 +

166 + delta = parser.extract_tool_calls_streaming(

167 + previous_text=previous_text,

168 + current_text=current_text,

169 + delta_text=chunk,

170 + previous_token_ids=tuple(previous_token_ids),

171 + current_token_ids=tuple(current_token_ids),

172 + delta_token_ids=tuple(delta_token_ids),

173 + request=mock_request,

174 + )

175 + results.append((delta, current_text))

176 + previous_text = current_text

177 + previous_token_ids = list(current_token_ids)

178 +

179 + return results

180 +

181 + def _collect_tool_calls(self, results) -> dict[int, dict[str, Any]]:

182 + """Properly tracks and concatenates streamed tool arguments by their Index."""

183 + tool_calls = {}

184 + for delta, _ in results:

185 + if not delta or not getattr(delta, "tool_calls", None):

186 + continue

187 +

188 + for tc in delta.tool_calls:

189 + idx = (

190 + tc.get("index", 0)

191 + if isinstance(tc, dict)

192 + else getattr(tc, "index", 0)

193 + )

194 + func = (

195 + tc.get("function", {})

196 + if isinstance(tc, dict)

197 + else getattr(tc, "function", None)

198 + )

199 + if not func:

200 + continue

201 +

202 + name = (

203 + func.get("name")

204 + if isinstance(func, dict)

205 + else getattr(func, "name", None)

206 + )

207 + args = (

208 + func.get("arguments")

209 + if isinstance(func, dict)

210 + else getattr(func, "arguments", None)

211 + )

212 +

213 + if idx not in tool_calls:

214 + tool_calls[idx] = {"name": "", "arguments": ""}

215 +

216 + if name:

217 + tool_calls[idx]["name"] += name

218 + if args:

219 + tool_calls[idx]["arguments"] += args

220 +

221 + return tool_calls

222 +

223 + def _collect_content(self, results) -> str:

224 + """Collects generated normal text outside of the tool calls."""

225 + return "".join(

226 + delta.content

227 + for delta, _ in results

228 + if delta and getattr(delta, "content", None)

229 + )

230 +

231 + def test_basic_streaming_single_tool(self, parser, mock_request):

232 + chunks = [

233 + "<|tools_prefix|>",

234 + '[{"get_weather": ',

235 + '{"location": "Paris, ',

236 + 'France"}}]',

237 + "<|tools_suffix|>",

238 + ]

239 +

240 + results = self._simulate_streaming(parser, mock_request, chunks)

241 + tcs = self._collect_tool_calls(results)

242 +

243 + assert len(tcs) == 1

244 + assert tcs[0]["name"] == "get_weather"

245 + assert json.loads(tcs[0]["arguments"]) == {"location": "Paris, France"}

246 +

247 + def test_streaming_missing_tool_suffix(self, parser, mock_request):

248 + chunks = [

249 + "<|tools_prefix|>",

250 + '[{"get_weather": ',

251 + '{"location": "Paris, ',

252 + 'France"}}]',

253 + ]

254 +

255 + results = self._simulate_streaming(parser, mock_request, chunks)

256 + tcs = self._collect_tool_calls(results)

257 +

258 + assert len(tcs) == 1

259 + assert tcs[0]["name"] == "get_weather"

260 + assert json.loads(tcs[0]["arguments"]) == {"location": "Paris, France"}

261 +

262 + def test_streaming_partial_tag_buffering_missing_tool_suffix(

263 + self, parser, mock_request

264 + ):

265 + chunks = ["Content", "<|tools_", "prefix|>", '[{"f": ', '{"a": 1}}]']

266 +

267 + results = self._simulate_streaming(parser, mock_request, chunks)

268 + content = self._collect_content(results)

269 +

270 + assert "Content" in content

271 + assert "<|tools_prefix|>" not in content

272 + assert "<|tools_suffix|>" not in content

273 +

274 + tcs = self._collect_tool_calls(results)

275 +

276 + assert len(tcs) == 1

277 + assert tcs[0]["name"] == "f"

278 + assert json.loads(tcs[0]["arguments"]) == {"a": 1}

279 +

280 + def test_streaming_multi_tool(self, parser, mock_request):

281 + chunks = [

282 + "<|tools_prefix|>",

283 + '[{"get_weather": {"location": "Tokyo"}}',

284 + ', {"get_time": {"location": "Tokyo"}}]',

285 + "<|tools_suffix|>",

286 + ]

287 +

288 + results = self._simulate_streaming(parser, mock_request, chunks)

289 + tcs = self._collect_tool_calls(results)

290 +

291 + assert len(tcs) == 2

292 + assert tcs[0]["name"] == "get_weather"

293 + assert json.loads(tcs[0]["arguments"]) == {"location": "Tokyo"}

294 + assert tcs[1]["name"] == "get_time"

295 + assert json.loads(tcs[1]["arguments"]) == {"location": "Tokyo"}

296 +

297 + def test_streaming_text_before_tool_call(self, parser, mock_request):

298 + chunks = [

299 + "Let me check ",

300 + "the weather. ",

301 + "<|tools_prefix|>",

302 + '[{"get_weather": {"location": "London"}}]',

303 + "<|tools_suffix|>",

304 + ]

305 +

306 + results = self._simulate_streaming(parser, mock_request, chunks)

307 + content = self._collect_content(results)

308 +

309 + assert content.strip() == "Let me check the weather."

310 + tcs = self._collect_tool_calls(results)

311 +

312 + assert len(tcs) == 1

313 + assert tcs[0]["name"] == "get_weather"

314 + assert json.loads(tcs[0]["arguments"]) == {"location": "London"}

315 +

316 + def test_streaming_partial_tag_buffering(self, parser, mock_request):

317 + chunks = [

318 + "Content",

319 + "<|tools_",

320 + "prefix|>",

321 + '[{"f": {"a": 1}}]',

322 + "<|tools_suf",

323 + "fix|>",

324 + ]

325 +

326 + results = self._simulate_streaming(parser, mock_request, chunks)

327 + content = self._collect_content(results)

328 +

329 + assert "Content" in content

330 + assert "<|tools_prefix|>" not in content

331 + assert "<|tools_suffix|>" not in content

332 +

333 + tc = self._collect_tool_calls(results)

334 + assert len(tc) == 1

335 + assert tc[0]["name"] == "f"

336 + assert json.loads(tc[0]["arguments"]) == {"a": 1}

337 +

338 + # ---------------------------------------------------------------------------

339 + # Edge Cases: Multi-Token Prediction (MTP) & vLLM Chunking Anomalies

340 + # ---------------------------------------------------------------------------

341 +

342 + def test_mtp_streaming_massive_chunk(self, parser, mock_request):

343 + """Simulates MTP predicting text, tool calls,

344 + and trailing text all in a single chunk."""

345 + chunks = [

346 + "Sure! "

347 + '<|tools_prefix|>[{"get_weather": {"location": "London"}}]<|tools_suffix|>'

348 + ]

349 + results = self._simulate_streaming(parser, mock_request, chunks)

350 +

351 + content = self._collect_content(results)

352 + assert "Sure! " in content

353 +

354 + tc = self._collect_tool_calls(results)

355 + assert len(tc) == 1

356 + assert tc[0]["name"] == "get_weather"

357 + assert json.loads(tc[0]["arguments"]) == {"location": "London"}

358 +

359 + def test_mtp_streaming_multiple_tools_burst(self, parser, mock_request):

360 + """Simulates MTP predicting an array of multiple tools in one single chunk."""

361 + chunks = [

362 + '<|tools_prefix|>[{"get_weather": '

363 + '{"location": "London"}}, '

364 + '{"get_time": {"location": "Paris"}}]<|tools_suffix|>'

365 + ]

366 + results = self._simulate_streaming(parser, mock_request, chunks)

367 +

368 + tc = self._collect_tool_calls(results)

369 + assert len(tc) == 2

370 + assert tc[0]["name"] == "get_weather"

371 + assert json.loads(tc[0]["arguments"]) == {"location": "London"}

372 + assert tc[1]["name"] == "get_time"

373 + assert json.loads(tc[1]["arguments"]) == {"location": "Paris"}

374 +

375 + def test_mtp_streaming_skip_and_catch_up(self, parser, mock_request):

376 + """Simulates MTP chunks that jump over entire tools

377 + (e.g., from middle of tool 1 to middle of tool 3)."""

378 + chunks = [

379 + '<|tools_prefix|>[{"t1": {"a": 1}',

380 + '}, {"t2": {"b": 2}}, {"t3": {"c": 3',

381 + "}}]<|tools_suffix|>",

382 + ]

383 + results = self._simulate_streaming(parser, mock_request, chunks)

384 +

385 + tc = self._collect_tool_calls(results)

386 + assert len(tc) == 3

387 + assert tc[0]["name"] == "t1"

388 + assert json.loads(tc[0]["arguments"]) == {"a": 1}

389 + assert tc[1]["name"] == "t2"

390 + assert json.loads(tc[1]["arguments"]) == {"b": 2}

391 + assert tc[2]["name"] == "t3"

392 + assert json.loads(tc[2]["arguments"]) == {"c": 3}

393 +

394 + def test_vllm_streaming_character_by_character(self, parser, mock_request):

395 + """Simulates worst-case vLLM fragmentation where

396 + chunks arrive character-by-character."""

397 + text = (

398 + 'Hi <|tools_prefix|>[{"get_weather": '

399 + '{"location": "London"}}]<|tools_suffix|> '

400 + )

401 + chunks = list(text)

402 + results = self._simulate_streaming(parser, mock_request, chunks)

403 +

404 + content = self._collect_content(results)

405 + assert "Hi" in content

406 +

407 + tc = self._collect_tool_calls(results)

408 + assert len(tc) == 1

409 + assert tc[0]["name"] == "get_weather"

410 + assert json.loads(tc[0]["arguments"]) == {"location": "London"}

411 +

412 + def test_vllm_streaming_empty_deltas(self, parser, mock_request):

413 + """Simulates vLLM stream producing empty string chunks

414 + (e.g., hidden tokens or artifacts)."""

415 + chunks = [

416 + "Wait",

417 + "",

418 + "<|tools_prefix|>",

419 + "",

420 + '[{"get_weather": ',

421 + "",

422 + '{"location": "London"}}]',

423 + "<|tools_suffix|>",

424 + ]

425 + results = self._simulate_streaming(parser, mock_request, chunks)

426 +

427 + content = self._collect_content(results)

428 + assert content == "Wait"

429 +

430 + tc = self._collect_tool_calls(results)

431 + assert len(tc) == 1

432 + assert tc[0]["name"] == "get_weather"

433 + assert json.loads(tc[0]["arguments"]) == {"location": "London"}

vllm/tool_parsers/__init__.py

@@ -186,6 +186,10 @@ _TOOL_PARSERS_TO_REGISTER = {

186 "gemma4_tool_parser",

187 "Gemma4ToolParser",

188 ),

189 + "apertus": (

190 + "apertus_tool_parser",

191 + "ApertusToolParser",

192 + ),

193 }

194

195

vllm/tool_parsers/apertus_tool_parser.py new

+553

@@ -0,0 +1,553 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +"""

4 +Tool call parser for Apertus models.

5 +

6 +Extracts tool calls from the format:

7 +<|tools_prefix|>[{"function_name": {"arg1": "value1", ...}}, ...]<|tools_suffix|>

8 +

9 +Used when --enable-auto-tool-choice --tool-call-parser apertus are set.

10 +"""

11 +

12 +import json

13 +from collections.abc import Sequence

14 +

15 +import regex as re

16 +from partial_json_parser.core.options import Allow

17 +

18 +from vllm.entrypoints.chat_utils import make_tool_call_id

19 +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest

20 +from vllm.entrypoints.openai.engine.protocol import (

21 + DeltaFunctionCall,

22 + DeltaMessage,

23 + DeltaToolCall,

24 + ExtractedToolCallInformation,

25 + FunctionCall,

26 + ToolCall,

27 +)

28 +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest

29 +from vllm.logger import init_logger

30 +from vllm.tokenizers import TokenizerLike

31 +from vllm.tool_parsers.abstract_tool_parser import (

32 + Tool,

33 + ToolParser,

34 +)

35 +from vllm.tool_parsers.utils import (

36 + find_common_prefix,

37 + partial_json_loads,

38 +)

39 +

40 +logger = init_logger(__name__)

41 +

42 +# Apertus special tokens for tool calls

43 +TOOL_CALLS_PREFIX = "<|tools_prefix|>"

44 +TOOL_CALLS_SUFFIX = "<|tools_suffix|>"

45 +

46 +

47 +class ApertusToolParser(ToolParser):

48 + """

49 + Tool call parser for Apertus models.

50 +

51 + Handles the extraction of tool calls from text in both non-streaming

52 + (complete string) and streaming (chunked token) environments.

53 +

54 + The expected Apertus function call format is a JSON array of single-key dictionaries

55 + sandwiched between special tokens:

56 + `<|tools_prefix|>[{"function_name": {"arg1": "value1"}}, ...]<|tools_suffix|>`

57 +

58 + Examples:

59 + >>> tokenizer = ... # Mock tokenizer

60 + >>> parser = ApertusToolParser(tokenizer)

61 + >>> output = 'I will check. <|tools_prefix|>[{"get_weather": '\

62 + '{"city": "Paris"}}]<|tools_suffix|>'

63 + >>> request = ChatCompletionRequest(...)

64 + >>> info = parser.extract_tool_calls(output, request)

65 + >>> info.content

66 + "I will check."

67 + >>> info.tool_calls[0].function.name

68 + "get_weather"

69 + >>> info.tool_calls[0].function.arguments

70 + '{"city": "Paris"}'

71 + """

72 +

73 + def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):

74 + """

75 + Initializes the ApertusToolParser.

76 +

77 + Args:

78 + tokenizer: The model's tokenizer.

79 + Must be provided to interact with special tokens.

80 + tools: Optional list of tools available for the current request.

81 +

82 + Raises:

83 + ValueError: If the `model_tokenizer`

84 + is not successfully passed to the base class.

85 + """

86 + super().__init__(tokenizer, tools)

87 +

88 + if not self.model_tokenizer:

89 + raise ValueError(

90 + "The model tokenizer must be passed to the ToolParser "

91 + "constructor during construction."

92 + )

93 + # Regex to extract tool calls block (suffix is optional for incomplete outputs)

94 + self.tool_call_regex = re.compile(

95 + rf"{re.escape(TOOL_CALLS_PREFIX)}"

96 + rf"(.*?)"

97 + rf"(?:{re.escape(TOOL_CALLS_SUFFIX)}|$)",

98 + re.DOTALL,

99 + )

100 +

101 + self._reset_streaming_state()

102 +

103 + def _reset_streaming_state(self) -> None:

104 + """

105 + Resets all streaming state variables for a new completion request.

106 +

107 + This clears the delta text buffer and resets the pointers used to

108 + track the currently streaming tool index and arguments. Called implicitly

109 + during initialization and should be called between separate streams.

110 + """

111 + self.buffered_delta_text = ""

112 + self.current_tool_id = -1

113 + self.current_tool_name_sent = False

114 + self.streamed_args_for_tool: list[str] = []

115 +

116 + def adjust_request(

117 + self, request: ChatCompletionRequest | ResponsesRequest

118 + ) -> ChatCompletionRequest | ResponsesRequest:

119 + """

120 + Adjusts the generation request to ensure special tool tokens are not skipped.

121 +

122 + Forces `skip_special_tokens=False` if tools are actively being evaluated,

123 + ensuring the tools special tokens are surfaced to the engine for parsing.

124 +

125 + Args:

126 + request: The incoming OpenAI-compatible chat completion request.

127 +

128 + Returns:

129 + The potentially modified chat completion request.

130 + """

131 + request = super().adjust_request(request)

132 + if request.tools and request.tool_choice != "none":

133 + request.skip_special_tokens = False

134 + return request

135 +

136 + def _buffer_delta_text(self, delta_text: str) -> str:

137 + """

138 + Buffers incoming delta chunks to prevent

139 + fragmentation of multi-token special tags.

140 +

141 + If a chunk ends with a partial match of

142 + `<|tools_prefix|>` or `<|tools_suffix|>`,

143 + it holds that part back until the next chunk clarifies if it's the actual tag

144 + or just normal text.

145 +

146 + Args:

147 + delta_text: The newly generated text chunk

148 +

149 + Returns:

150 + The safe, verified text chunk free of partial tag collisions.

151 +

152 + Examples:

153 + >>> parser = ApertusToolParser(...)

154 + >>> parser._buffer_delta_text("Let me check <|tool" \

155 + "Let me check " # "<|tool" is buffered internally

156 + >>> parser._buffer_delta_text("s_prefix|>" \

157 + "<|tools_prefix|>" # Buffer released on completion

158 + """

159 + self.buffered_delta_text += delta_text

160 + text = self.buffered_delta_text

161 +

162 + for tag in (TOOL_CALLS_PREFIX, TOOL_CALLS_SUFFIX):

163 + if text.endswith(tag):

164 + self.buffered_delta_text = ""

165 + return text

166 +

167 + # Evaluate longest possible partial match first

168 + for i in range(len(tag) - 1, 0, -1):

169 + if text.endswith(tag[:i]):

170 + self.buffered_delta_text = text[-i:]

171 + return text[:-i]

172 +

173 + self.buffered_delta_text = ""

174 + return text

175 +

176 + def extract_tool_calls(

177 + self,

178 + model_output: str,

179 + request: ChatCompletionRequest,

180 + ) -> ExtractedToolCallInformation:

181 + """

182 + Extracts tool calls from a completely generated model response (Non-Streaming).

183 +

184 + Args:

185 + model_output: The full completion string generated by the model.

186 + request: The current chat completion

187 + request context containing tool schemas.

188 +

189 + Returns:

190 + An `ExtractedToolCallInformation` object containing normal text content

191 + and a list of fully formatted `ToolCall` objects.

192 +

193 + Examples:

194 + >>> output = 'Let me see. <|tools_prefix|>[{"get_weather":' \

195 + '{"loc": "Paris"}}]<|tools_suffix|>'

196 + >>> info = parser.extract_tool_calls(output, request)

197 + >>> info.tools_called

198 + True

199 + >>> info.content

200 + 'Let me see.'

201 + >>> info.tool_calls[0].function.name

202 + 'get_weather'

203 + """

204 + match = self.tool_call_regex.search(model_output)

205 + if not match:

206 + return ExtractedToolCallInformation(

207 + tools_called=False, tool_calls=[], content=model_output

208 + )

209 +

210 + try:

211 + # group(1) might contain trailing text if the suffix is missing

212 + matched_text = match.group(1)

213 + stripped_text = matched_text.lstrip()

214 +

215 + try:

216 + # Use raw_decode to robustly isolate

217 + # the valid JSON array from any trailing garbage

218 + parsed_json, idx = json.JSONDecoder().raw_decode(stripped_text)

219 + trailing_in_group = stripped_text[idx:]

220 + except json.JSONDecodeError:

221 + # Fallback sequentially to partial parser for token-truncated requests

222 + parsed_json, _ = partial_json_loads(matched_text, Allow.ALL)

223 + trailing_in_group = ""

224 +

225 + if not isinstance(parsed_json, list):

226 + parsed_json = [parsed_json] if parsed_json else []

227 +

228 + tool_calls: list[ToolCall] = []

229 + for obj in parsed_json:

230 + if isinstance(obj, dict) and obj:

231 + name, args = next(iter(obj.items()))

232 + tool_calls.append(

233 + ToolCall(

234 + type="function",

235 + id=make_tool_call_id(),

236 + function=FunctionCall(

237 + name=name,

238 + arguments=json.dumps(args, ensure_ascii=False),

239 + ),

240 + )

241 + )

242 +

243 + # Content combines any generated text

244 + # prior to and safely after the tool block

245 + content_str = model_output[: match.start()].strip()

246 +

247 + # Surface any hallucinated text inside

248 + # the regex group (due to missing suffix)

249 + if trailing_in_group.strip():

250 + trailing = trailing_in_group.replace(TOOL_CALLS_SUFFIX, "").strip()

251 + if trailing:

252 + content_str = (content_str + "\n" + trailing).strip()

253 +

254 + # Surface text natively generated after the explicit suffix

255 + after_suffix = (

256 + model_output[match.end() :].replace(TOOL_CALLS_SUFFIX, "").strip()

257 + )

258 + if after_suffix:

259 + content_str = (content_str + "\n" + after_suffix).strip()

260 +

261 + return ExtractedToolCallInformation(

262 + tools_called=True,

263 + tool_calls=tool_calls,

264 + content=content_str if content_str else None,

265 + )

266 +

267 + except Exception:

268 + logger.exception("Error extracting tool calls from Apertus response")

269 + return ExtractedToolCallInformation(

270 + tools_called=False, tool_calls=[], content=model_output

271 + )

272 +

273 + def extract_tool_calls_streaming(

274 + self,

275 + previous_text: str,

276 + current_text: str,

277 + delta_text: str,

278 + previous_token_ids: Sequence[int],

279 + current_token_ids: Sequence[int],

280 + delta_token_ids: Sequence[int],

281 + request: ChatCompletionRequest,

282 + ) -> DeltaMessage | None:

283 + """

284 + Handles streaming chunks

285 +

286 + Args:

287 + previous_text: The complete model text generated prior to this chunk.

288 + current_text: The complete model text including this chunk.

289 + delta_text: The incremental text addition.

290 + previous_token_ids: Tokens generated prior to this chunk.

291 + current_token_ids: Total tokens generated.

292 + delta_token_ids: Incremental token additions.

293 + request: The chat completion request.

294 +

295 + Returns:

296 + A `DeltaMessage` with updated content or tool argument diffs, or `None` if

297 + the chunk shouldn't emit visible changes yet (e.g. it was purely buffered).

298 +

299 + Examples:

300 + >>> prev = '<|tools_prefix|>[{"get_weather": {"loc'

301 + >>> cur = '<|tools_prefix|>[{"get_weather": {"location": "Paris"}}'

302 + >>> delta = 'ation": "Paris"}}'

303 + >>> msg = parser.extract_tool_calls_streaming(

304 + ... prev, cur, delta, ..., request

305 + ... )

306 + >>> msg.tool_calls[0].function.arguments

307 + 'ation": "Paris"}'

308 + """

309 + delta_text = self._buffer_delta_text(delta_text)

310 + if not delta_text:

311 + return None

312 +

313 + # Fast path: normal text generation before any tools are invoked

314 + if TOOL_CALLS_PREFIX not in current_text:

315 + return DeltaMessage(content=delta_text)

316 +

317 + try:

318 + return self._extract_streaming(current_text, delta_text)

319 + except Exception:

320 + logger.exception("Error in Apertus streaming tool call extraction")

321 + return None

322 +

323 + def _extract_streaming(

324 + self, current_text: str, delta_text: str

325 + ) -> DeltaMessage | None:

326 + """

327 + Core streaming logic.

328 + Separates visible chat text from JSON blocks and computes diffs.

329 +

330 + Args:

331 + current_text: The full generated output string so far.

332 + delta_text: The latest chunk of text added.

333 +

334 + Returns:

335 + A `DeltaMessage` containing the `content` delta and/or `tool_calls` delta.

336 + """

337 + prefix_idx = current_text.rfind(TOOL_CALLS_PREFIX)

338 + suffix_idx = current_text.rfind(TOOL_CALLS_SUFFIX)

339 +

340 + is_inside_tools = prefix_idx > suffix_idx

341 +

342 + json_completed = False

343 + json_end_idx: int | None = None

344 +

345 + # Check if the JSON array successfully closed implicitly

346 + if is_inside_tools:

347 + json_start = prefix_idx + len(TOOL_CALLS_PREFIX)

348 + s = current_text[json_start:].lstrip()

349 + try:

350 + # If raw_decode succeeds,

351 + # the JSON array is fully formed and implicitly closed

352 + _, idx = json.JSONDecoder().raw_decode(s)

353 + json_end_idx = len(current_text) - len(s) + idx

354 + json_completed, is_inside_tools = True, False

355 + except Exception:

356 + pass

357 +

358 + just_finished = (TOOL_CALLS_SUFFIX in delta_text) or json_completed

359 +

360 + # 1. Fast path: Output normal text immediately

361 + # if we are completely outside tool block constraints

362 + if not is_inside_tools and not just_finished:

363 + text = delta_text.replace(TOOL_CALLS_PREFIX, "").replace(

364 + TOOL_CALLS_SUFFIX, ""

365 + )

366 + return DeltaMessage(content=text) if text else None

367 +

368 + # 2. Extract leading and trailing normal text directly adjacent to tool blocks

369 + content_str = ""

370 + if TOOL_CALLS_PREFIX in delta_text:

371 + content_str += delta_text.split(TOOL_CALLS_PREFIX)[0].replace(

372 + TOOL_CALLS_SUFFIX, ""

373 + )

374 +

375 + if just_finished:

376 + if json_completed and json_end_idx is not None:

377 + # The tool block finished in this chunk via implicit JSON completion

378 + # Ensure we strictly isolate

379 + # and extract only trailing text that is part of `delta_text`

380 + delta_start_idx = len(current_text) - len(delta_text)

381 + content_start = max(json_end_idx, delta_start_idx)

382 + if content_start < len(current_text):

383 + content_str += current_text[content_start:].replace(

384 + TOOL_CALLS_SUFFIX, ""

385 + )

386 + else:

387 + content_str += delta_text.split(TOOL_CALLS_SUFFIX)[-1]

388 +

389 + # 3. Extract the isolated JSON array string for the active block

390 + json_start = prefix_idx + len(TOOL_CALLS_PREFIX)

391 + json_end = suffix_idx if suffix_idx > prefix_idx else json_end_idx

392 + json_str = current_text[json_start:json_end]

393 +

394 + tool_calls = self._parse_and_diff_json(json_str, is_final=not is_inside_tools)

395 +

396 + if tool_calls or content_str:

397 + return DeltaMessage(

398 + content=content_str if content_str else None,

399 + tool_calls=tool_calls if tool_calls else None,

400 + )

401 +

402 + return None

403 +

404 + def _parse_and_diff_json(

405 + self, json_str: str, is_final: bool

406 + ) -> list[DeltaToolCall]:

407 + """

408 + Parses an isolated, potentially incomplete streaming JSON array and returns

409 + newly accumulated tool call diffs.

410 +

411 + Args:

412 + json_str: The extracted JSON array string so far

413 + (e.g. `[{"weather": {"city": "Par"}]`).

414 + is_final: True if the tool block has received its closing`<|tools_suffix|>`

415 +

416 + Returns:

417 + A list of `DeltaToolCall`

418 + items representing string diffs in function arguments

419 + to stream back to the client.

420 + """

421 + try:

422 + parsed, _ = partial_json_loads(json_str, Allow.ALL)

423 + if not isinstance(parsed, list):

424 + parsed = [parsed] if parsed else []

425 + except Exception:

426 + return []

427 +

428 + if not parsed:

429 + return []

430 +

431 + tool_calls: list[DeltaToolCall] = []

432 + latest_index = len(parsed) - 1

433 +

434 + # Catch up and finalize any tools we fully skipped over in one large text delta

435 + while self.current_tool_id < latest_index:

436 + if self.current_tool_id >= 0:

437 + if not self.current_tool_name_sent:

438 + self._emit_tool_name(parsed, self.current_tool_id, tool_calls)

439 +

440 + delta = self._get_tool_diff(parsed, self.current_tool_id, is_final=True)

441 + if delta:

442 + tool_calls.append(delta)

443 +

444 + self.current_tool_id += 1

445 + self.current_tool_name_sent = False

446 + while len(self.streamed_args_for_tool) <= self.current_tool_id:

447 + self.streamed_args_for_tool.append("")

448 +

449 + # Stream the currently active tool

450 + if self.current_tool_id >= 0:

451 + if not self.current_tool_name_sent:

452 + self._emit_tool_name(parsed, self.current_tool_id, tool_calls)

453 +

454 + delta = self._get_tool_diff(parsed, self.current_tool_id, is_final)

455 + if delta:

456 + tool_calls.append(delta)

457 +

458 + return tool_calls

459 +

460 + def _emit_tool_name(

461 + self, parsed: list, index: int, tool_calls: list[DeltaToolCall]

462 + ) -> None:

463 + """

464 + Extracts and emits the function name mapped to a new tool call ID.

465 +

466 + Args:

467 + parsed: The partially parsed JSON list containing tool dictionaries.

468 + index: The active index within the JSON list.

469 + tool_calls: The running list of delta chunks to mutate.

470 +

471 + Examples:

472 + Appends `DeltaToolCall(index=0,

473 + function=DeltaFunctionCall(name="get_weather", ...))`

474 + to the `tool_calls` list and marks the name as sent.

475 + """

476 + obj = parsed[index]

477 + if isinstance(obj, dict) and obj:

478 + name = next(iter(obj))

479 + self.current_tool_name_sent = True

480 + tool_calls.append(

481 + DeltaToolCall(

482 + index=index,

483 + type="function",

484 + id=make_tool_call_id(),

485 + function=DeltaFunctionCall(name=name, arguments="").model_dump(

486 + exclude_none=True

487 + ),

488 + )

489 + )

490 +

491 + def _get_tool_diff(

492 + self, parsed: list, index: int, is_final: bool

493 + ) -> DeltaToolCall | None:

494 + """

495 + Calculates the exact string difference to safely append new tool parameters.

496 +

497 + This ensures characters like `{`, `}`, and `"` don't jump around unevenly

498 + in the UI frontend while streaming incomplete JSON arguments.

499 +

500 + Args:

501 + parsed: The latest list of parsed JSON objects.

502 + index: The active tool's array index.

503 + is_final: Whether to emit

504 + trailing structural brackets (True if block is done).

505 +

506 + Returns:

507 + A `DeltaToolCall` mapping to the arguments diff,

508 + or None if no text was appended.

509 +

510 + Examples:

511 + >>> # Previous streamed state: '{"city": "Pari'

512 + >>> # Current full parse state: '{"city": "Paris"}'

513 + >>> # Returns diff (closing bracket suppressed until final):

514 + >>> parser._get_tool_diff(parsed, index=0, is_final=False)

515 + DeltaToolCall(index=0, function=DeltaFunctionCall(arguments='s'))

516 + """

517 + obj = parsed[index]

518 + if not isinstance(obj, dict) or not obj:

519 + return None

520 +

521 + name, args = next(iter(obj.items()))

522 + if args is None:

523 + return None

524 +

525 + args_json = json.dumps(args, ensure_ascii=False)

526 +

527 + # Suppress trailing structural characters

528 + # during stream (looks cleaner in frontends)

529 + if not is_final:

530 + while args_json and args_json[-1] in ("}", '"', "]", " ", ","):

531 + args_json = args_json[:-1]

532 +

533 + prev_sent = self.streamed_args_for_tool[index]

534 + if args_json == prev_sent:

535 + return None

536 +

537 + prefix = find_common_prefix(prev_sent, args_json)

538 + if len(prefix) < len(prev_sent):

539 + # Backtrack state if partial parser structurally updates a past assumption

540 + self.streamed_args_for_tool[index] = prefix

541 + return None

542 +

543 + diff = args_json[len(prev_sent) :]

544 + if diff:

545 + self.streamed_args_for_tool[index] = args_json

546 + return DeltaToolCall(

547 + index=index,

548 + function=DeltaFunctionCall(arguments=diff).model_dump(

549 + exclude_none=True

550 + ),

551 + )

552 +

553 + return None