@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8477fe427d

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 5, 11:00 UTC

[Tool] `adjust_request` to reasoning parser, and Gemma4 fixes (#39027)

Signed-off-by: Ben Browning <bbrownin@redhat.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> Co-authored-by: Cursor <cursoragent@cursor.com>

Ben Browning committed Apr 8, 2026 at 15:04 UTC 8477fe427d174df15204aae2819922cce384ecf2

12 files changed +878 -16

examples/tool_chat_template_gemma4.jinja new

+331

@@ -0,0 +1,331 @@

1 +{%- macro format_parameters(properties, required) -%}

2 + {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%}

3 + {%- set ns = namespace(found_first=false) -%}

4 + {%- for key, value in properties | dictsort -%}

5 + {%- set add_comma = false -%}

6 + {%- if key not in standard_keys -%}

7 + {%- if ns.found_first %},{% endif -%}

8 + {%- set ns.found_first = true -%}

9 + {{ key }}:{

10 + {%- if value['description'] -%}

11 + description:<|"|>{{ value['description'] }}<|"|>

12 + {%- set add_comma = true -%}

13 + {%- endif -%}

14 + {%- if value['nullable'] %}

15 + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}

16 + nullable:true

17 + {%- endif -%}

18 + {%- if value['type'] | upper == 'STRING' -%}

19 + {%- if value['enum'] -%}

20 + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}

21 + enum:{{ format_argument(value['enum']) }}

22 + {%- endif -%}

23 + {%- elif value['type'] | upper == 'OBJECT' -%}

24 + ,properties:{

25 + {%- if value['properties'] is defined and value['properties'] is mapping -%}

26 + {{- format_parameters(value['properties'], value['required'] | default([])) -}}

27 + {%- elif value is mapping -%}

28 + {{- format_parameters(value, value['required'] | default([])) -}}

29 + {%- endif -%}

30 + }

31 + {%- if value['required'] -%}

32 + ,required:[

33 + {%- for item in value['required'] | default([]) -%}

34 + <|"|>{{- item -}}<|"|>

35 + {%- if not loop.last %},{% endif -%}

36 + {%- endfor -%}

37 + ]

38 + {%- endif -%}

39 + {%- elif value['type'] | upper == 'ARRAY' -%}

40 + {%- if value['items'] is mapping and value['items'] -%}

41 + ,items:{

42 + {%- set ns_items = namespace(found_first=false) -%}

43 + {%- for item_key, item_value in value['items'] | dictsort -%}

44 + {%- if item_value is not none -%}

45 + {%- if ns_items.found_first %},{% endif -%}

46 + {%- set ns_items.found_first = true -%}

47 + {%- if item_key == 'properties' -%}

48 + properties:{

49 + {%- if item_value is mapping -%}

50 + {{- format_parameters(item_value, value['items']['required'] | default([])) -}}

51 + {%- endif -%}

52 + }

53 + {%- elif item_key == 'required' -%}

54 + required:[

55 + {%- for req_item in item_value -%}

56 + <|"|>{{- req_item -}}<|"|>

57 + {%- if not loop.last %},{% endif -%}

58 + {%- endfor -%}

59 + ]

60 + {%- elif item_key == 'type' -%}

61 + {%- if item_value is string -%}

62 + type:{{ format_argument(item_value | upper) }}

63 + {%- else -%}

64 + type:{{ format_argument(item_value | map('upper') | list) }}

65 + {%- endif -%}

66 + {%- else -%}

67 + {{ item_key }}:{{ format_argument(item_value) }}

68 + {%- endif -%}

69 + {%- endif -%}

70 + {%- endfor -%}

71 + }

72 + {%- endif -%}

73 + {%- endif -%}

74 + {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}

75 + type:<|"|>{{ value['type'] | upper }}<|"|>}

76 + {%- endif -%}

77 + {%- endfor -%}

78 +{%- endmacro -%}

79 +{%- macro format_function_declaration(tool_data) -%}

80 + declaration:{{- tool_data['function']['name'] -}}{description:<|"|>{{- tool_data['function']['description'] -}}<|"|>

81 + {%- set params = tool_data['function']['parameters'] -%}

82 + {%- if params -%}

83 + ,parameters:{

84 + {%- if params['properties'] -%}

85 + properties:{ {{- format_parameters(params['properties'], params['required']) -}} },

86 + {%- endif -%}

87 + {%- if params['required'] -%}

88 + required:[

89 + {%- for item in params['required'] -%}

90 + <|"|>{{- item -}}<|"|>

91 + {{- ',' if not loop.last -}}

92 + {%- endfor -%}

93 + ],

94 + {%- endif -%}

95 + {%- if params['type'] -%}

96 + type:<|"|>{{- params['type'] | upper -}}<|"|>}

97 + {%- endif -%}

98 + {%- endif -%}

99 + {%- if 'response' in tool_data['function'] -%}

100 + {%- set response_declaration = tool_data['function']['response'] -%}

101 + ,response:{

102 + {%- if response_declaration['description'] -%}

103 + description:<|"|>{{- response_declaration['description'] -}}<|"|>,

104 + {%- endif -%}

105 + {%- if response_declaration['type'] | upper == 'OBJECT' -%}

106 + type:<|"|>{{- response_declaration['type'] | upper -}}<|"|>}

107 + {%- endif -%}

108 + {%- endif -%}

109 + }

110 +{%- endmacro -%}

111 +{%- macro format_argument(argument, escape_keys=True) -%}

112 + {%- if argument is string -%}

113 + {{- '<|"|>' + argument + '<|"|>' -}}

114 + {%- elif argument is boolean -%}

115 + {{- 'true' if argument else 'false' -}}

116 + {%- elif argument is mapping -%}

117 + {{- '{' -}}

118 + {%- set ns = namespace(found_first=false) -%}

119 + {%- for key, value in argument | dictsort -%}

120 + {%- if ns.found_first %},{% endif -%}

121 + {%- set ns.found_first = true -%}

122 + {%- if escape_keys -%}

123 + {{- '<|"|>' + key + '<|"|>' -}}

124 + {%- else -%}

125 + {{- key -}}

126 + {%- endif -%}

127 + :{{- format_argument(value, escape_keys=escape_keys) -}}

128 + {%- endfor -%}

129 + {{- '}' -}}

130 + {%- elif argument is sequence -%}

131 + {{- '[' -}}

132 + {%- for item in argument -%}

133 + {{- format_argument(item, escape_keys=escape_keys) -}}

134 + {%- if not loop.last %},{% endif -%}

135 + {%- endfor -%}

136 + {{- ']' -}}

137 + {%- else -%}

138 + {{- argument -}}

139 + {%- endif -%}

140 +{%- endmacro -%}

141 +{%- macro strip_thinking(text) -%}

142 + {%- set ns = namespace(result='') -%}

143 + {%- for part in text.split('<channel|>') -%}

144 + {%- if '<|channel>' in part -%}

145 + {%- set ns.result = ns.result + part.split('<|channel>')[0] -%}

146 + {%- else -%}

147 + {%- set ns.result = ns.result + part -%}

148 + {%- endif -%}

149 + {%- endfor -%}

150 + {{- ns.result | trim -}}

151 +{%- endmacro -%}

152 +

153 +{%- macro format_tool_response_block(tool_name, response) -%}

154 + {{- '<|tool_response>' -}}

155 + {%- if response is mapping -%}

156 + {{- 'response:' + tool_name + '{' -}}

157 + {%- for key, value in response | dictsort -%}

158 + {{- key -}}:{{- format_argument(value, escape_keys=False) -}}

159 + {%- if not loop.last %},{% endif -%}

160 + {%- endfor -%}

161 + {{- '}' -}}

162 + {%- else -%}

163 + {{- 'response:' + tool_name + '{value:' + format_argument(response, escape_keys=False) + '}' -}}

164 + {%- endif -%}

165 + {{- '<tool_response|>' -}}

166 +{%- endmacro -%}

167 +

168 +{%- set ns = namespace(prev_message_type=None) -%}

169 +{%- set loop_messages = messages -%}

170 +{{ bos_token }}

171 +{%- if (enable_thinking is defined and enable_thinking) or tools or messages[0]['role'] in ['system', 'developer'] -%}

172 + {{- '<|turn>system\n' -}}

173 +

174 + {%- if enable_thinking is defined and enable_thinking -%}

175 + {{- '<|think|>' -}}

176 + {%- set ns.prev_message_type = 'think' -%}

177 + {%- endif -%}

178 +

179 + {%- if messages[0]['role'] in ['system', 'developer'] -%}

180 + {{- messages[0]['content'] | trim -}}

181 + {%- set loop_messages = messages[1:] -%}

182 + {%- endif -%}

183 +

184 + {%- if tools -%}

185 + {%- for tool in tools %}

186 + {{- '<|tool>' -}}

187 + {{- format_function_declaration(tool) | trim -}}

188 + {{- '<tool|>' -}}

189 + {%- endfor %}

190 + {%- set ns.prev_message_type = 'tool' -%}

191 + {%- endif -%}

192 +

193 + {{- '<turn|>\n' -}}

194 +{%- endif %}

195 +

196 +{%- set ns_turn = namespace(last_user_idx=-1) -%}

197 +{%- for i in range(loop_messages | length) -%}

198 + {%- if loop_messages[i]['role'] == 'user' -%}

199 + {%- set ns_turn.last_user_idx = i -%}

200 + {%- endif -%}

201 +{%- endfor -%}

202 +

203 +{%- for message in loop_messages -%}

204 + {%- if message['role'] != 'tool' -%}

205 + {%- set ns.prev_message_type = None -%}

206 + {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%}

207 + {#- OpenAI may emit multiple assistant messages in one tool loop (user → asst → tool → asst → tool).

208 + Only the first of those should open <|turn>model; later ones continue the same model turn. -#}

209 + {%- set prev_nt = namespace(role=None, found=false) -%}

210 + {%- if loop.index0 > 0 -%}

211 + {%- for j in range(loop.index0 - 1, -1, -1) -%}

212 + {%- if not prev_nt.found -%}

213 + {%- if loop_messages[j]['role'] != 'tool' -%}

214 + {%- set prev_nt.role = loop_messages[j]['role'] -%}

215 + {%- set prev_nt.found = true -%}

216 + {%- endif -%}

217 + {%- endif -%}

218 + {%- endfor -%}

219 + {%- endif -%}

220 + {%- set continue_same_model_turn = (role == 'model' and prev_nt.role == 'assistant') -%}

221 + {%- if not continue_same_model_turn -%}

222 + {{- '<|turn>' + role + '\n' }}

223 + {%- endif -%}

224 +

225 + {%- if message.get('reasoning') and loop.index0 > ns_turn.last_user_idx and message.get('tool_calls') -%}

226 + {{- '<|channel>thought\n' + message['reasoning'] + '\n<channel|>'}}

227 + {%- endif -%}

228 +

229 + {%- if message['tool_calls'] -%}

230 + {%- for tool_call in message['tool_calls'] -%}

231 + {%- set function = tool_call['function'] -%}

232 + {{- '<|tool_call>call:' + function['name'] + '{' -}}

233 + {%- if function['arguments'] is mapping -%}

234 + {%- set ns_args = namespace(found_first=false) -%}

235 + {%- for key, value in function['arguments'] | dictsort -%}

236 + {%- if ns_args.found_first %},{% endif -%}

237 + {%- set ns_args.found_first = true -%}

238 + {{- key -}}:{{- format_argument(value, escape_keys=False) -}}

239 + {%- endfor -%}

240 + {%- elif function['arguments'] is string -%}

241 + {{- function['arguments'] -}}

242 + {%- endif -%}

243 + {{- '}<tool_call|>' -}}

244 + {%- endfor -%}

245 + {%- set ns.prev_message_type = 'tool_call' -%}

246 + {%- endif -%}

247 +

248 + {%- set ns_tr_out = namespace(flag=false) -%}

249 + {%- if message.get('tool_responses') -%}

250 + {#- Legacy: tool_responses embedded on the assistant message -#}

251 + {%- for tool_response in message['tool_responses'] -%}

252 + {{- format_tool_response_block(tool_response['name'] | default('unknown'), tool_response['response']) -}}

253 + {%- set ns_tr_out.flag = true -%}

254 + {%- set ns.prev_message_type = 'tool_response' -%}

255 + {%- endfor -%}

256 + {%- elif message.get('tool_calls') -%}

257 + {#- OpenAI Chat Completions: consecutive following messages with role "tool" (no break/continue; range scan) -#}

258 + {%- set ns_tool_scan = namespace(stopped=false) -%}

259 + {%- for k in range(loop.index0 + 1, loop_messages | length) -%}

260 + {%- if ns_tool_scan.stopped -%}

261 + {%- elif loop_messages[k]['role'] != 'tool' -%}

262 + {%- set ns_tool_scan.stopped = true -%}

263 + {%- else -%}

264 + {%- set follow = loop_messages[k] -%}

265 + {%- set ns_tname = namespace(name=follow.get('name') | default('unknown')) -%}

266 + {%- for tc in message['tool_calls'] -%}

267 + {%- if tc.get('id') == follow.get('tool_call_id') -%}

268 + {%- set ns_tname.name = tc['function']['name'] -%}

269 + {%- endif -%}

270 + {%- endfor -%}

271 + {%- set tool_body = follow.get('content') -%}

272 + {%- if tool_body is string -%}

273 + {{- format_tool_response_block(ns_tname.name, tool_body) -}}

274 + {%- elif tool_body is sequence and tool_body is not string -%}

275 + {%- set ns_txt = namespace(s='') -%}

276 + {%- for part in tool_body -%}

277 + {%- if part.get('type') == 'text' -%}

278 + {%- set ns_txt.s = ns_txt.s + (part.get('text') | default('')) -%}

279 + {%- endif -%}

280 + {%- endfor -%}

281 + {{- format_tool_response_block(ns_tname.name, ns_txt.s) -}}

282 + {%- else -%}

283 + {{- format_tool_response_block(ns_tname.name, tool_body) -}}

284 + {%- endif -%}

285 + {%- set ns_tr_out.flag = true -%}

286 + {%- set ns.prev_message_type = 'tool_response' -%}

287 + {%- endif -%}

288 + {%- endfor -%}

289 + {%- endif -%}

290 +

291 + {%- if message['content'] is string -%}

292 + {%- if role == 'model' -%}

293 + {{- strip_thinking(message['content']) -}}

294 + {%- else -%}

295 + {{- message['content'] | trim -}}

296 + {%- endif -%}

297 + {%- elif message['content'] is sequence -%}

298 + {%- for item in message['content'] -%}

299 + {%- if item['type'] == 'text' -%}

300 + {%- if role == 'model' -%}

301 + {{- strip_thinking(item['text']) -}}

302 + {%- else -%}

303 + {{- item['text'] | trim -}}

304 + {%- endif -%}

305 + {%- elif item['type'] == 'image' -%}

306 + {{- '\n\n<|image|>\n\n' -}}

307 + {%- set ns.prev_message_type = 'image' -%}

308 + {%- elif item['type'] == 'audio' -%}

309 + {{- '<|audio|>' -}}

310 + {%- set ns.prev_message_type = 'audio' -%}

311 + {%- elif item['type'] == 'video' -%}

312 + {{- '\n\n<|video|>\n\n' -}}

313 + {%- set ns.prev_message_type = 'video' -%}

314 + {%- endif -%}

315 + {%- endfor -%}

316 + {%- endif -%}

317 +

318 + {%- if not (ns_tr_out.flag and not message.get('content')) -%}

319 + {{- '<turn|>\n' -}}

320 + {%- endif -%}

321 + {%- endif -%}

322 +{%- endfor -%}

323 +

324 +{%- if add_generation_prompt -%}

325 + {%- if ns.prev_message_type != 'tool_response' -%}

326 + {{- '<|turn>model\n' -}}

327 + {%- endif -%}

328 + {%- if not enable_thinking | default(false) -%}

329 + {{- '<|channel>thought\n<channel|>' -}}

330 + {%- endif -%}

331 +{%- endif -%}

tests/reasoning/test_gemma4_reasoning_parser.py

+87 -8

@@ -4,6 +4,9 @@

4 import pytest

6 from tests.reasoning.utils import run_reasoning_extraction

7 +from vllm.entrypoints.openai.chat_completion.protocol import (

8 + ChatCompletionRequest,

9 +)

10 from vllm.reasoning import ReasoningParser, ReasoningParserManager

12 # Using mistral tokenizer as a generic mock since the actual model is not on HF

@@ -100,6 +103,39 @@ NEW_LINE_STREAMING = {

103 "is_reasoning_end": True,

104 }

105

106 +THOUGHT_PREFIX = {

107 + "output": "<|channel>thought\nActual reasoning here<channel|>Final answer",

108 + "reasoning": "Actual reasoning here",

109 + "content": "Final answer",

110 + "is_reasoning_end": True,

111 +}

112 +THOUGHT_PREFIX_ONLY = {

113 + "output": "<|channel>thought\n<channel|>",

114 + "reasoning": "",

115 + "content": None,

116 + "is_reasoning_end": True,

117 +}

118 +THOUGHT_PREFIX_MULTILINE = {

119 + "output": "<|channel>thought\nLine1\nLine2<channel|>Answer",

120 + "reasoning": "Line1\nLine2",

121 + "content": "Answer",

122 + "is_reasoning_end": True,

123 +}

124 +# "thousand" starts like "thought" but diverges — exercises Case 2→3 in streaming.

125 +THOUGHT_PREFIX_DIVERGE = {

126 + "output": "<|channel>thousand reasons<channel|>Done",

127 + "reasoning": "thousand reasons",

128 + "content": "Done",

129 + "is_reasoning_end": True,

130 +}

131 +# The model isn't reasoning if we're generating tool calls.

132 +TOOL_CALL_STARTED = {

133 + "output": "<|tool_call>",

134 + "reasoning": None,

135 + "content": "<|tool_call>",

136 + "is_reasoning_end": True,

137 +}

138 +

139 TEST_CASES = [

140 pytest.param(False, INVALID_SIMPLE_NONSTREAMING, id="invalid_simple"),

141 pytest.param(True, INVALID_SIMPLE_STREAMING, id="invalid_simple_streaming"),

@@ -120,17 +156,22 @@ TEST_CASES = [

156 pytest.param(False, EMPTY, id="empty"),

157 pytest.param(False, NEW_LINE_NONSTREAMING, id="new_line"),

158 pytest.param(True, NEW_LINE_STREAMING, id="new_line_streaming"),

159 + pytest.param(False, THOUGHT_PREFIX, id="thought_prefix"),

160 + pytest.param(True, THOUGHT_PREFIX, id="thought_prefix_streaming"),

161 + pytest.param(False, THOUGHT_PREFIX_ONLY, id="thought_prefix_only"),

162 + pytest.param(True, THOUGHT_PREFIX_ONLY, id="thought_prefix_only_streaming"),

163 + pytest.param(False, THOUGHT_PREFIX_MULTILINE, id="thought_prefix_multiline"),

164 + pytest.param(

165 + True, THOUGHT_PREFIX_MULTILINE, id="thought_prefix_multiline_streaming"

166 + ),

167 + pytest.param(False, THOUGHT_PREFIX_DIVERGE, id="thought_prefix_diverge"),

168 + pytest.param(True, THOUGHT_PREFIX_DIVERGE, id="thought_prefix_diverge_streaming"),

169 + pytest.param(False, TOOL_CALL_STARTED, id="tool_call_started"),

170 + pytest.param(True, TOOL_CALL_STARTED, id="tool_call_started_streaming"),

171 ]

172

173

126 -@pytest.mark.parametrize("streaming, param_dict", TEST_CASES)

127 -def test_gemma4_reasoning(

128 - streaming: bool,

129 - param_dict: dict,

130 - generic_tokenizer,

131 -):

132 - output = param_dict["output"]

133 -

174 +def gemma4_encode_output(generic_tokenizer, output: str) -> list[int]:

175 # Resolve token IDs dynamically from the real tokenizer

176 vocab = generic_tokenizer.get_vocab()

177 start_token_id = vocab["<|channel>"]

@@ -176,6 +217,18 @@ def test_gemma4_reasoning(

217 else:

218 output_tokens += _encode(output)

219

220 + return output_tokens

221 +

222 +

223 +@pytest.mark.parametrize("streaming, param_dict", TEST_CASES)

224 +def test_gemma4_reasoning(

225 + streaming: bool,

226 + param_dict: dict,

227 + generic_tokenizer,

228 +):

229 + output = param_dict["output"]

230 + output_tokens = gemma4_encode_output(generic_tokenizer, output)

231 +

232 parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(parser_name)(

233 generic_tokenizer

234 )

@@ -194,3 +247,29 @@ def test_gemma4_reasoning(

247 # Test is_reasoning_end

248 is_reasoning_end = parser.is_reasoning_end(output_tokens)

249 assert is_reasoning_end == param_dict["is_reasoning_end"]

250 +

251 +

252 +def test_gemma4_adjust_request(generic_tokenizer):

253 + parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(parser_name)(

254 + generic_tokenizer

255 + )

256 +

257 + request = ChatCompletionRequest(messages=[], model="test-model")

258 + assert request.skip_special_tokens is True

259 +

260 + result = parser.adjust_request(request)

261 + assert result.skip_special_tokens is False

262 + assert result is request

263 +

264 +

265 +def test_gemma4_previous_turn_reasoning_is_reasoning_end(generic_tokenizer):

266 + output = (

267 + "<|channel>thought\n1st thought<channel|>1st content<turn|>\n"

268 + "<|turn>user\nThanks<|turn>model\n"

269 + )

270 + output_tokens = gemma4_encode_output(generic_tokenizer, output)

271 + parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(parser_name)(

272 + generic_tokenizer

273 + )

274 + is_reasoning_end = parser.is_reasoning_end(output_tokens)

275 + assert not is_reasoning_end

tests/renderers/test_gemma4_chat_template.py new

+345

@@ -0,0 +1,345 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

4 +"""Tests for Gemma4 chat template rendering."""

5 +

6 +from pathlib import Path

7 +

8 +import jinja2.sandbox

9 +import pytest

10 +

11 +TEMPLATE_PATH = (

12 + Path(__file__).resolve().parent.parent.parent

13 + / "examples"

14 + / "tool_chat_template_gemma4.jinja"

15 +)

16 +

17 +

18 +@pytest.fixture(scope="module")

19 +def gemma4_template():

20 + """Load and compile the Gemma4 chat template."""

21 + template_str = TEMPLATE_PATH.read_text()

22 + env = jinja2.sandbox.ImmutableSandboxedEnvironment()

23 + return env.from_string(template_str)

24 +

25 +

26 +def _render(template, messages, **kwargs):

27 + """Render the template with sensible defaults."""

28 + kwargs.setdefault("bos_token", "<bos>")

29 + kwargs.setdefault("add_generation_prompt", False)

30 + return template.render(messages=messages, **kwargs)

31 +

32 +

33 +class TestGemma4ChatTemplate:

34 + def test_basic_multiturn_thinking_disabled(self, gemma4_template):

35 + """With enable_thinking=False (default), generation prompt ends with

36 + an empty thought channel to suppress thinking."""

37 + messages = [

38 + {"role": "user", "content": "Hello"},

39 + {"role": "assistant", "content": "Hi there!"},

40 + {"role": "user", "content": "How are you?"},

41 + ]

42 + result = _render(gemma4_template, messages, add_generation_prompt=True)

43 + assert "<|turn>user\n" in result

44 + assert "<|turn>model\n" in result

45 + assert "Hello" in result

46 + assert "Hi there!" in result

47 + assert "How are you?" in result

48 + assert result.rstrip("\n").endswith("<|channel>thought\n<channel|>")

49 +

50 + def test_basic_multiturn_thinking_enabled(self, gemma4_template):

51 + """With enable_thinking=True, generation prompt ends with model

52 + turn opener (no thought suppression)."""

53 + messages = [

54 + {"role": "user", "content": "Hello"},

55 + {"role": "assistant", "content": "Hi there!"},

56 + {"role": "user", "content": "How are you?"},

57 + ]

58 + result = _render(

59 + gemma4_template,

60 + messages,

61 + add_generation_prompt=True,

62 + enable_thinking=True,

63 + )

64 + assert "<|turn>user\n" in result

65 + assert "<|turn>model\n" in result

66 + assert "Hello" in result

67 + assert "Hi there!" in result

68 + assert "How are you?" in result

69 + assert result.rstrip("\n").endswith("<|turn>model")

70 +

71 + def test_system_message(self, gemma4_template):

72 + messages = [

73 + {"role": "system", "content": "You are helpful."},

74 + {"role": "user", "content": "Hi"},

75 + ]

76 + result = _render(gemma4_template, messages)

77 + assert "<|turn>system\n" in result

78 + assert "You are helpful." in result

79 +

80 + def test_thinking_enabled(self, gemma4_template):

81 + messages = [{"role": "user", "content": "Think about this"}]

82 + result = _render(

83 + gemma4_template,

84 + messages,

85 + add_generation_prompt=True,

86 + enable_thinking=True,

87 + )

88 + assert "<|think|>" in result

89 + assert "<|turn>system\n" in result

90 +

91 + def test_tool_declarations(self, gemma4_template):

92 + tools = [

93 + {

94 + "type": "function",

95 + "function": {

96 + "name": "get_weather",

97 + "description": "Get weather for a city",

98 + "parameters": {

99 + "type": "object",

100 + "properties": {

101 + "city": {

102 + "type": "string",

103 + "description": "City name",

104 + }

105 + },

106 + "required": ["city"],

107 + },

108 + },

109 + }

110 + ]

111 + messages = [{"role": "user", "content": "What is the weather?"}]

112 + result = _render(

113 + gemma4_template,

114 + messages,

115 + tools=tools,

116 + add_generation_prompt=True,

117 + )

118 + assert "<|tool>" in result

119 + assert "declaration:get_weather" in result

120 + assert "<tool|>" in result

121 + assert '<|"|>City name<|"|>' in result

122 +

123 + def test_tool_calls_in_assistant(self, gemma4_template):

124 + messages = [

125 + {"role": "user", "content": "Weather in London?"},

126 + {

127 + "role": "assistant",

128 + "content": "",

129 + "tool_calls": [

130 + {

131 + "id": "call_1",

132 + "function": {

133 + "name": "get_weather",

134 + "arguments": {"city": "London"},

135 + },

136 + }

137 + ],

138 + },

139 + ]

140 + result = _render(gemma4_template, messages)

141 + assert "<|tool_call>call:get_weather{" in result

142 + assert "}<tool_call|>" in result

143 + assert '<|"|>London<|"|>' in result

144 +

145 + def test_tool_responses_openai_style(self, gemma4_template):

146 + """role='tool' messages are formatted as <|tool_response> blocks

147 + with content dumped as-is."""

148 + messages = [

149 + {"role": "user", "content": "Weather?"},

150 + {

151 + "role": "assistant",

152 + "content": "",

153 + "tool_calls": [

154 + {

155 + "id": "call_1",

156 + "function": {

157 + "name": "get_weather",

158 + "arguments": {"city": "London"},

159 + },

160 + }

161 + ],

162 + },

163 + {

164 + "role": "tool",

165 + "tool_call_id": "call_1",

166 + "content": '{"temperature": 15, "condition": "sunny"}',

167 + },

168 + ]

169 + result = _render(gemma4_template, messages, add_generation_prompt=True)

170 + assert "<|tool_response>" in result

171 + assert "response:get_weather{" in result

172 + assert "<tool_response|>" in result

173 + assert '"temperature": 15' in result

174 +

175 + def test_tool_responses_legacy_style(self, gemma4_template):

176 + """tool_responses embedded on the assistant message."""

177 + messages = [

178 + {"role": "user", "content": "Weather?"},

179 + {

180 + "role": "assistant",

181 + "content": "",

182 + "tool_calls": [

183 + {

184 + "function": {

185 + "name": "get_weather",

186 + "arguments": {"city": "London"},

187 + },

188 + }

189 + ],

190 + "tool_responses": [

191 + {

192 + "name": "get_weather",

193 + "response": {"temperature": 20},

194 + }

195 + ],

196 + },

197 + ]

198 + result = _render(gemma4_template, messages)

199 + assert "<|tool_response>" in result

200 + assert "response:get_weather{" in result

201 + assert "temperature:" in result

202 +

203 + def test_generation_prompt_not_after_tool_response(self, gemma4_template):

204 + """add_generation_prompt=True should NOT add <|turn>model when the

205 + last message type was tool_response (the model turn continues)."""

206 + messages = [

207 + {"role": "user", "content": "Weather?"},

208 + {

209 + "role": "assistant",

210 + "content": "",

211 + "tool_calls": [

212 + {

213 + "id": "call_1",

214 + "function": {

215 + "name": "get_weather",

216 + "arguments": {"city": "London"},

217 + },

218 + }

219 + ],

220 + },

221 + {

222 + "role": "tool",

223 + "tool_call_id": "call_1",

224 + "content": "sunny",

225 + },

226 + ]

227 + result = _render(gemma4_template, messages, add_generation_prompt=True)

228 + assert not result.strip().endswith("<|turn>model\n")

229 +

230 + def test_reasoning_in_tool_chains(self, gemma4_template):

231 + """reasoning field on assistant with tool_calls after last user

232 + message emits <|channel>thought\\n...<channel|>."""

233 + messages = [

234 + {"role": "user", "content": "Calculate something"},

235 + {

236 + "role": "assistant",

237 + "content": "",

238 + "reasoning": "Let me think about this...",

239 + "tool_calls": [

240 + {

241 + "function": {

242 + "name": "calculator",

243 + "arguments": {"expr": "2+2"},

244 + },

245 + }

246 + ],

247 + },

248 + ]

249 + result = _render(gemma4_template, messages)

250 + assert "<|channel>thought\n" in result

251 + assert "Let me think about this..." in result

252 + assert "<channel|>" in result

253 +

254 + def test_reasoning_not_before_last_user(self, gemma4_template):

255 + """reasoning on assistant BEFORE the last user message is dropped."""

256 + messages = [

257 + {"role": "user", "content": "First"},

258 + {

259 + "role": "assistant",

260 + "content": "Response",

261 + "reasoning": "Old reasoning that should be dropped",

262 + "tool_calls": [

263 + {

264 + "function": {

265 + "name": "fn",

266 + "arguments": {},

267 + },

268 + }

269 + ],

270 + },

271 + {"role": "user", "content": "Second"},

272 + ]

273 + result = _render(gemma4_template, messages, add_generation_prompt=True)

274 + assert "Old reasoning" not in result

275 +

276 + def test_strip_thinking_in_model_content(self, gemma4_template):

277 + """<|channel>...<channel|> in model content is stripped by the

278 + strip_thinking macro."""

279 + messages = [

280 + {"role": "user", "content": "Hi"},

281 + {

282 + "role": "assistant",

283 + "content": ("<|channel>internal thought<channel|>Visible answer"),

284 + },

285 + ]

286 + result = _render(gemma4_template, messages)

287 + assert "internal thought" not in result

288 + assert "Visible answer" in result

289 +

290 + def test_multi_turn_tool_chain(self, gemma4_template):

291 + """assistant->tool->assistant->tool produces exactly one

292 + <|turn>model (later assistants continue the same turn)."""

293 + messages = [

294 + {"role": "user", "content": "Do two things"},

295 + {

296 + "role": "assistant",

297 + "content": "",

298 + "tool_calls": [

299 + {

300 + "id": "c1",

301 + "function": {"name": "step1", "arguments": {}},

302 + },

303 + ],

304 + },

305 + {"role": "tool", "tool_call_id": "c1", "content": "result1"},

306 + {

307 + "role": "assistant",

308 + "content": "",

309 + "tool_calls": [

310 + {

311 + "id": "c2",

312 + "function": {"name": "step2", "arguments": {}},

313 + },

314 + ],

315 + },

316 + {"role": "tool", "tool_call_id": "c2", "content": "result2"},

317 + ]

318 + result = _render(gemma4_template, messages, add_generation_prompt=True)

319 + assert result.count("<|turn>model\n") == 1

320 +

321 + def test_format_argument_types(self, gemma4_template):

322 + """Strings wrapped in <|"|>, booleans as true/false, numbers bare."""

323 + messages = [

324 + {"role": "user", "content": "Test"},

325 + {

326 + "role": "assistant",

327 + "content": "",

328 + "tool_calls": [

329 + {

330 + "function": {

331 + "name": "test_fn",

332 + "arguments": {

333 + "name": "Alice",

334 + "active": True,

335 + "count": 42,

336 + },

337 + },

338 + }

339 + ],

340 + },

341 + ]

342 + result = _render(gemma4_template, messages)

343 + assert '<|"|>Alice<|"|>' in result

344 + assert "active:true" in result

345 + assert "count:42" in result

tests/tool_parsers/test_gemma4_tool_parser.py

+40

@@ -114,6 +114,19 @@ class TestParseGemma4Args:

114 result = _parse_gemma4_args("key:")

115 assert result == {"key": ""}

116

117 + def test_empty_value_partial_withheld(self):

118 + """Key with no value is withheld in partial mode to avoid premature emission."""

119 + result = _parse_gemma4_args("key:", partial=True)

120 + assert result == {}

121 + # also with a space after the colon

122 + result = _parse_gemma4_args("key: ", partial=True)

123 + assert result == {}

124 +

125 + def test_empty_value_after_other_keys_partial_withheld(self):

126 + """Trailing key with no value is withheld; earlier keys are kept."""

127 + result = _parse_gemma4_args('name:<|"|>test<|"|>,flag:', partial=True)

128 + assert result == {"name": "test"}

129 +

130

131 class TestParseGemma4Array:

132 def test_string_array(self):

@@ -636,3 +649,30 @@ class TestStreamingExtraction:

649 ' <meta charset="UTF-8">\n'

650 ' <meta name="viewport" content="width=device-width">\n'

651 )

652 +

653 + def test_streaming_trailing_bare_bool_not_duplicated(self, parser, mock_request):

654 + """Trailing bare boolean must not be streamed twice."""

655 + chunks = [

656 + "<|tool_call>",

657 + "call:Edit{",

658 + 'file_path:<|"|>src/env.py<|"|>,',

659 + 'old_string:<|"|>old_val<|"|>,',

660 + 'new_string:<|"|>new_val<|"|>,',

661 + "replace_all:",

662 + "false}",

663 + "<tool_call|>",

664 + ]

665 +

666 + results = self._simulate_streaming(parser, mock_request, chunks)

667 + args_text = self._collect_arguments(results)

668 + assert args_text, "No arguments were streamed"

669 +

670 + parsed_args = json.loads(args_text)

671 + assert parsed_args == {

672 + "file_path": "src/env.py",

673 + "old_string": "old_val",

674 + "new_string": "new_val",

675 + "replace_all": False,

676 + }

677 +

678 + assert args_text.count("replace_all") == 1

vllm/entrypoints/anthropic/serving.py

+2 -1

@@ -170,7 +170,8 @@ class AnthropicServingMessages(OpenAIServingChat):

170 else:

171 cls._convert_message_content(msg, openai_msg, openai_messages)

172

173 - openai_messages.append(openai_msg)

173 + if not (msg.role == "user" and "content" not in openai_msg):

174 + openai_messages.append(openai_msg)

175

176 @classmethod

177 def _convert_message_content(

vllm/entrypoints/openai/api_server.py

@@ -379,6 +379,7 @@ async def init_app_state(

379 enable_auto_tools=args.enable_auto_tool_choice,

380 exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,

381 tool_parser=args.tool_call_parser,

382 + reasoning_parser=args.structured_outputs_config.reasoning_parser,

383 default_chat_template_kwargs=args.default_chat_template_kwargs,

384 log_error_stack=args.log_error_stack,

385 )

@@ -481,6 +482,7 @@ async def init_render_app_state(

482 enable_auto_tools=args.enable_auto_tool_choice,

483 exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,

484 tool_parser=args.tool_call_parser,

485 + reasoning_parser=args.structured_outputs_config.reasoning_parser,

486 default_chat_template_kwargs=args.default_chat_template_kwargs,

487 log_error_stack=args.log_error_stack,

488 )

vllm/entrypoints/openai/responses/serving.py

@@ -594,6 +594,7 @@ class OpenAIServingResponses(OpenAIServing):

594 default_template_kwargs=None,

595 tool_dicts=tool_dicts,

596 tool_parser=self.parser.tool_parser_cls if self.parser else None,

597 + reasoning_parser=self.parser.reasoning_parser_cls if self.parser else None,

598 )

599 return messages, engine_inputs

600

@@ -618,6 +619,7 @@ class OpenAIServingResponses(OpenAIServing):

619 default_template_kwargs=None,

620 tool_dicts=tool_dicts,

621 tool_parser=tool_parser,

622 + reasoning_parser=self.parser.reasoning_parser_cls if self.parser else None,

623 )

624 return engine_inputs

625

vllm/entrypoints/serve/render/serving.py

+13

@@ -44,6 +44,7 @@ from vllm.inputs import (

44 )

45 from vllm.logger import init_logger

46 from vllm.parser import ParserManager

47 +from vllm.reasoning.abs_reasoning_parsers import ReasoningParser

48 from vllm.renderers import BaseRenderer, merge_kwargs

49 from vllm.renderers.inputs.preprocess import (

50 extract_prompt_components,

@@ -74,6 +75,7 @@ class OpenAIServingRender:

75 enable_auto_tools: bool = False,

76 exclude_tools_when_tool_choice_none: bool = False,

77 tool_parser: str | None = None,

78 + reasoning_parser: str | None = None,

79 default_chat_template_kwargs: dict[str, Any] | None = None,

80 log_error_stack: bool = False,

81 ) -> None:

@@ -94,6 +96,11 @@ class OpenAIServingRender:

96 enable_auto_tools=enable_auto_tools,

97 model_name=model_config.model,

98 )

99 + self.reasoning_parser: type[ReasoningParser] | None = (

100 + ParserManager.get_reasoning_parser(

101 + reasoning_parser_name=reasoning_parser,

102 + )

103 + )

104 self.default_chat_template_kwargs: dict[str, Any] = (

105 default_chat_template_kwargs or {}

106 )

@@ -245,6 +252,7 @@ class OpenAIServingRender:

252 default_template_kwargs=self.default_chat_template_kwargs,

253 tool_dicts=tool_dicts,

254 tool_parser=tool_parser,

255 + reasoning_parser=self.reasoning_parser,

256 )

257 else:

258 # For GPT-OSS.

@@ -503,6 +511,7 @@ class OpenAIServingRender:

511 default_template_kwargs: dict[str, Any] | None,

512 tool_dicts: list[dict[str, Any]] | None = None,

513 tool_parser: type[ToolParser] | None = None,

514 + reasoning_parser: type[ReasoningParser] | None = None,

515 *,

516 skip_mm_cache: bool = False,

517 ) -> tuple[list[ConversationMessage], list[EngineInput]]:

@@ -539,6 +548,10 @@ class OpenAIServingRender:

548 skip_mm_cache=skip_mm_cache,

549 )

550

551 + if reasoning_parser is not None:

552 + tokenizer = renderer.get_tokenizer()

553 + request = reasoning_parser(tokenizer).adjust_request(request=request)

554 +

555 # tool parsing is done only if a tool_parser has been set and if

556 # tool_choice is not "none" (if tool_choice is "none" but a tool_parser

557 # is set, we want to prevent parsing a tool_call hallucinated by the LLM

vllm/parser/abstract_parser.py

@@ -496,6 +496,15 @@ class DelegatingParser(Parser):

496 # No tool calls

497 return [], content

498

499 + def adjust_request(

500 + self, request: ChatCompletionRequest | ResponsesRequest

501 + ) -> ChatCompletionRequest | ResponsesRequest:

502 + if self._reasoning_parser is not None:

503 + request = self._reasoning_parser.adjust_request(request)

504 + if self._tool_parser is not None:

505 + request = self._tool_parser.adjust_request(request)

506 + return request

507 +

508 def extract_reasoning_streaming(

509 self,

510 previous_text: str,

vllm/reasoning/abs_reasoning_parsers.py

+8 -2

@@ -6,7 +6,7 @@ import os

6 from abc import abstractmethod

7 from collections.abc import Callable, Iterable, Sequence

8 from functools import cached_property

9 -from typing import TYPE_CHECKING

9 +from typing import TYPE_CHECKING, cast

11 from vllm.entrypoints.mcp.tool_server import ToolServer

12 from vllm.logger import init_logger

@@ -150,6 +150,12 @@ class ReasoningParser:

150 previously been parsed and extracted (see constructor)

151 """

152

153 + def adjust_request(

154 + self, request: "ChatCompletionRequest | ResponsesRequest"

155 + ) -> "ChatCompletionRequest | ResponsesRequest":

156 + """Adjust request parameters; override in subclasses as needed."""

157 + return request

158 +

159 def prepare_structured_tag(

160 self,

161 original_tag: str | None,

@@ -298,7 +304,7 @@ class ReasoningParserManager:

304 if isinstance(name, str):

305 names = [name]

306 elif is_list_of(name, str):

301 - names = name

307 + names = cast(list[str], name)

308 else:

309 names = [class_name]

310

vllm/reasoning/gemma4_reasoning_parser.py

+35 -3

@@ -52,6 +52,16 @@ class Gemma4ReasoningParser(BaseThinkingReasoningParser):

52 # skip_special_tokens=True).

53 self._reasoning_text: str = ""

54 self._prefix_stripped: bool = False

55 + self.new_turn_token_id = self.vocab["<|turn>"]

56 + self.tool_call_token_id = self.vocab["<|tool_call>"]

57 + self.tool_response_token_id = self.vocab["<|tool_response>"]

58 +

59 + def adjust_request(

60 + self, request: "ChatCompletionRequest | ResponsesRequest"

61 + ) -> "ChatCompletionRequest | ResponsesRequest":

62 + """Disable special-token stripping to preserve boundary tokens."""

63 + request.skip_special_tokens = False

64 + return request

66 @property

67 def start_token(self) -> str:

@@ -63,6 +73,29 @@ class Gemma4ReasoningParser(BaseThinkingReasoningParser):

73 """The token that ends reasoning content."""

74 return "<channel|>"

76 + def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:

77 + start_token_id = self.start_token_id

78 + end_token_id = self.end_token_id

79 + new_turn_token_id = self.new_turn_token_id

80 + tool_call_token_id = self.tool_call_token_id

81 + tool_response_token_id = self.tool_response_token_id

82 +

83 + # Search from the end of input_ids to find the last match.

84 + for i in range(len(input_ids) - 1, -1, -1):

85 + if input_ids[i] == start_token_id:

86 + return False

87 + if input_ids[i] == tool_call_token_id:

88 + # We're generating a tool call, so reasoning must be ended.

89 + return True

90 + if input_ids[i] in (new_turn_token_id, tool_response_token_id):

91 + # We found a new turn or tool response token so don't consider

92 + # reasoning ended yet, since the model starts new reasoning

93 + # after these tokens.

94 + return False

95 + if input_ids[i] == end_token_id:

96 + return True

97 + return False

98 +

99 # ------------------------------------------------------------------

100 # Non-streaming path

101 # ------------------------------------------------------------------

@@ -159,11 +192,10 @@ class Gemma4ReasoningParser(BaseThinkingReasoningParser):

192 result.reasoning = stripped

193 return result

194 else:

162 - # This entire delta was prefix — suppress it.

163 - # Don't set _prefix_stripped yet; there may be more

164 - # prefix chars to consume in the next delta.

195 if len(self._reasoning_text) >= prefix_len:

196 self._prefix_stripped = True

197 + result.reasoning = ""

198 + return result

199 return None

200

201 # Case 2: Accumulated text is a strict prefix of

vllm/tool_parsers/gemma4_tool_parser.py

+4 -2

@@ -122,14 +122,16 @@ def _parse_gemma4_args(args_str: str, *, partial: bool = False) -> dict:

122

123 # Parse value

124 if i >= n:

125 - result[key] = ""

125 + if not partial:

126 + result[key] = ""

127 break

128

129 # Skip whitespace after ':'

130 while i < n and args_str[i] in (" ", "\n", "\t"):

131 i += 1

132 if i >= n:

132 - result[key] = ""

133 + if not partial:

134 + result[key] = ""

135 break

136

137 # String value: <|"|>...<|"|>