A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 12:15 UTC
fix(frontend): Add multimodal placeholders to Gemma4 tool message template (#41459)
Signed-off-by: Harshal Janjani <harshaljanjani@gmail.com> Co-authored-by: Ben Browning <bbrownin@redhat.com>
Harshal Janjani committed
May 29, 2026 at 01:48 UTC
69c9f199574e7fe3a05847f6652bf914639f4c08
2 files changed
+78
examples/tool_chat_template_gemma4.jinja
+9
@@ -295,6 +295,15 @@
295
{%- endif -%}
296
{%- endfor -%}
297
{{- format_tool_response_block(ns_tname.name, ns_txt.s) -}}
298
+ {%- for part in tool_body -%}
299
+ {%- if part.get('type') == 'image' -%}
300
+ {{- '<|image|>' -}}
301
+ {%- elif part.get('type') == 'audio' -%}
302
+ {{- '<|audio|>' -}}
303
+ {%- elif part.get('type') == 'video' -%}
304
+ {{- '<|video|>' -}}
305
+ {%- endif -%}
306
+ {%- endfor -%}
307
{%- else -%}
308
{{- format_tool_response_block(ns_tname.name, tool_body) -}}
309
{%- endif -%}
tests/renderers/test_gemma4_chat_template.py
+69
@@ -343,3 +343,72 @@ class TestGemma4ChatTemplate:
343
assert '<|"|>Alice<|"|>' in result
344
assert "active:true" in result
345
assert "count:42" in result
346
+
347
+ def test_tool_response_with_multimodal_content(self, gemma4_template):
348
+ """Multimodal placeholders in tool messages are emitted after the
349
+ tool_response block."""
350
+ messages = [
351
+ {"role": "user", "content": "Download the image and describe it."},
352
+ {
353
+ "role": "assistant",
354
+ "content": "",
355
+ "tool_calls": [
356
+ {
357
+ "id": "call_1",
358
+ "type": "function",
359
+ "function": {
360
+ "name": "download_image",
361
+ "arguments": '{"url": "https://example.com/x.png"}',
362
+ },
363
+ },
364
+ ],
365
+ },
366
+ {
367
+ "role": "tool",
368
+ "tool_call_id": "call_1",
369
+ "content": [
370
+ {"type": "text", "text": "Image downloaded successfully."},
371
+ {"type": "image"},
372
+ ],
373
+ },
374
+ ]
375
+ result = _render(gemma4_template, messages, add_generation_prompt=True)
376
+ assert "<|tool_response>" in result
377
+ assert "response:download_image{" in result
378
+ assert "<tool_response|>" in result
379
+ assert "<|image|>" in result
380
+
381
+ def test_tool_response_with_all_modalities(self, gemma4_template):
382
+ """All multimodal types (image, audio, video) in a single tool
383
+ response are rendered."""
384
+ messages = [
385
+ {"role": "user", "content": "Process media"},
386
+ {
387
+ "role": "assistant",
388
+ "content": "",
389
+ "tool_calls": [
390
+ {
391
+ "id": "c1",
392
+ "type": "function",
393
+ "function": {
394
+ "name": "process",
395
+ "arguments": "{}",
396
+ },
397
+ },
398
+ ],
399
+ },
400
+ {
401
+ "role": "tool",
402
+ "tool_call_id": "c1",
403
+ "content": [
404
+ {"type": "text", "text": "Results."},
405
+ {"type": "image"},
406
+ {"type": "audio"},
407
+ {"type": "video"},
408
+ ],
409
+ },
410
+ ]
411
+ result = _render(gemma4_template, messages, add_generation_prompt=True)
412
+ assert "<|image|>" in result
413
+ assert "<|audio|>" in result
414
+ assert "<|video|>" in result