Consolidate vision tool prompts
Alessandro committed
Aug 25, 2026 at 19:25 UTC
0354324e1340f37c2842ac9b44ceccbceea43009
7 files changed
+65
-78
extensions/python/system_prompt/_11_tools_prompt.py
+5
-4
@@ -57,9 +57,10 @@ async def build_prompt(agent: Agent) -> str:
57
)
58
59
chat_cfg = get_chat_model_config(agent)
60
- if use_vision_sidecar(agent):
61
- prompt += "\n\n" + agent.read_prompt("agent.system.tools_vision_sidecar.md")
62
- elif chat_cfg.get("vision", False):
63
- prompt += "\n\n" + agent.read_prompt("agent.system.tools_vision.md")
60
+ sidecar = use_vision_sidecar(agent)
61
+ if sidecar or chat_cfg.get("vision", False):
62
+ prompt += "\n\n" + agent.read_prompt(
63
+ "agent.system.tools_vision.md", sidecar=sidecar
64
+ )
65
66
return prompt
helpers/responses_tools.py
+3
-4
@@ -119,11 +119,10 @@ def _vision_tool_prompt(agent: Any) -> str:
119
use_vision_sidecar,
120
)
121
122
- if use_vision_sidecar(agent):
123
- return agent.read_prompt("agent.system.tools_vision_sidecar.md")
124
- if not get_chat_model_config(agent).get("vision", False):
122
+ sidecar = use_vision_sidecar(agent)
123
+ if not (sidecar or get_chat_model_config(agent).get("vision", False)):
124
return ""
126
- return agent.read_prompt("agent.system.tools_vision.md")
125
+ return agent.read_prompt("agent.system.tools_vision.md", sidecar=sidecar)
126
except Exception:
127
return ""
128
helpers/responses_tools.py.dox.md
+1
-1
@@ -12,7 +12,7 @@
12
13
## Local Contracts
14
15
-- Build local function tools from enabled `agent.system.tool.*.md` prompt files and include `vision_load` when either Main native vision or the effective preset's Vision Model enables the matching prompt.
15
+- Build local function tools from enabled `agent.system.tool.*.md` prompt files and include `vision_load` when either Main native vision or the effective preset's Vision Model enables the canonical vision prompt.
16
- Discover local prompt files through `helpers.subagents.get_paths`; this module
17
owns the Responses-specific prompt-name compatibility rules.
18
- Local prompt-derived function names use existing bullet declarations that pair a backticked name with `arg` or `args` for multi-tool prompt files, otherwise prefer explicit `"tool_name"` examples, then the first prompt heading, and finally the prompt filename.
prompts/AGENTS.md
+1
-1
@@ -25,7 +25,7 @@
25
- Read the rendering path before changing placeholders or filenames.
26
- Prefer small prompt additions over broad rewrites when fixing a specific behavior.
27
- Keep document/OCR routing explicit: image files, screenshots, scans, charts, photos, and diagrams should prefer vision tools when available, while `document_query` is for documents, large text-heavy files, and fallback OCR.
28
-- Keep native and sidecar vision prompts synchronized with `vision_load`: related images belong in one call, while a sidecar result is text-only unless Main explicitly requests its native raw route.
28
+- Keep the single `vision_load` prompt accurate for both native and sidecar routing: related images belong in one call, while a sidecar result is text-only unless Main explicitly requests its native raw route.
29
- Update tests or snapshots when prompt budget, required sections, or generated system content changes.
30
31
## Verification
prompts/agent.system.tools_vision.md
+37
-5
@@ -1,11 +1,41 @@
1
## multimodal vision tools
2
3
### vision_load
4
-load images into the model for visual reasoning
5
-args: `paths` list of absolute image paths or tool-returned ephemeral image refs
4
+{{if sidecar}}analyze images with the preset's separate Vision Model and return a text result{{endif}}
5
+{{if not sidecar}}load images into Main for visual reasoning{{endif}}
6
+args: `paths` list of absolute image paths or ephemeral image refs{{if sidecar}}, `query` optional focused instruction, `raw` optional boolean{{endif}}
7
+{{if sidecar}}
8
+Input schema for tool_args:
9
+```json
10
+{
11
+ "type": "object",
12
+ "properties": {
13
+ "paths": {
14
+ "type": "array",
15
+ "items": {"type": "string"},
16
+ "description": "Absolute image paths or ephemeral image refs."
17
+ },
18
+ "query": {
19
+ "type": "string",
20
+ "description": "What the Vision Model should inspect, compare, locate, or read."
21
+ },
22
+ "raw": {
23
+ "type": "boolean",
24
+ "description": "Use Main's native vision instead of the separate Vision Model, when Main supports vision."
25
+ }
26
+ },
27
+ "required": ["paths"],
28
+ "additionalProperties": false
29
+}
30
+```
31
+{{endif}}
32
rules:
7
-- load all relevant images in one call when comparing screenshots or pages
33
+- put all images needed for one comparison or visual task in the same `paths` array
34
- use when the task depends on screenshots, diagrams, scanned documents, charts, or photos
35
+{{if sidecar}}
36
+- when the separate Vision Model is active, use a focused `query`; the result is a text capsule and Main does not receive raw images
37
+- use `raw=true` only when Main supports vision and must inspect the pixels itself
38
+{{endif}}
39
- only bitmaps are supported; convert other formats first if needed
40
- the tool result includes loaded/skipped image totals and the corresponding path lists
41
example:
@@ -14,10 +44,12 @@ example:
44
"thoughts": [
45
"I need to inspect the screenshot before answering."
46
],
17
- "headline": "Loading screenshot for visual analysis",
47
+ "headline": "Comparing screenshots",
48
"tool_name": "vision_load",
49
"tool_args": {
20
- "paths": ["/path/to/screenshot.png"]
50
+ "paths": ["/path/to/before.png", "/path/to/after.png"]{{if sidecar}},
51
+ "query": "Compare the error banners and describe what changed."
52
+ {{endif}}
53
}
54
}
55
```
prompts/agent.system.tools_vision_sidecar.md
deleted
-48
@@ -1,48 +0,0 @@
1
-## multimodal vision tools
2
-
3
-### vision_load
4
-analyze images with the separate Vision Model and return a text result
5
-args: `paths` list of absolute image paths or ephemeral image refs, `query` optional focused instruction, `raw` optional boolean
6
-Input schema for tool_args:
7
-```json
8
-{
9
- "type": "object",
10
- "properties": {
11
- "paths": {
12
- "type": "array",
13
- "items": {"type": "string"},
14
- "description": "Absolute image paths or ephemeral image refs."
15
- },
16
- "query": {
17
- "type": "string",
18
- "description": "What the Vision Model should inspect, compare, locate, or read."
19
- },
20
- "raw": {
21
- "type": "boolean",
22
- "description": "Use the Main model's native vision instead, when Main supports vision."
23
- }
24
- },
25
- "required": ["paths"],
26
- "additionalProperties": false
27
-}
28
-```
29
-rules:
30
-- put all images needed for one comparison or visual task in the same `paths` array; they are sent in one Vision Model call
31
-- use a focused `query`; if omitted, the Vision Model returns a concise general description
32
-- the result is a text capsule; the Main model does not receive the raw images
33
-- use `raw=true` only when Main supports vision and must inspect the pixels itself
34
-- only bitmaps are supported
35
-example:
36
-```json
37
-{
38
- "thoughts": [
39
- "I need to compare both screenshots before answering."
40
- ],
41
- "headline": "Comparing screenshots",
42
- "tool_name": "vision_load",
43
- "tool_args": {
44
- "paths": ["/path/to/before.png", "/path/to/after.png"],
45
- "query": "Compare the error banners and describe what changed."
46
- }
47
-}
48
-```
tests/test_tool_policy.py
+18
-15
@@ -8,7 +8,7 @@ from types import SimpleNamespace
8
import pytest
9
10
from extensions.python.system_prompt import _11_tools_prompt, _13_skills_prompt
11
-from helpers import mcp_handler, responses_tools, tool_policy
11
+from helpers import files, mcp_handler, responses_tools, tool_policy
12
from helpers.errors import RepairableException
13
from plugins._tool_access.extensions.python.tool_execute_before._10_enforce_tool_policy import (
14
EnforceToolPolicy,
@@ -547,19 +547,14 @@ async def test_vision_tool_follows_chat_config_not_profile_policy(
547
548
549
@pytest.mark.asyncio
550
-async def test_vision_sidecar_prompt_replaces_native_vision_prompt(
550
+async def test_vision_sidecar_uses_canonical_vision_prompt(
551
monkeypatch, tmp_path: Path
552
) -> None:
553
_write_prompt(tmp_path, "agent.system.tools.md", "TOOLS\n{{tools}}")
554
_write_prompt(
555
tmp_path,
556
"agent.system.tools_vision.md",
557
- "### vision_load\nnative pixels\nargs: `paths`",
558
- )
559
- _write_prompt(
560
- tmp_path,
561
- "agent.system.tools_vision_sidecar.md",
562
- "### vision_load\nsidecar capsule\nargs: `paths`, `query`",
557
+ "### vision_load\ncanonical vision\nargs: `paths`, `query`",
558
)
559
monkeypatch.setattr(tool_policy.subagents, "get_paths", _prompt_paths(tmp_path))
560
monkeypatch.setattr(
@@ -576,19 +571,20 @@ async def test_vision_sidecar_prompt_replaces_native_vision_prompt(
571
prompt = await _11_tools_prompt.build_prompt(agent)
572
schemas, _name_map = responses_tools.build_responses_function_tools(agent)
573
579
- assert "sidecar capsule" in prompt
580
- assert "native pixels" not in prompt
574
+ assert prompt.count("canonical vision") == 1
575
assert schemas[0]["name"] == "vision_load"
582
- assert schemas[0]["description"] == "sidecar capsule"
576
+ assert schemas[0]["description"] == "canonical vision"
577
578
585
-def test_vision_sidecar_prompt_declares_multi_image_native_schema() -> None:
586
- prompt = (
579
+def test_vision_prompt_declares_multi_image_native_schema() -> None:
580
+ source = (
581
Path(__file__).resolve().parents[1]
582
/ "prompts"
589
- / "agent.system.tools_vision_sidecar.md"
583
+ / "agent.system.tools_vision.md"
584
).read_text(encoding="utf-8")
591
- schema = responses_tools._schema_from_prompt(prompt)
585
+ sidecar_prompt = files.evaluate_text_conditions(source, sidecar=True)
586
+ native_prompt = files.evaluate_text_conditions(source, sidecar=False)
587
+ schema = responses_tools._schema_from_prompt(sidecar_prompt)
588
589
assert schema["required"] == ["paths"]
590
assert schema["properties"]["paths"] == {
@@ -597,6 +593,13 @@ def test_vision_sidecar_prompt_declares_multi_image_native_schema() -> None:
593
"description": "Absolute image paths or ephemeral image refs.",
594
}
595
assert {"query", "raw"} <= schema["properties"].keys()
596
+ assert "separate Vision Model" in sidecar_prompt
597
+ assert "separate Vision Model" not in native_prompt
598
+ assert responses_tools._schema_from_prompt(native_prompt) == {
599
+ "type": "object",
600
+ "properties": {},
601
+ "additionalProperties": True,
602
+ }
603
604
605
def test_mcp_prompt_and_native_schema_omit_blocked_tool(