Restore focused vision queries
Expose an optional route-agnostic query on vision_load and pass it to delegated image analysis alongside the current user request. Keep native image history unchanged while preserving one-call multi-image batching, permissive Responses schemas, and generic parallel execution.
GreifMax committed
Aug 26, 2026 at 00:49 UTC
54534ba6368bdb434c29f75de82affb448e4dc14
6 files changed
+74
-20
prompts/agent.system.tools_vision.md
+6
-4
@@ -2,9 +2,10 @@
2
3
### vision_load
4
load images into the model for visual reasoning
5
-args: `paths` list of absolute image paths or tool-returned ephemeral image refs
5
+args: `paths` list of absolute image paths or tool-returned ephemeral image refs, optional `query` for focused inspection
6
rules:
7
- load all relevant images in one call when comparing screenshots or pages
8
+- add `query` when the visual task is narrower than the user's request or derived during the work
9
- use when the task depends on screenshots, diagrams, scanned documents, charts, or photos
10
- only bitmaps are supported; convert other formats first if needed
11
- the tool result includes loaded/skipped image totals and the corresponding path lists
@@ -12,12 +13,13 @@ example:
13
```json
14
{
15
"thoughts": [
15
- "I need to inspect the screenshot before answering."
16
+ "I need to compare the screenshots."
17
],
17
- "headline": "Loading screenshot for visual analysis",
18
+ "headline": "Comparing screenshots",
19
"tool_name": "vision_load",
20
"tool_args": {
20
- "paths": ["/path/to/screenshot.png"]
21
+ "paths": ["/path/to/before.png", "/path/to/after.png"],
22
+ "query": "Compare the error-banner alignment."
23
}
24
}
25
```
prompts/fw.vision_load.md
+6
-1
@@ -1,4 +1,9 @@
1
-Analyze the attached image(s) for the current request. Return only relevant visible details, text, and layout.
1
+Analyze the attached image(s) for the current request, focusing on the visual query when provided. Return only relevant visible details, text, and layout.
2
3
Current request:
4
{{request}}
5
+
6
+{{if query}}
7
+Visual query:
8
+{{query}}
9
+{{endif}}
tests/test_tool_policy.py
+24
-2
@@ -8,7 +8,7 @@ from types import SimpleNamespace
8
import pytest
9
10
from extensions.python.system_prompt import _11_tools_prompt, _13_skills_prompt
11
-from helpers import mcp_handler, responses_tools, tool_policy
11
+from helpers import files, mcp_handler, responses_tools, tool_policy
12
from helpers.errors import RepairableException
13
from plugins._tool_access.extensions.python.tool_execute_before._10_enforce_tool_policy import (
14
EnforceToolPolicy,
@@ -596,7 +596,29 @@ def test_vision_prompt_stays_route_agnostic_and_batches_paths() -> None:
596
assert "load all relevant images in one call" in source
597
assert "Input schema for tool_args" not in source
598
assert "Vision Model" not in source
599
- assert "query" not in source
599
+ assert "sidecar" not in source.lower()
600
+ assert "optional `query`" in source
601
+
602
+
603
+def test_vision_framework_prompt_renders_optional_query() -> None:
604
+ prompt_dir = str(Path(__file__).resolve().parents[1] / "prompts")
605
+
606
+ without_query = files.read_prompt_file(
607
+ "fw.vision_load.md",
608
+ _directories=[prompt_dir],
609
+ request="Review these screenshots.",
610
+ query="",
611
+ )
612
+ with_query = files.read_prompt_file(
613
+ "fw.vision_load.md",
614
+ _directories=[prompt_dir],
615
+ request="Review these screenshots.",
616
+ query="Compare the error banners.",
617
+ )
618
+
619
+ assert "Visual query:" not in without_query
620
+ assert "Current request:\nReview these screenshots." in with_query
621
+ assert "Visual query:\nCompare the error banners." in with_query
622
623
624
def test_mcp_prompt_and_native_schema_omit_blocked_tool(
tests/test_vision_load_image_refs.py
+19
-7
@@ -126,11 +126,15 @@ async def test_vision_load_materializes_local_image_to_chat_artifact(monkeypatch
126
invalid = await tool.execute(paths=None)
127
assert invalid.message == "vision_load error: `paths` must be a string or an array."
128
129
- response = await tool.execute(paths=str(image_path))
129
+ response = await tool.execute(
130
+ paths=str(image_path),
131
+ query="Read the footer text.",
132
+ )
133
image_path.unlink()
134
await tool.after_execution(response)
135
136
raw_message = messages[0][1]["content"]
137
+ assert [item["type"] for item in raw_message["raw_content"]] == ["image_url"]
138
stored_ref = raw_message["raw_content"][0]["image_url"]["url"]
139
assert stored_ref.startswith("/a0/usr/chats/ctx-vision/images/vision-load/sample-image-")
140
stored_path = tmp_path / stored_ref.removeprefix("/a0/")
@@ -206,9 +210,11 @@ async def test_vision_model_sends_multiple_images_once_and_keeps_history_text_on
210
context=SimpleNamespace(id=""),
211
agent_name="Agent 0",
212
last_user_message=SimpleNamespace(
209
- output_text=lambda: "Compare the login errors."
213
+ output_text=lambda: "Review these UI screenshots."
214
+ ),
215
+ read_prompt=lambda _name, request, query: (
216
+ f"Current request: {request}\n\nVisual query: {query}"
217
),
211
- read_prompt=lambda _name, request: f"Analyze: {request}",
218
hist_add_tool_result=lambda *args, **kwargs: tool_results.append((args, kwargs)),
219
hist_add_message=lambda *args, **kwargs: raw_messages.append((args, kwargs)),
220
)
@@ -222,7 +228,10 @@ async def test_vision_model_sends_multiple_images_once_and_keeps_history_text_on
228
)
229
tool.log = SimpleNamespace(id="vision-log", update=lambda **kwargs: None)
230
225
- response = await tool.execute(paths=[str(path) for path in image_paths])
231
+ response = await tool.execute(
232
+ paths=[str(path) for path in image_paths],
233
+ query="Compare the login error banners.",
234
+ )
235
response.additional = {"_responses_output_item": {"output": response.message}}
236
await tool.after_execution(response)
237
@@ -230,7 +239,10 @@ async def test_vision_model_sends_multiple_images_once_and_keeps_history_text_on
239
content = calls[0]["messages"][0].content
240
assert content[0] == {
241
"type": "text",
233
- "text": "Analyze: Compare the login errors.",
242
+ "text": (
243
+ "Current request: Review these UI screenshots.\n\n"
244
+ "Visual query: Compare the login error banners."
245
+ ),
246
}
247
assert [item["type"] for item in content].count("image_url") == 2
248
assert "max_tokens" not in calls[0]
@@ -265,7 +277,7 @@ async def test_vision_model_empty_response_is_reported_as_error(monkeypatch):
277
agent = SimpleNamespace(
278
context=SimpleNamespace(id="", get_data=lambda _key: ""),
279
last_user_message=SimpleNamespace(output_text=lambda: "Inspect the image."),
268
- read_prompt=lambda _name, request: request,
280
+ read_prompt=lambda _name, request, query: f"{request}\n{query}",
281
)
282
tool = vision_load_module.VisionLoad(
283
agent=agent,
@@ -398,7 +410,7 @@ async def test_independent_vision_model_calls_can_run_concurrently(monkeypatch,
410
last_user_message=SimpleNamespace(
411
output_text=lambda: f"inspection {index}"
412
),
401
- read_prompt=lambda _name, request: request,
413
+ read_prompt=lambda _name, request, query: f"{request}\n{query}",
414
)
415
return vision_load_module.VisionLoad(
416
agent=agent,
tools/vision_load.py
+16
-4
@@ -23,7 +23,12 @@ TOKENS_ESTIMATE = 1500
23
24
25
class VisionLoad(Tool):
26
- async def execute(self, paths: list[str] | str = [], **kwargs) -> Response:
26
+ async def execute(
27
+ self,
28
+ paths: list[str] | str = [],
29
+ query: str = "",
30
+ **kwargs,
31
+ ) -> Response:
32
33
self.images_dict = {}
34
self.loaded_paths: list[str] = []
@@ -87,7 +92,10 @@ class VisionLoad(Tool):
92
message = self._summary() if self.images_dict or self.skipped_paths else "No images processed"
93
if self.vision_config and self.images_dict:
94
try:
90
- capsule = await self._call_vision_model(list(self.images_dict.values()))
95
+ capsule = await self._call_vision_model(
96
+ list(self.images_dict.values()),
97
+ query,
98
+ )
99
message = (
100
f"Analyzed {len(self.images_dict)} image(s)"
101
f"; {len(self.skipped_paths)} skipped.\n\n{capsule.strip()}"
@@ -110,14 +118,18 @@ class VisionLoad(Tool):
118
)
119
return str(parent_id or getattr(context, "id", "") or "").strip()
120
113
- async def _call_vision_model(self, image_paths: list[str]) -> str:
121
+ async def _call_vision_model(self, image_paths: list[str], query: str) -> str:
122
user_message = getattr(self.agent, "last_user_message", None)
123
output_text = getattr(user_message, "output_text", None)
124
request = str(output_text() if callable(output_text) else "").strip()
125
content = [
126
{
127
"type": "text",
120
- "text": self.agent.read_prompt("fw.vision_load.md", request=request),
128
+ "text": self.agent.read_prompt(
129
+ "fw.vision_load.md",
130
+ request=request,
131
+ query=str(query or "").strip(),
132
+ ),
133
}
134
]
135
content.extend(
tools/vision_load.py.dox.md
+3
-2
@@ -12,7 +12,7 @@
12
- `vision_load.py.dox.md` owns durable notes about responsibilities, contracts, side effects, and verification for that implementation.
13
- Classes:
14
- `VisionLoad` (`Tool`)
15
- - `async execute(self, paths, **kwargs) -> Response`
15
+ - `async execute(self, paths, query="", **kwargs) -> Response`
16
- `async after_execution(self, response: Response, **kwargs)`
17
- Notable constants/configuration names: `TOKENS_ESTIMATE`.
18
@@ -21,7 +21,8 @@
21
- Tool modules must define `helpers.tool.Tool` subclasses and return `helpers.tool.Response` from `execute(...)`.
22
- One call may contain multiple paths; a bare string is treated as one path. The Vision Model route sends every selected path in one request and returns one textual capsule.
23
- Model configuration exposes a Vision Model only when the effective preset selects that route; otherwise this tool follows Main's native vision path.
24
-- The public tool contract is route-agnostic. A Vision Model receives the current user request through `fw.vision_load.md`; direct parallel workers inherit that request from their parent.
24
+- The public tool contract is route-agnostic and accepts an optional focused `query`. A Vision Model receives both that query and the current user request through `fw.vision_load.md`; direct parallel workers inherit the request from their parent.
25
+- Native Main vision already retains the query in its authored tool-call transcript, so native raw history remains image-only and does not repeat model-authored instructions as user content.
26
- Delegation completes during `execute(...)` so native Responses function output contains the real capsule before `after_execution(...)` persists it.
27
- Delegated history contains the text capsule only. Native history contains the tool result followed by one raw message holding all loaded image blocks.
28
- In a direct parallel worker, native image content is queued for the parent and promoted immediately after the outer `parallel` result; the disposable worker never owns the only copy of model-visible pixels.