Simplify vision sidecar execution
Alessandro committed
Aug 25, 2026 at 19:25 UTC
5d4f49ad616137dda7e7ca348674d5c80779bc94
4 files changed
+34
-68
tests/test_parallel_tool.py
-36
@@ -160,42 +160,6 @@ def test_normalize_parallel_tool_calls_accepts_json_string_array() -> None:
160
assert calls[1].tool_args["message"] == "Research nuclear fusion news in Italian."
161
162
163
-def test_parallel_keeps_mixed_tools_and_batched_vision_paths() -> None:
164
- calls = parallel_tools.normalize_parallel_tool_calls(
165
- [
166
- {"tool_name": "search_a", "tool_args": {"query": "one"}},
167
- {"tool_name": "search_b", "tool_args": {"query": "two"}},
168
- {"tool_name": "browser_agent", "tool_args": {"message": "open page"}},
169
- {
170
- "tool_name": "vision_load",
171
- "tool_args": {
172
- "paths": ["/tmp/before.png", "/tmp/after.png"],
173
- "query": "compare",
174
- },
175
- },
176
- ]
177
- )
178
-
179
- assert [call.tool_name for call in calls] == [
180
- "search_a",
181
- "search_b",
182
- "browser_agent",
183
- "vision_load",
184
- ]
185
- assert calls[3].tool_args["paths"] == ["/tmp/before.png", "/tmp/after.png"]
186
-
187
-
188
-def test_parallel_allows_multiple_independent_vision_calls() -> None:
189
- calls = parallel_tools.normalize_parallel_tool_calls(
190
- [
191
- {"tool_name": "vision_load", "tool_args": {"paths": ["/tmp/a.png"]}},
192
- {"tool_name": "vision_load", "tool_args": {"paths": ["/tmp/b.png"]}},
193
- ]
194
- )
195
-
196
- assert [call.tool_name for call in calls] == ["vision_load", "vision_load"]
197
-
198
-
163
def test_subordinate_prompts_share_reusable_tree_contract() -> None:
164
call_prompt = (PROJECT_ROOT / "prompts/agent.system.tool.call_sub.md").read_text(
165
encoding="utf-8"
tests/test_vision_load_image_refs.py
+23
-3
@@ -217,6 +217,8 @@ async def test_vision_sidecar_sends_multiple_images_once_and_keeps_history_text_
217
content = calls[0]["messages"][1].content
218
assert content[0] == {"type": "text", "text": "Compare the login errors."}
219
assert [item["type"] for item in content].count("image_url") == 2
220
+ assert "max_tokens" not in calls[0]
221
+ assert "explicit_caching" not in calls[0]
222
assert "fixes the red login error" in response.message
223
assert response.message != "dummy"
224
assert raw_messages == []
@@ -237,16 +239,31 @@ async def test_parallel_worker_consumes_parent_ephemeral_image(monkeypatch, tmp_
239
240
monkeypatch.setattr(vision_load_module.chat_media.files, "get_abs_path", fake_get_abs_path)
241
monkeypatch.setattr(vision_load_module.chat_media.files, "normalize_a0_path", fake_normalize_a0_path)
240
- monkeypatch.setattr(vision_load_module.VisionLoad, "_config_agent", lambda self: self.agent)
242
+ parent_id = "parent-vision"
243
+ parent_agent = SimpleNamespace(
244
+ context=SimpleNamespace(id=parent_id),
245
+ agent_name="Parent Agent",
246
+ )
247
+ parent_context = SimpleNamespace(agent0=parent_agent)
248
+ agent_stub = types.ModuleType("agent")
249
+ agent_stub.AgentContext = SimpleNamespace(
250
+ get=lambda context_id: parent_context if context_id == parent_id else None
251
+ )
252
+ monkeypatch.setitem(sys.modules, "agent", agent_stub)
253
+ config_owners = []
254
+
255
+ def get_chat_config(owner):
256
+ config_owners.append(owner)
257
+ return {"vision": True, "max_embeds": 10}
258
+
259
monkeypatch.setattr(
260
vision_load_module,
261
"get_chat_model_config",
244
- lambda _agent: {"vision": True, "max_embeds": 10},
262
+ get_chat_config,
263
)
264
monkeypatch.setattr(vision_load_module, "get_vision_model_config", lambda _agent: {})
265
monkeypatch.setattr(vision_load_module, "use_vision_sidecar", lambda _agent: False)
266
249
- parent_id = "parent-vision"
267
ref = vision_load_module.ephemeral_images.put_image_bytes(
268
context_id=parent_id,
269
mime="image/png",
@@ -271,6 +288,9 @@ async def test_parallel_worker_consumes_parent_ephemeral_image(monkeypatch, tmp_
288
289
await tool.execute(paths=[ref])
290
291
+ assert tool._config_owner is parent_agent
292
+ assert config_owners and all(owner is parent_agent for owner in config_owners)
293
+ assert tool._context_id() == parent_id
294
assert tool.loaded_paths == ["shot.png"]
295
assert vision_load_module.ephemeral_images.get_image(ref, context_id=parent_id) is None
296
stored_ref = tool.images_dict["shot.png"]
tools/vision_load.py
+9
-28
@@ -1,4 +1,3 @@
1
-import asyncio
1
import json
2
from mimetypes import guess_type
3
@@ -16,7 +15,6 @@ from plugins._model_config.helpers.model_config import (
15
)
16
17
TOKENS_ESTIMATE = 1500
19
-VISION_TIMEOUT_SECONDS = 300
18
VISION_SYSTEM_PROMPT = (
19
"You are a precise vision analyst. Answer only what was asked about the images. "
20
"Be concise and factual. Preserve exact visible text when asked to read it."
@@ -93,7 +91,8 @@ class VisionLoad(Tool):
91
if self._delegated and self.images_dict:
92
try:
93
capsule = await self._call_vision_model(
96
- list(self.images_dict.values()), self._query(query, kwargs)
94
+ list(self.images_dict.values()),
95
+ str(query or "").strip() or DEFAULT_VISION_QUERY,
96
)
97
message = (
98
f"Vision Model analyzed {len(self.images_dict)} image(s)"
@@ -163,12 +162,8 @@ class VisionLoad(Tool):
162
return 10
163
164
def _context_id(self) -> str:
166
- context = getattr(self.agent, "context", None)
167
- if not context:
168
- return ""
169
- get_data = getattr(context, "get_data", None)
170
- parent_id = get_data(PARALLEL_WORKER_PARENT_CONTEXT_KEY) if get_data else ""
171
- return str(parent_id or getattr(context, "id", "") or "").strip()
165
+ context = getattr(self._config_owner, "context", None)
166
+ return str(getattr(context, "id", "") or "").strip()
167
168
def _config_agent(self):
169
context = getattr(self.agent, "context", None)
@@ -189,16 +184,11 @@ class VisionLoad(Tool):
184
{"type": "image_url", "image_url": {"url": path}}
185
for path in image_paths
186
)
192
- response, _ = await asyncio.wait_for(
193
- model.unified_call(
194
- messages=[
195
- SystemMessage(content=VISION_SYSTEM_PROMPT),
196
- HumanMessage(content=content),
197
- ],
198
- explicit_caching=False,
199
- max_tokens=2000,
200
- ),
201
- timeout=VISION_TIMEOUT_SECONDS,
187
+ response, _ = await model.unified_call(
188
+ messages=[
189
+ SystemMessage(content=VISION_SYSTEM_PROMPT),
190
+ HumanMessage(content=content),
191
+ ],
192
)
193
if not str(response or "").strip():
194
raise RuntimeError("Vision Model returned an empty response.")
@@ -266,15 +256,6 @@ class VisionLoad(Tool):
256
return "vision_load error: `paths` must be an array of image paths."
257
return [str(path or "").strip() for path in paths]
258
269
- @staticmethod
270
- def _query(query: str, kwargs: dict) -> str:
271
- if str(query or "").strip():
272
- return str(query).strip()
273
- for key in ("prompt", "question", "instruction", "focus", "request"):
274
- if str(kwargs.get(key) or "").strip():
275
- return str(kwargs[key]).strip()
276
- return DEFAULT_VISION_QUERY
277
-
259
@staticmethod
260
def _is_data_image_url(value: str) -> bool:
261
normalized = str(value or "").strip().lower()
tools/vision_load.py.dox.md
+2
-1
@@ -25,6 +25,7 @@
25
- Delegated history contains the text capsule only. Native history contains the tool result followed by one raw message holding all loaded image blocks.
26
- Direct parallel workers resolve ephemeral refs, model routing, and durable chat-media storage against their recorded parent context; independent vision jobs remain generic parallel jobs.
27
- `max_embeds` comes from the model that actually receives the images.
28
+- Vision Model calls use the selected model's Advanced `kwargs`; this tool does not impose a separate timeout or output-token limit.
29
- Update this file whenever tool arguments, output shape, `break_loop` behavior, intervention handling, prompt instructions, or side effects change.
30
- `VisionLoad` is a `Tool`.
31
- `VisionLoad` defines `execute(...)`.
@@ -33,7 +34,7 @@
34
35
## Key Concepts
36
36
-- Important called helpers/classes observed in the source: `build_vision_model`, `use_vision_sidecar`, `self._get_max_embeds`, `Response`, `self._context_id`, `chat_media.save_image_base64`, `chat_media.save_image_data_url`, `chat_media.materialize_image_ref`, `ephemeral_images.consume_image`, `images.to_data_url`, `self.agent.hist_add_tool_result`, `history.RawMessage`.
37
+- Important called helpers/classes observed in the source: `build_vision_model`, `use_vision_sidecar`, `self._get_max_embeds`, `Response`, `self._context_id`, `chat_media.save_image_base64`, `chat_media.save_image_data_url`, `chat_media.materialize_image_ref`, `ephemeral_images.consume_image`, `images.to_data_url`, `self.agent.hist_add_tool_result`, `history.RawMessage`, `model.unified_call`.
38
- Keep request/response, tool, or helper semantics documented here at the same time as source changes.
39
40
## Work Guidance