@samitouri / QOSami_Agents_Zero / commits / 2c09a4ed

Simplify Vision Model routing and execution

Alessandro committed Aug 25, 2026 at 20:20 UTC 2c09a4ed7241d9a2966f14baccda2d39cd0dcc9a
12 files changed +162 -263
extensions/python/system_prompt/_11_tools_prompt.py
+3 -6
@@ -53,14 +53,11 @@ async def build_prompt(agent: Agent) -> str:
53 # vision support
54 from plugins._model_config.helpers.model_config import (
55 get_chat_model_config,
56 - use_vision_sidecar,
56 + get_vision_model_config,
57 )
58
59 chat_cfg = get_chat_model_config(agent)
60 - sidecar = use_vision_sidecar(agent)
61 - if sidecar or chat_cfg.get("vision", False):
62 - prompt += "\n\n" + agent.read_prompt(
63 - "agent.system.tools_vision.md", sidecar=sidecar
64 - )
60 + if get_vision_model_config(agent) or chat_cfg.get("vision", False):
61 + prompt += "\n\n" + agent.read_prompt("agent.system.tools_vision.md")
62
63 return prompt
helpers/responses_tools.py
+6 -4
@@ -116,13 +116,15 @@ def _vision_tool_prompt(agent: Any) -> str:
116 try:
117 from plugins._model_config.helpers.model_config import (
118 get_chat_model_config,
119 - use_vision_sidecar,
119 + get_vision_model_config,
120 )
121
122 - sidecar = use_vision_sidecar(agent)
123 - if not (sidecar or get_chat_model_config(agent).get("vision", False)):
122 + if not (
123 + get_vision_model_config(agent)
124 + or get_chat_model_config(agent).get("vision", False)
125 + ):
126 return ""
125 - return agent.read_prompt("agent.system.tools_vision.md", sidecar=sidecar)
127 + return agent.read_prompt("agent.system.tools_vision.md")
128 except Exception:
129 return ""
130
plugins/_model_config/AGENTS.md
+1 -1
@@ -23,7 +23,7 @@
23 - Coordinate OAuth-backed providers with `_oauth` instead of hardcoding provider-specific auth here.
24 - `model_config_get` exposes `model_configured` as a derived chat-model readiness flag from provider, model name, and API-key availability.
25 - Non-default presets may inherit omitted main, utility, or embedding slots and durable tuning from `Default`, but must replace or clear per-slot `kwargs` so provider-specific extra params never leak across model providers.
26 -- The optional `vision` slot is strictly per preset and never inherited from `Default`; an empty slot disables sidecar vision for that preset.
26 +- The optional `vision` slot is strictly per preset and never inherited from `Default`; an empty slot disables the separate Vision Model for that preset.
27 - Main native vision wins by default. A configured Vision Model handles `vision_load` when Main lacks vision, or when that preset explicitly enables `override_main`.
28 - Keep the optional Vision provider/model selector inside the Main Model card and flush with Main's field alignment, without a nested left inset. Show it only while Main vision is disabled or `override_main` is enabled; do not render a standalone Vision Model card.
29 - Show `Use separate Vision Model` immediately below `Supports Vision` while Main vision is enabled, not inside Advanced Settings; describe the disabled state as using Main's native vision.
plugins/_model_config/helpers/model_config.py
+14 -16
@@ -759,20 +759,17 @@ def get_chat_model_config(agent=None) -> dict:
759
760
761 def get_vision_model_config(agent=None) -> dict:
762 - """Get the optional, strictly per-preset Vision Model config."""
763 - return get_effective_config(agent).get("vision_model", {})
764 -
765 -
766 -def use_vision_sidecar(agent=None) -> bool:
767 - """Return whether vision_load should delegate to the preset's Vision Model."""
768 - vision_cfg = get_vision_model_config(agent)
769 - if not (
770 - str(vision_cfg.get("provider") or "").strip()
771 - and str(vision_cfg.get("name") or "").strip()
772 - ):
773 - return False
774 - chat_cfg = get_chat_model_config(agent)
775 - return not bool(chat_cfg.get("vision")) or bool(vision_cfg.get("override_main"))
762 + """Get the active Vision Model config after applying Main-first routing."""
763 + cfg = get_effective_config(agent)
764 + vision_cfg = cfg.get("vision_model", {})
765 + if not _slot_has_identity(vision_cfg):
766 + return {}
767 + chat_cfg = cfg.get("chat_model", {})
768 + return (
769 + vision_cfg
770 + if not chat_cfg.get("vision") or vision_cfg.get("override_main")
771 + else {}
772 + )
773
774
775 def get_utility_model_config(agent=None) -> dict:
@@ -928,8 +925,9 @@ def get_missing_api_key_providers(agent=None) -> list[dict]:
925 ("Utility Model", cfg.get("utility_model", {})),
926 ("Embedding Model", get_embedding_model_config(agent)),
927 ]
931 - if use_vision_sidecar(agent):
932 - checks.insert(1, ("Vision Model", cfg.get("vision_model", {})))
928 + vision_cfg = get_vision_model_config(agent)
929 + if vision_cfg:
930 + checks.insert(1, ("Vision Model", vision_cfg))
931
932 for label, model_cfg in checks:
933 provider = model_cfg.get("provider", "")
prompts/AGENTS.md
+1 -1
@@ -25,7 +25,7 @@
25 - Read the rendering path before changing placeholders or filenames.
26 - Prefer small prompt additions over broad rewrites when fixing a specific behavior.
27 - Keep document/OCR routing explicit: image files, screenshots, scans, charts, photos, and diagrams should prefer vision tools when available, while `document_query` is for documents, large text-heavy files, and fallback OCR.
28 -- Keep the single `vision_load` prompt accurate for both native and sidecar routing: related images belong in one call, while a sidecar result is text-only unless Main explicitly requests its native raw route.
28 +- Keep the single `vision_load` prompt accurate for both native and separate Vision Model routing: related images belong in one call, and a Vision Model result is text-only.
29 - Update tests or snapshots when prompt budget, required sections, or generated system content changes.
30
31 ## Verification
prompts/agent.system.tools_vision.md
+7 -22
@@ -1,10 +1,8 @@
1 ## multimodal vision tools
2
3 ### vision_load
4 -{{if sidecar}}analyze images with the preset's separate Vision Model and return a text result{{endif}}
5 -{{if not sidecar}}load images into Main for visual reasoning{{endif}}
6 -args: `paths` list of absolute image paths or ephemeral image refs{{if sidecar}}, `query` optional focused instruction, `raw` optional boolean{{endif}}
7 -{{if sidecar}}
4 +load or analyze images for visual reasoning
5 +args: `paths` list of absolute image paths or ephemeral image refs, `query` optional focused instruction
6 Input schema for tool_args:
7 ```json
8 {
@@ -12,32 +10,20 @@ Input schema for tool_args:
10 "properties": {
11 "paths": {
12 "type": "array",
15 - "items": {"type": "string"},
16 - "description": "Absolute image paths or ephemeral image refs."
13 + "items": {"type": "string"}
14 },
18 - "query": {
19 - "type": "string",
20 - "description": "What the Vision Model should inspect, compare, locate, or read."
21 - },
22 - "raw": {
23 - "type": "boolean",
24 - "description": "Use Main's native vision instead of the separate Vision Model, when Main supports vision."
25 - }
15 + "query": {"type": "string"}
16 },
17 "required": ["paths"],
18 "additionalProperties": false
19 }
20 ```
31 -{{endif}}
21 rules:
22 - put all images needed for one comparison or visual task in the same `paths` array
23 - use when the task depends on screenshots, diagrams, scanned documents, charts, or photos
35 -{{if sidecar}}
36 -- when the separate Vision Model is active, use a focused `query`; the result is a text capsule and Main does not receive raw images
37 -- use `raw=true` only when Main supports vision and must inspect the pixels itself
38 -{{endif}}
24 +- use a focused `query` when asking the configured Vision Model to inspect, compare, locate, or read something
25 - only bitmaps are supported; convert other formats first if needed
40 -- the tool result includes loaded/skipped image totals and the corresponding path lists
26 +- the tool result reports loaded and skipped image counts
27 example:
28 ```json
29 {
@@ -47,9 +33,8 @@ example:
33 "headline": "Comparing screenshots",
34 "tool_name": "vision_load",
35 "tool_args": {
50 - "paths": ["/path/to/before.png", "/path/to/after.png"]{{if sidecar}},
36 + "paths": ["/path/to/before.png", "/path/to/after.png"],
37 "query": "Compare the error banners and describe what changed."
52 - {{endif}}
38 }
39 }
40 ```
tests/test_browser_agent_regressions.py
+10 -3
@@ -63,6 +63,15 @@ class _TestTool:
63 self.message = message
64 self.loop_data = loop_data
65
66 + async def after_execution(self, response, **kwargs):
67 + self.agent.hist_add_tool_result(
68 + self.name,
69 + response.message.strip(),
70 + id=self.log.id,
71 + **(response.additional or {}),
72 + )
73 + self.log.update(content=response.message.strip())
74 +
75
76 class _TestWsHandler:
77 def __init__(self, *args, **kwargs):
@@ -102,7 +111,6 @@ _model_config_stub.get_presets = lambda: []
111 _model_config_stub.get_preset_by_name = lambda name: None
112 _model_config_stub.get_chat_model_config = lambda agent=None: {}
113 _model_config_stub.get_vision_model_config = lambda agent=None: {}
105 -_model_config_stub.use_vision_sidecar = lambda agent=None: False
114 _model_config_stub.build_vision_model = lambda agent=None: None
115 sys.modules.setdefault("plugins._model_config.helpers.model_config", _model_config_stub)
116
@@ -4307,7 +4315,6 @@ async def test_vision_load_materializes_ephemeral_browser_refs(monkeypatch, tmp_
4315 monkeypatch.setattr(vision_load_module.chat_media.files, "normalize_a0_path", fake_normalize_a0_path)
4316 monkeypatch.setattr(vision_load_module, "get_chat_model_config", lambda _agent: {"vision": True, "max_embeds": 10})
4317 monkeypatch.setattr(vision_load_module, "get_vision_model_config", lambda _agent: {})
4310 - monkeypatch.setattr(vision_load_module, "use_vision_sidecar", lambda _agent: False)
4318
4319 tool_results = []
4320 messages = []
@@ -4344,7 +4351,7 @@ async def test_vision_load_materializes_ephemeral_browser_refs(monkeypatch, tmp_
4351 assert stored_ref.startswith("/a0/usr/chats/ctx-vision/screenshots/browser/browser-shot-")
4352 stored_path = tmp_path / stored_ref.removeprefix("/a0/")
4353 assert stored_path.read_bytes() == __import__("base64").b64decode(SMALL_JPEG_10X10)
4347 - assert updates[-1]["result"] == "1 images loaded, 0 skipped"
4354 + assert updates[-1]["content"] == response.message
4355
4356
4357 @pytest.mark.anyio
tests/test_model_config_api_keys.py
+2 -4
@@ -93,7 +93,7 @@ def test_missing_api_key_checks_only_the_active_vision_model(monkeypatch):
93 from plugins._model_config.helpers import model_config
94
95 config = {
96 - "chat_model": {"provider": "ollama", "name": "text-main"},
96 + "chat_model": {"provider": "ollama", "name": "text-main", "vision": True},
97 "vision_model": {"provider": "openai", "name": "vision"},
98 "utility_model": {"provider": "ollama", "name": "utility"},
99 "embedding_model": {
@@ -101,19 +101,17 @@ def test_missing_api_key_checks_only_the_active_vision_model(monkeypatch):
101 "name": "sentence-transformers/all-MiniLM-L6-v2",
102 },
103 }
104 - active = False
104 monkeypatch.setattr(model_config, "get_effective_config", lambda _agent=None: config)
105 monkeypatch.setattr(
106 model_config,
107 "get_embedding_model_config",
108 lambda _agent=None: config["embedding_model"],
109 )
111 - monkeypatch.setattr(model_config, "use_vision_sidecar", lambda _agent=None: active)
110 monkeypatch.setattr(model_config, "has_provider_api_key", lambda *args: False)
111
112 assert model_config.get_missing_api_key_providers() == []
113
116 - active = True
114 + config["chat_model"]["vision"] = False
115 assert model_config.get_missing_api_key_providers() == [
116 {"model_type": "Vision Model", "provider": "openai"}
117 ]
tests/test_tool_policy.py
+13 -17
@@ -8,7 +8,7 @@ from types import SimpleNamespace
8 import pytest
9
10 from extensions.python.system_prompt import _11_tools_prompt, _13_skills_prompt
11 -from helpers import files, mcp_handler, responses_tools, tool_policy
11 +from helpers import mcp_handler, responses_tools, tool_policy
12 from helpers.errors import RepairableException
13 from plugins._tool_access.extensions.python.tool_execute_before._10_enforce_tool_policy import (
14 EnforceToolPolicy,
@@ -528,6 +528,10 @@ async def test_vision_tool_follows_chat_config_not_profile_policy(
528 "plugins._model_config.helpers.model_config.get_chat_model_config",
529 lambda agent: {"vision": True},
530 )
531 + monkeypatch.setattr(
532 + "plugins._model_config.helpers.model_config.get_vision_model_config",
533 + lambda agent: {},
534 + )
535 monkeypatch.setattr(
536 tool_policy,
537 "get_policy",
@@ -547,7 +551,7 @@ async def test_vision_tool_follows_chat_config_not_profile_policy(
551
552
553 @pytest.mark.asyncio
550 -async def test_vision_sidecar_uses_canonical_vision_prompt(
554 +async def test_active_vision_model_uses_canonical_vision_prompt(
555 monkeypatch, tmp_path: Path
556 ) -> None:
557 _write_prompt(tmp_path, "agent.system.tools.md", "TOOLS\n{{tools}}")
@@ -559,11 +563,11 @@ async def test_vision_sidecar_uses_canonical_vision_prompt(
563 monkeypatch.setattr(tool_policy.subagents, "get_paths", _prompt_paths(tmp_path))
564 monkeypatch.setattr(
565 "plugins._model_config.helpers.model_config.get_chat_model_config",
562 - lambda agent: {"vision": True},
566 + lambda agent: {"vision": False},
567 )
568 monkeypatch.setattr(
565 - "plugins._model_config.helpers.model_config.use_vision_sidecar",
566 - lambda agent: True,
569 + "plugins._model_config.helpers.model_config.get_vision_model_config",
570 + lambda agent: {"provider": "test", "name": "vision"},
571 )
572 monkeypatch.setattr(responses_tools, "_mcp_tools", lambda agent: [])
573 agent = _Agent(tmp_path)
@@ -582,24 +586,16 @@ def test_vision_prompt_declares_multi_image_native_schema() -> None:
586 / "prompts"
587 / "agent.system.tools_vision.md"
588 ).read_text(encoding="utf-8")
585 - sidecar_prompt = files.evaluate_text_conditions(source, sidecar=True)
586 - native_prompt = files.evaluate_text_conditions(source, sidecar=False)
587 - schema = responses_tools._schema_from_prompt(sidecar_prompt)
589 + schema = responses_tools._schema_from_prompt(source)
590
591 assert schema["required"] == ["paths"]
592 assert schema["properties"]["paths"] == {
593 "type": "array",
594 "items": {"type": "string"},
593 - "description": "Absolute image paths or ephemeral image refs.",
594 - }
595 - assert {"query", "raw"} <= schema["properties"].keys()
596 - assert "separate Vision Model" in sidecar_prompt
597 - assert "separate Vision Model" not in native_prompt
598 - assert responses_tools._schema_from_prompt(native_prompt) == {
599 - "type": "object",
600 - "properties": {},
601 - "additionalProperties": True,
595 }
596 + assert "query" in schema["properties"]
597 + assert "raw" not in schema["properties"]
598 + assert "Vision Model" in source
599
600
601 def test_mcp_prompt_and_native_schema_omit_blocked_tool(
tests/test_vision_load_image_refs.py
+19 -29
@@ -41,6 +41,15 @@ class _TestTool:
41 self.message = message
42 self.loop_data = loop_data
43
44 + async def after_execution(self, response, **kwargs):
45 + self.agent.hist_add_tool_result(
46 + self.name,
47 + response.message.strip(),
48 + id=self.log.id,
49 + **(response.additional or {}),
50 + )
51 + self.log.update(content=response.message.strip())
52 +
53
54 def _install_tool_stub(monkeypatch):
55 tool_stub = types.ModuleType("helpers.tool")
@@ -82,7 +91,6 @@ async def test_vision_load_materializes_local_image_to_chat_artifact(monkeypatch
91 monkeypatch.setattr(vision_load_module.chat_media.files, "normalize_a0_path", fake_normalize_a0_path)
92 monkeypatch.setattr(vision_load_module, "get_chat_model_config", lambda _agent: {"vision": True, "max_embeds": 10})
93 monkeypatch.setattr(vision_load_module, "get_vision_model_config", lambda _agent: {})
85 - monkeypatch.setattr(vision_load_module, "use_vision_sidecar", lambda _agent: False)
94
95 async def direct_call(func, *args, **kwargs):
96 return func(*args, **kwargs)
@@ -115,6 +123,9 @@ async def test_vision_load_materializes_local_image_to_chat_artifact(monkeypatch
123 )
124 tool.log = SimpleNamespace(id="vision-log", update=lambda **kwargs: updates.append(kwargs))
125
126 + invalid = await tool.execute(paths=None)
127 + assert invalid.message == "vision_load error: `paths` must be an array."
128 +
129 response = await tool.execute(paths=[str(image_path)])
130 image_path.unlink()
131 await tool.after_execution(response)
@@ -124,10 +135,10 @@ async def test_vision_load_materializes_local_image_to_chat_artifact(monkeypatch
135 assert stored_ref.startswith("/a0/usr/chats/ctx-vision/images/vision-load/sample-image-")
136 stored_path = tmp_path / stored_ref.removeprefix("/a0/")
137 assert stored_path.read_bytes() == b"png-data"
127 - assert updates[-1]["result"] == "1 images loaded, 0 skipped"
138 + assert updates[-1]["content"] == response.message
139
140
130 -def test_vision_sidecar_route_matrix_prefers_main_native_vision(monkeypatch):
141 +def test_active_vision_model_route_prefers_main_native_vision(monkeypatch):
142 from plugins._model_config.helpers import model_config
143
144 cases = [
@@ -149,11 +160,11 @@ def test_vision_sidecar_route_matrix_prefers_main_native_vision(monkeypatch):
160 "vision_model": vision,
161 },
162 )
152 - assert model_config.use_vision_sidecar() is expected
163 + assert bool(model_config.get_vision_model_config()) is expected
164
165
166 @pytest.mark.anyio
156 -async def test_vision_sidecar_sends_multiple_images_once_and_keeps_history_text_only(
167 +async def test_vision_model_sends_multiple_images_once_and_keeps_history_text_only(
168 monkeypatch,
169 tmp_path,
170 ):
@@ -182,7 +193,6 @@ async def test_vision_sidecar_sends_multiple_images_once_and_keeps_history_text_
193 "get_vision_model_config",
194 lambda _agent: {"provider": "test", "name": "vision", "max_embeds": 5},
195 )
185 - monkeypatch.setattr(vision_load_module, "use_vision_sidecar", lambda _agent: True)
196
197 image_paths = [tmp_path / "before.png", tmp_path / "after.png"]
198 for path in image_paths:
@@ -214,7 +224,7 @@ async def test_vision_sidecar_sends_multiple_images_once_and_keeps_history_text_
224 await tool.after_execution(response)
225
226 assert len(calls) == 1
217 - content = calls[0]["messages"][1].content
227 + content = calls[0]["messages"][0].content
228 assert content[0] == {"type": "text", "text": "Compare the login errors."}
229 assert [item["type"] for item in content].count("image_url") == 2
230 assert "max_tokens" not in calls[0]
@@ -240,29 +250,12 @@ async def test_parallel_worker_consumes_parent_ephemeral_image(monkeypatch, tmp_
250 monkeypatch.setattr(vision_load_module.chat_media.files, "get_abs_path", fake_get_abs_path)
251 monkeypatch.setattr(vision_load_module.chat_media.files, "normalize_a0_path", fake_normalize_a0_path)
252 parent_id = "parent-vision"
243 - parent_agent = SimpleNamespace(
244 - context=SimpleNamespace(id=parent_id),
245 - agent_name="Parent Agent",
246 - )
247 - parent_context = SimpleNamespace(agent0=parent_agent)
248 - agent_stub = types.ModuleType("agent")
249 - agent_stub.AgentContext = SimpleNamespace(
250 - get=lambda context_id: parent_context if context_id == parent_id else None
251 - )
252 - monkeypatch.setitem(sys.modules, "agent", agent_stub)
253 - config_owners = []
254 -
255 - def get_chat_config(owner):
256 - config_owners.append(owner)
257 - return {"vision": True, "max_embeds": 10}
258 -
253 monkeypatch.setattr(
254 vision_load_module,
255 "get_chat_model_config",
262 - get_chat_config,
256 + lambda _agent: {"vision": True, "max_embeds": 10},
257 )
258 monkeypatch.setattr(vision_load_module, "get_vision_model_config", lambda _agent: {})
265 - monkeypatch.setattr(vision_load_module, "use_vision_sidecar", lambda _agent: False)
259
260 ref = vision_load_module.ephemeral_images.put_image_bytes(
261 context_id=parent_id,
@@ -288,8 +281,6 @@ async def test_parallel_worker_consumes_parent_ephemeral_image(monkeypatch, tmp_
281
282 await tool.execute(paths=[ref])
283
291 - assert tool._config_owner is parent_agent
292 - assert config_owners and all(owner is parent_agent for owner in config_owners)
284 assert tool._context_id() == parent_id
285 assert tool.loaded_paths == ["shot.png"]
286 assert vision_load_module.ephemeral_images.get_image(ref, context_id=parent_id) is None
@@ -298,7 +289,7 @@ async def test_parallel_worker_consumes_parent_ephemeral_image(monkeypatch, tmp_
289
290
291 @pytest.mark.anyio
301 -async def test_independent_vision_sidecar_calls_can_run_concurrently(monkeypatch, tmp_path):
292 +async def test_independent_vision_model_calls_can_run_concurrently(monkeypatch, tmp_path):
293 _install_tool_stub(monkeypatch)
294 import tools.vision_load as vision_load_module
295
@@ -327,7 +318,6 @@ async def test_independent_vision_sidecar_calls_can_run_concurrently(monkeypatch
318 "get_vision_model_config",
319 lambda _agent: {"provider": "test", "name": "vision", "max_embeds": 10},
320 )
330 - monkeypatch.setattr(vision_load_module, "use_vision_sidecar", lambda _agent: True)
321
322 image_paths = [tmp_path / "one.png", tmp_path / "two.png"]
323 for path in image_paths:
tools/vision_load.py
+80 -154
@@ -1,24 +1,18 @@
1 -import json
1 from mimetypes import guess_type
2
4 -from langchain_core.messages import HumanMessage, SystemMessage
3 +from langchain_core.messages import HumanMessage
4
5 from helpers import chat_media, ephemeral_images, files, history, images, runtime
7 -from helpers.parallel_tools import PARALLEL_WORKER_PARENT_CONTEXT_KEY, coerce_bool
8 -from helpers.print_style import PrintStyle
6 +from helpers.parallel_tools import PARALLEL_WORKER_PARENT_CONTEXT_KEY
7 from helpers.tool import Response, Tool
8 from plugins._model_config.helpers.model_config import (
9 build_vision_model,
10 get_chat_model_config,
11 get_vision_model_config,
14 - use_vision_sidecar,
12 )
13
14 +# image token estimation for context window
15 TOKENS_ESTIMATE = 1500
18 -VISION_SYSTEM_PROMPT = (
19 - "You are a precise vision analyst. Answer only what was asked about the images. "
20 - "Be concise and factual. Preserve exact visible text when asked to read it."
21 -)
16 DEFAULT_VISION_QUERY = (
17 "Describe the images precisely, including the key objects, visible text, and layout."
18 )
@@ -26,172 +20,99 @@ DEFAULT_VISION_QUERY = (
20
21 class VisionLoad(Tool):
22 async def execute(
29 - self,
30 - paths: list[str] | str | None = None,
31 - query: str = "",
32 - raw: bool = False,
33 - **kwargs,
23 + self, paths: list[str] = [], query: str = "", **kwargs
24 ) -> Response:
35 - self.images_dict: dict[str, str] = {}
25 +
26 + self.images_dict = {}
27 self.loaded_paths: list[str] = []
28 self.skipped_paths: list[str] = []
38 - self._config_owner = self._config_agent()
39 - self._main_has_vision = bool(
40 - get_chat_model_config(self._config_owner).get("vision", False)
41 - )
42 - self._delegated = use_vision_sidecar(self._config_owner) and not (
43 - coerce_bool(raw, False) and self._main_has_vision
44 - )
45 - self._max_embeds = self._get_max_embeds()
46 -
47 - normalized = self._normalize_paths(paths)
48 - if isinstance(normalized, str):
49 - self._history_result = normalized
50 - return Response(message=normalized, break_loop=False)
29 + self.vision_config = get_vision_model_config(self.agent)
30 + if not isinstance(paths, list):
31 + return Response(
32 + message="vision_load error: `paths` must be an array.",
33 + break_loop=False,
34 + )
35
36 + max_embeds = self._get_max_embeds()
37 requested = [
53 - (path.strip(), self._display_input_path(path.strip(), index + 1))
54 - for index, path in enumerate(normalized)
38 + (str(path or "").strip(), self._display_input_path(str(path or "").strip(), idx + 1))
39 + for idx, path in enumerate(paths)
40 ]
56 - limited = requested if self._max_embeds <= 0 else requested[-self._max_embeds :]
57 - if self._max_embeds > 0 and len(requested) > self._max_embeds:
58 - self.skipped_paths = [display for _, display in requested[: -self._max_embeds]]
41 + limited_paths = requested if max_embeds <= 0 else requested[-max_embeds:]
42 + self.skipped_paths = (
43 + [display for _, display in requested[:-max_embeds]]
44 + if max_embeds > 0 and len(requested) > max_embeds
45 + else []
46 + )
47
60 - for index, (path, display_path) in enumerate(limited):
48 + for idx, (path, display_path) in enumerate(limited_paths):
49 if not path:
50 continue
51 if ephemeral_images.is_ref(path):
64 - image = ephemeral_images.consume_image(path, context_id=self._context_id())
52 + image = ephemeral_images.consume_image(
53 + path,
54 + context_id=self._context_id(),
55 + )
56 if image is None:
57 continue
67 - display_path = image.display_name or display_path
58 + display = image.display_name or display_path
59 stored_ref = self._store_ephemeral_image(image)
69 - elif self._is_data_image_url(path):
70 - stored_ref = self._store_data_url(
71 - path, preferred_name=f"vision-load-{index + 1}.png"
72 - )
73 - elif await runtime.call_development_function(files.exists, path):
74 - mime_type, _ = guess_type(path)
75 - if not mime_type or not mime_type.startswith("image/"):
76 - continue
77 - try:
78 - stored_ref = self._store_local_image(
79 - path, preferred_name=files.basename(path)
80 - )
81 - except (FileNotFoundError, OSError, ValueError):
82 - continue
83 - else:
60 + if stored_ref:
61 + self.images_dict[display] = stored_ref
62 + self.loaded_paths.append(display)
63 + continue
64 + if self._is_data_image_url(path):
65 + stored_ref = self._store_data_url(path, preferred_name=f"vision-load-{idx + 1}.png")
66 + if stored_ref:
67 + self.images_dict[display_path] = stored_ref
68 + self.loaded_paths.append(display_path)
69 + continue
70 + if not await runtime.call_development_function(files.exists, str(path)):
71 continue
72
86 - if stored_ref:
87 - self.images_dict[display_path] = stored_ref
88 - self.loaded_paths.append(display_path)
89 -
90 - summary = self._summary()
91 - if self._delegated and self.images_dict:
73 + if path not in self.images_dict:
74 + mime_type, _ = guess_type(str(path))
75 + if mime_type and mime_type.startswith("image/"):
76 + try:
77 + stored_ref = self._store_local_image(path, preferred_name=files.basename(path))
78 + self.images_dict[display_path] = stored_ref
79 + self.loaded_paths.append(display_path)
80 + except (FileNotFoundError, OSError, ValueError):
81 + continue
82 +
83 + message = self._summary() if self.images_dict or self.skipped_paths else "No images processed"
84 + if self.vision_config and self.images_dict:
85 try:
86 capsule = await self._call_vision_model(
94 - list(self.images_dict.values()),
95 - str(query or "").strip() or DEFAULT_VISION_QUERY,
87 + list(self.images_dict.values()), query
88 )
89 message = (
90 f"Vision Model analyzed {len(self.images_dict)} image(s)"
91 f"; {len(self.skipped_paths)} skipped.\n\n{capsule.strip()}"
92 )
101 - self._history_result = message
102 - return Response(message=message, break_loop=False)
93 except Exception as exc:
94 message = f"Vision Model error: {str(exc)[:1000]}"
105 - self._history_result = f"{summary}\n\n{message}"
106 - return Response(message=message, break_loop=False)
107 -
108 - if self.images_dict and not self._main_has_vision:
109 - summary += (
110 - "\n\nImages were not injected because neither Main native vision nor "
111 - "a usable Vision Model is active."
112 - )
113 - self._history_result = (
114 - summary if self.images_dict or self.skipped_paths else "No images processed"
115 - )
116 - message = (
117 - "No images processed"
118 - if not self.images_dict and not self.skipped_paths
119 - else f"{len(self.images_dict)} images loaded, {len(self.skipped_paths)} skipped"
120 - )
95 return Response(message=message, break_loop=False)
96
123 - async def after_execution(self, response: Response, **kwargs):
124 - log_id = str(getattr(getattr(self, "log", None), "id", "") or "")
125 - self.agent.hist_add_tool_result(
126 - self.name,
127 - self._history_result,
128 - id=log_id,
129 - **(response.additional or {}),
130 - )
131 -
132 - if self.images_dict and self._main_has_vision and not self._delegated:
133 - content = [
134 - {"type": "image_url", "image_url": {"url": image_path}}
135 - for image_path in self.images_dict.values()
136 - ]
137 - self.agent.hist_add_message(
138 - False,
139 - content=history.RawMessage(
140 - raw_content=content,
141 - preview="<Image attachments loaded by path>",
142 - ),
143 - tokens=TOKENS_ESTIMATE * len(content),
144 - )
145 -
146 - PrintStyle(
147 - font_color="#1B4F72", background_color="white", padding=True, bold=True
148 - ).print(f"{self.agent.agent_name}: Response from tool '{self.name}'")
149 - PrintStyle(font_color="#85C1E9").print(response.message)
150 - if getattr(self, "log", None):
151 - self.log.update(result=response.message)
152 -
97 def _get_max_embeds(self) -> int:
154 - cfg = (
155 - get_vision_model_config(self._config_owner)
156 - if self._delegated
157 - else get_chat_model_config(self._config_owner)
158 - )
159 - try:
160 - return int(cfg.get("max_embeds", 10) or 0)
161 - except (TypeError, ValueError):
162 - return 10
98 + cfg = self.vision_config or get_chat_model_config(self.agent)
99 + return int(cfg.get("max_embeds", 10) or 0)
100
101 def _context_id(self) -> str:
165 - context = getattr(self._config_owner, "context", None)
166 - return str(getattr(context, "id", "") or "").strip()
167 -
168 - def _config_agent(self):
102 context = getattr(self.agent, "context", None)
103 get_data = getattr(context, "get_data", None)
104 parent_id = get_data(PARALLEL_WORKER_PARENT_CONTEXT_KEY) if get_data else ""
172 - if parent_id:
173 - from agent import AgentContext
174 -
175 - parent = AgentContext.get(str(parent_id))
176 - if parent:
177 - return parent.agent0
178 - return self.agent
105 + return str(parent_id or getattr(context, "id", "") or "").strip()
106
107 async def _call_vision_model(self, image_paths: list[str], query: str) -> str:
181 - model = build_vision_model(self._config_owner)
182 - content = [{"type": "text", "text": query or DEFAULT_VISION_QUERY}]
108 + content = [{"type": "text", "text": str(query or "").strip() or DEFAULT_VISION_QUERY}]
109 content.extend(
110 {"type": "image_url", "image_url": {"url": path}}
111 for path in image_paths
112 )
187 - response, _ = await model.unified_call(
188 - messages=[
189 - SystemMessage(content=VISION_SYSTEM_PROMPT),
190 - HumanMessage(content=content),
191 - ],
113 + response, _ = await build_vision_model(self.agent).unified_call(
114 + messages=[HumanMessage(content=content)],
115 )
193 - if not str(response or "").strip():
194 - raise RuntimeError("Vision Model returned an empty response.")
116 return str(response)
117
118 def _store_ephemeral_image(self, image: ephemeral_images.EphemeralImage) -> str:
@@ -199,11 +120,12 @@ class VisionLoad(Tool):
120 if not context_id:
121 return image.data_url
122 source = chat_media.infer_source(image.ref, image.display_name)
123 + category = chat_media.category_for_source(source)
124 saved = chat_media.save_image_base64(
125 context_id=context_id,
126 data=image.data,
127 mime_type=image.mime,
206 - category=chat_media.category_for_source(source),
128 + category=category,
129 source=source,
130 preferred_name=image.display_name,
131 )
@@ -214,10 +136,11 @@ class VisionLoad(Tool):
136 if not context_id:
137 return data_url
138 source = chat_media.infer_source(data_url, preferred_name)
139 + category = chat_media.category_for_source(source)
140 saved = chat_media.save_image_data_url(
141 context_id=context_id,
142 data_url=data_url,
220 - category=chat_media.category_for_source(source),
143 + category=category,
144 source=source,
145 preferred_name=preferred_name,
146 )
@@ -239,23 +162,9 @@ class VisionLoad(Tool):
162 skipped = "\n".join(self.skipped_paths) if self.skipped_paths else "none"
163 return (
164 f"Loaded images ({len(self.loaded_paths)}):\n{loaded}\n\n"
242 - f"Skipped images ({len(self.skipped_paths)}, max {self._max_embeds}):\n{skipped}"
165 + f"Skipped images ({len(self.skipped_paths)}, max {self._get_max_embeds()}):\n{skipped}"
166 )
167
245 - @staticmethod
246 - def _normalize_paths(paths: list[str] | str | None) -> list[str] | str:
247 - if isinstance(paths, str):
248 - try:
249 - decoded = json.loads(paths)
250 - except json.JSONDecodeError:
251 - decoded = paths
252 - paths = decoded if isinstance(decoded, list) else [paths]
253 - if paths is None:
254 - return []
255 - if not isinstance(paths, (list, tuple)):
256 - return "vision_load error: `paths` must be an array of image paths."
257 - return [str(path or "").strip() for path in paths]
258 -
168 @staticmethod
169 def _is_data_image_url(value: str) -> bool:
170 normalized = str(value or "").strip().lower()
@@ -266,5 +175,22 @@ class VisionLoad(Tool):
175 if ephemeral_images.is_ref(value):
176 return ephemeral_images.display_ref(value)
177 if cls._is_data_image_url(value):
269 - return f"{value.split(',', 1)[0]},<ephemeral-image-{index}>"
178 + prefix = value.split(",", 1)[0]
179 + return f"{prefix},<ephemeral-image-{index}>"
180 return value
181 +
182 + async def after_execution(self, response: Response, **kwargs):
183 + await super().after_execution(response, **kwargs)
184 + if self.images_dict and not self.vision_config:
185 + content = [
186 + {"type": "image_url", "image_url": {"url": image_path}}
187 + for image_path in self.images_dict.values()
188 + ]
189 + self.agent.hist_add_message(
190 + False,
191 + content=history.RawMessage(
192 + raw_content=content,
193 + preview="<Image attachments loaded by path>",
194 + ),
195 + tokens=TOKENS_ESTIMATE * len(content),
196 + )
tools/vision_load.py.dox.md
+6 -6
@@ -12,29 +12,29 @@
12 - `vision_load.py.dox.md` owns durable notes about responsibilities, contracts, side effects, and verification for that implementation.
13 - Classes:
14 - `VisionLoad` (`Tool`)
15 - - `async execute(self, paths, query="", raw=False, **kwargs) -> Response`
15 + - `async execute(self, paths, query="", **kwargs) -> Response`
16 - `async after_execution(self, response: Response, **kwargs)`
17 - Notable constants/configuration names: `TOKENS_ESTIMATE`.
18
19 ## Runtime Contracts
20
21 - Tool modules must define `helpers.tool.Tool` subclasses and return `helpers.tool.Response` from `execute(...)`.
22 -- One call may contain multiple paths. The delegated route sends every selected path in one Vision Model request and returns one textual capsule.
23 -- Main native vision wins unless the effective preset selects the sidecar route; `raw=true` returns to Main native vision only when Main supports it.
22 +- One call may contain multiple paths. The Vision Model route sends every selected path in one request and returns one textual capsule.
23 +- Model configuration exposes a Vision Model only when the effective preset selects that route; otherwise this tool follows Main's native vision path.
24 - Delegation completes during `execute(...)` so native Responses function output contains the real capsule before `after_execution(...)` persists it.
25 - Delegated history contains the text capsule only. Native history contains the tool result followed by one raw message holding all loaded image blocks.
26 -- Direct parallel workers resolve ephemeral refs, model routing, and durable chat-media storage against their recorded parent context; independent vision jobs remain generic parallel jobs.
26 +- Direct parallel workers inherit the parent's model override generically. This tool uses their recorded parent context only to resolve ephemeral refs and durable chat media.
27 - `max_embeds` comes from the model that actually receives the images.
28 - Vision Model calls use the selected model's Advanced `kwargs`; this tool does not impose a separate timeout or output-token limit.
29 - Update this file whenever tool arguments, output shape, `break_loop` behavior, intervention handling, prompt instructions, or side effects change.
30 - `VisionLoad` is a `Tool`.
31 - `VisionLoad` defines `execute(...)`.
32 - Observed side-effect areas: filesystem writes, model calls, plugin state, settings/state persistence, secret handling.
33 -- Imported dependency areas include: `helpers`, `helpers.print_style`, `helpers.tool`, `mimetypes`.
33 +- Imported dependency areas include: `helpers`, `helpers.tool`, `langchain_core.messages`, `mimetypes`, and `_model_config`.
34
35 ## Key Concepts
36
37 -- Important called helpers/classes observed in the source: `build_vision_model`, `use_vision_sidecar`, `self._get_max_embeds`, `Response`, `self._context_id`, `chat_media.save_image_base64`, `chat_media.save_image_data_url`, `chat_media.materialize_image_ref`, `ephemeral_images.consume_image`, `images.to_data_url`, `self.agent.hist_add_tool_result`, `history.RawMessage`, `model.unified_call`.
37 +- Important called helpers/classes observed in the source: `build_vision_model`, `get_vision_model_config`, `self._get_max_embeds`, `Response`, `self._context_id`, `chat_media.save_image_base64`, `chat_media.save_image_data_url`, `chat_media.materialize_image_ref`, `ephemeral_images.consume_image`, `images.to_data_url`, `history.RawMessage`, `super().after_execution`, `model.unified_call`.
38 - Keep request/response, tool, or helper semantics documented here at the same time as source changes.
39
40 ## Work Guidance