| 1 | import asyncio |
| 2 | import types |
| 3 | from types import SimpleNamespace |
| 4 | import sys |
| 5 | from pathlib import Path |
| 6 | |
| 7 | import pytest |
| 8 | |
| 9 | PROJECT_ROOT = Path(__file__).resolve().parents[1] |
| 10 | if str(PROJECT_ROOT) not in sys.path: |
| 11 | sys.path.insert(0, str(PROJECT_ROOT)) |
| 12 | |
| 13 | from helpers import images |
| 14 | |
| 15 | |
| 16 | class _TestResponse(SimpleNamespace): |
| 17 | def __init__(self, message="", break_loop=False, additional=None, **kwargs): |
| 18 | super().__init__( |
| 19 | message=message, |
| 20 | break_loop=break_loop, |
| 21 | additional=additional, |
| 22 | **kwargs, |
| 23 | ) |
| 24 | |
| 25 | |
| 26 | class _TestTool: |
| 27 | def __init__( |
| 28 | self, |
| 29 | agent=None, |
| 30 | name="", |
| 31 | method=None, |
| 32 | args=None, |
| 33 | message="", |
| 34 | loop_data=None, |
| 35 | **kwargs, |
| 36 | ): |
| 37 | self.agent = agent |
| 38 | self.name = name |
| 39 | self.method = method |
| 40 | self.args = args or {} |
| 41 | self.message = message |
| 42 | self.loop_data = loop_data |
| 43 | |
| 44 | async def after_execution(self, response, **kwargs): |
| 45 | self.agent.hist_add_tool_result( |
| 46 | self.name, |
| 47 | response.message.strip(), |
| 48 | id=self.log.id, |
| 49 | **(response.additional or {}), |
| 50 | ) |
| 51 | self.log.update(content=response.message.strip()) |
| 52 | |
| 53 | |
| 54 | def _install_tool_stub(monkeypatch): |
| 55 | tool_stub = types.ModuleType("helpers.tool") |
| 56 | tool_stub.Response = _TestResponse |
| 57 | tool_stub.Tool = _TestTool |
| 58 | history_stub = types.ModuleType("helpers.history") |
| 59 | |
| 60 | class _RawMessage(dict): |
| 61 | def __init__(self, raw_content, preview): |
| 62 | super().__init__(raw_content=raw_content, preview=preview) |
| 63 | |
| 64 | history_stub.RawMessage = _RawMessage |
| 65 | monkeypatch.setitem(sys.modules, "helpers.tool", tool_stub) |
| 66 | monkeypatch.setitem(sys.modules, "helpers.history", history_stub) |
| 67 | monkeypatch.delitem(sys.modules, "tools.vision_load", raising=False) |
| 68 | |
| 69 | |
| 70 | def test_prepare_content_keeps_missing_local_image_refs_strict(): |
| 71 | missing_path = "/tmp/a0-missing-desktop-screenshot.png" |
| 72 | |
| 73 | with pytest.raises(FileNotFoundError): |
| 74 | images.prepare_content( |
| 75 | [{"type": "image_url", "image_url": {"url": missing_path}}] |
| 76 | ) |
| 77 | |
| 78 | |
| 79 | @pytest.mark.anyio |
| 80 | async def test_vision_load_materializes_local_image_to_chat_artifact(monkeypatch, tmp_path): |
| 81 | _install_tool_stub(monkeypatch) |
| 82 | import tools.vision_load as vision_load_module |
| 83 | |
| 84 | def fake_get_abs_path(*parts): |
| 85 | return str(tmp_path.joinpath(*parts)) |
| 86 | |
| 87 | def fake_normalize_a0_path(path): |
| 88 | return "/a0/" + str(Path(path).relative_to(tmp_path)).replace("\\", "/") |
| 89 | |
| 90 | monkeypatch.setattr(vision_load_module.chat_media.files, "get_abs_path", fake_get_abs_path) |
| 91 | monkeypatch.setattr(vision_load_module.chat_media.files, "normalize_a0_path", fake_normalize_a0_path) |
| 92 | monkeypatch.setattr(vision_load_module, "get_chat_model_config", lambda _agent: {"vision": True, "max_embeds": 10}) |
| 93 | monkeypatch.setattr(vision_load_module, "get_vision_model_config", lambda _agent: {}) |
| 94 | |
| 95 | async def direct_call(func, *args, **kwargs): |
| 96 | return func(*args, **kwargs) |
| 97 | |
| 98 | monkeypatch.setattr( |
| 99 | vision_load_module.runtime, |
| 100 | "call_development_function", |
| 101 | direct_call, |
| 102 | ) |
| 103 | |
| 104 | image_path = tmp_path / "sample-image.png" |
| 105 | image_path.write_bytes(b"png-data") |
| 106 | |
| 107 | tool_results = [] |
| 108 | messages = [] |
| 109 | updates = [] |
| 110 | agent = SimpleNamespace( |
| 111 | context=SimpleNamespace(id="ctx-vision", get_data=lambda *_args, **_kwargs: None), |
| 112 | agent_name="Agent 0", |
| 113 | hist_add_tool_result=lambda *args, **kwargs: tool_results.append((args, kwargs)), |
| 114 | hist_add_message=lambda *args, **kwargs: messages.append((args, kwargs)), |
| 115 | ) |
| 116 | tool = vision_load_module.VisionLoad( |
| 117 | agent=agent, |
| 118 | name="vision_load", |
| 119 | method=None, |
| 120 | args={"paths": [str(image_path)]}, |
| 121 | message="", |
| 122 | loop_data=None, |
| 123 | ) |
| 124 | tool.log = SimpleNamespace(id="vision-log", update=lambda **kwargs: updates.append(kwargs)) |
| 125 | |
| 126 | invalid = await tool.execute(paths=None) |
| 127 | assert invalid.message == "vision_load error: `paths` must be a string or an array." |
| 128 | |
| 129 | response = await tool.execute( |
| 130 | paths=str(image_path), |
| 131 | query="Read the footer text.", |
| 132 | ) |
| 133 | image_path.unlink() |
| 134 | await tool.after_execution(response) |
| 135 | |
| 136 | raw_message = messages[0][1]["content"] |
| 137 | assert [item["type"] for item in raw_message["raw_content"]] == ["image_url"] |
| 138 | stored_ref = raw_message["raw_content"][0]["image_url"]["url"] |
| 139 | assert stored_ref.startswith("/a0/usr/chats/ctx-vision/images/vision-load/sample-image-") |
| 140 | stored_path = tmp_path / stored_ref.removeprefix("/a0/") |
| 141 | assert stored_path.read_bytes() == b"png-data" |
| 142 | assert updates[-1]["content"] == response.message |
| 143 | |
| 144 | |
| 145 | def test_active_vision_model_route_prefers_main_native_vision(monkeypatch): |
| 146 | from plugins._model_config.helpers import model_config |
| 147 | |
| 148 | cases = [ |
| 149 | ({"vision": False}, {}, False), |
| 150 | ({"vision": False}, {"provider": "p"}, False), |
| 151 | ({"vision": False}, {"name": "v"}, False), |
| 152 | ({"vision": True}, {"provider": "p", "name": "v"}, False), |
| 153 | ({"vision": False}, {"provider": "p", "name": "v"}, True), |
| 154 | ( |
| 155 | {"vision": True}, |
| 156 | {"provider": "p", "name": "v", "override_main": True}, |
| 157 | True, |
| 158 | ), |
| 159 | ] |
| 160 | for chat, vision, expected in cases: |
| 161 | monkeypatch.setattr( |
| 162 | model_config, |
| 163 | "get_effective_config", |
| 164 | lambda _agent=None, chat=chat, vision=vision: { |
| 165 | "chat_model": chat, |
| 166 | "vision_model": vision, |
| 167 | }, |
| 168 | ) |
| 169 | assert bool(model_config.get_vision_model_config()) is expected |
| 170 | |
| 171 | |
| 172 | def test_vision_summary_only_shows_skipped_section_when_needed(monkeypatch): |
| 173 | _install_tool_stub(monkeypatch) |
| 174 | import tools.vision_load as vision_load_module |
| 175 | |
| 176 | tool = vision_load_module.VisionLoad(agent=None) |
| 177 | tool.vision_config = {"max_embeds": 10} |
| 178 | tool.loaded_paths = ["loaded.png"] |
| 179 | tool.skipped_paths = [] |
| 180 | |
| 181 | assert tool._summary() == "Loaded images (1):\nloaded.png" |
| 182 | |
| 183 | tool.skipped_paths = ["skipped.png"] |
| 184 | assert tool._summary() == ( |
| 185 | "Loaded images (1):\nloaded.png\n\n" |
| 186 | "Skipped images (1, max 10):\nskipped.png" |
| 187 | ) |
| 188 | |
| 189 | |
| 190 | @pytest.mark.anyio |
| 191 | async def test_vision_model_sends_multiple_images_once_and_keeps_history_text_only( |
| 192 | monkeypatch, |
| 193 | tmp_path, |
| 194 | ): |
| 195 | _install_tool_stub(monkeypatch) |
| 196 | import tools.vision_load as vision_load_module |
| 197 | |
| 198 | async def direct_call(func, *args, **kwargs): |
| 199 | return func(*args, **kwargs) |
| 200 | |
| 201 | calls = [] |
| 202 | |
| 203 | class FakeVisionModel: |
| 204 | async def unified_call(self, **kwargs): |
| 205 | calls.append(kwargs) |
| 206 | return "The second screenshot fixes the red login error.", "" |
| 207 | |
| 208 | monkeypatch.setattr(vision_load_module.runtime, "call_development_function", direct_call) |
| 209 | monkeypatch.setattr(vision_load_module, "build_vision_model", lambda _agent: FakeVisionModel()) |
| 210 | monkeypatch.setattr( |
| 211 | vision_load_module, |
| 212 | "get_chat_model_config", |
| 213 | lambda _agent: {"vision": True, "max_embeds": 1}, |
| 214 | ) |
| 215 | monkeypatch.setattr( |
| 216 | vision_load_module, |
| 217 | "get_vision_model_config", |
| 218 | lambda _agent: {"provider": "test", "name": "vision", "max_embeds": 5}, |
| 219 | ) |
| 220 | |
| 221 | image_paths = [tmp_path / "before.png", tmp_path / "after.png"] |
| 222 | for path in image_paths: |
| 223 | path.write_bytes(b"png-data") |
| 224 | |
| 225 | tool_results = [] |
| 226 | raw_messages = [] |
| 227 | agent = SimpleNamespace( |
| 228 | context=SimpleNamespace(id=""), |
| 229 | agent_name="Agent 0", |
| 230 | last_user_message=SimpleNamespace( |
| 231 | output_text=lambda: "Review these UI screenshots." |
| 232 | ), |
| 233 | read_prompt=lambda _name, request, query: ( |
| 234 | f"Current request: {request}\n\nVisual query: {query}" |
| 235 | ), |
| 236 | hist_add_tool_result=lambda *args, **kwargs: tool_results.append((args, kwargs)), |
| 237 | hist_add_message=lambda *args, **kwargs: raw_messages.append((args, kwargs)), |
| 238 | ) |
| 239 | tool = vision_load_module.VisionLoad( |
| 240 | agent=agent, |
| 241 | name="vision_load", |
| 242 | method=None, |
| 243 | args={"paths": [str(path) for path in image_paths]}, |
| 244 | message="", |
| 245 | loop_data=None, |
| 246 | ) |
| 247 | tool.log = SimpleNamespace(id="vision-log", update=lambda **kwargs: None) |
| 248 | |
| 249 | response = await tool.execute( |
| 250 | paths=[str(path) for path in image_paths], |
| 251 | query="Compare the login error banners.", |
| 252 | ) |
| 253 | response.additional = {"_responses_output_item": {"output": response.message}} |
| 254 | await tool.after_execution(response) |
| 255 | |
| 256 | assert len(calls) == 1 |
| 257 | content = calls[0]["messages"][0].content |
| 258 | assert content[0] == { |
| 259 | "type": "text", |
| 260 | "text": ( |
| 261 | "Current request: Review these UI screenshots.\n\n" |
| 262 | "Visual query: Compare the login error banners." |
| 263 | ), |
| 264 | } |
| 265 | assert [item["type"] for item in content].count("image_url") == 2 |
| 266 | assert "max_tokens" not in calls[0] |
| 267 | assert "explicit_caching" not in calls[0] |
| 268 | assert "fixes the red login error" in response.message |
| 269 | assert response.message != "dummy" |
| 270 | assert raw_messages == [] |
| 271 | assert tool.loaded_paths == [str(path) for path in image_paths] |
| 272 | assert tool_results[0][1]["_responses_output_item"]["output"] == response.message |
| 273 | |
| 274 | |
| 275 | @pytest.mark.anyio |
| 276 | async def test_vision_model_empty_response_is_reported_as_error(monkeypatch): |
| 277 | _install_tool_stub(monkeypatch) |
| 278 | import tools.vision_load as vision_load_module |
| 279 | |
| 280 | class FakeVisionModel: |
| 281 | async def unified_call(self, **kwargs): |
| 282 | return "", "" |
| 283 | |
| 284 | monkeypatch.setattr( |
| 285 | vision_load_module, |
| 286 | "build_vision_model", |
| 287 | lambda _agent: FakeVisionModel(), |
| 288 | ) |
| 289 | monkeypatch.setattr( |
| 290 | vision_load_module, |
| 291 | "get_vision_model_config", |
| 292 | lambda _agent: {"provider": "test", "name": "vision", "max_embeds": 10}, |
| 293 | ) |
| 294 | |
| 295 | agent = SimpleNamespace( |
| 296 | context=SimpleNamespace(id="", get_data=lambda _key: ""), |
| 297 | last_user_message=SimpleNamespace(output_text=lambda: "Inspect the image."), |
| 298 | read_prompt=lambda _name, request, query: f"{request}\n{query}", |
| 299 | ) |
| 300 | tool = vision_load_module.VisionLoad( |
| 301 | agent=agent, |
| 302 | name="vision_load", |
| 303 | method=None, |
| 304 | args={"paths": ["data:image/png;base64,AA=="]}, |
| 305 | message="", |
| 306 | loop_data=None, |
| 307 | ) |
| 308 | |
| 309 | response = await tool.execute(paths=["data:image/png;base64,AA=="]) |
| 310 | |
| 311 | assert response.message == ( |
| 312 | "Image analysis error: Vision Model returned an empty response." |
| 313 | ) |
| 314 | |
| 315 | |
| 316 | @pytest.mark.anyio |
| 317 | async def test_parallel_worker_consumes_parent_ephemeral_image(monkeypatch, tmp_path): |
| 318 | _install_tool_stub(monkeypatch) |
| 319 | import tools.vision_load as vision_load_module |
| 320 | |
| 321 | def fake_get_abs_path(*parts): |
| 322 | return str(tmp_path.joinpath(*parts)) |
| 323 | |
| 324 | def fake_normalize_a0_path(path): |
| 325 | return "/a0/" + str(Path(path).relative_to(tmp_path)).replace("\\", "/") |
| 326 | |
| 327 | monkeypatch.setattr(vision_load_module.chat_media.files, "get_abs_path", fake_get_abs_path) |
| 328 | monkeypatch.setattr(vision_load_module.chat_media.files, "normalize_a0_path", fake_normalize_a0_path) |
| 329 | parent_id = "parent-vision" |
| 330 | monkeypatch.setattr( |
| 331 | vision_load_module, |
| 332 | "get_chat_model_config", |
| 333 | lambda _agent: {"vision": True, "max_embeds": 10}, |
| 334 | ) |
| 335 | monkeypatch.setattr(vision_load_module, "get_vision_model_config", lambda _agent: {}) |
| 336 | queued = [] |
| 337 | monkeypatch.setattr( |
| 338 | vision_load_module.parallel_tools, |
| 339 | "queue_parallel_parent_history", |
| 340 | lambda _agent, **message: queued.append(message) or True, |
| 341 | ) |
| 342 | |
| 343 | ref = vision_load_module.ephemeral_images.put_image_bytes( |
| 344 | context_id=parent_id, |
| 345 | mime="image/png", |
| 346 | payload=b"png-data", |
| 347 | name="shot.png", |
| 348 | ) |
| 349 | context = SimpleNamespace( |
| 350 | id="parallel-worker", |
| 351 | get_data=lambda key: parent_id |
| 352 | if key == vision_load_module.parallel_tools.PARALLEL_WORKER_PARENT_CONTEXT_KEY |
| 353 | else None, |
| 354 | ) |
| 355 | tool_results = [] |
| 356 | local_messages = [] |
| 357 | agent = SimpleNamespace( |
| 358 | context=context, |
| 359 | agent_name="Agent 0", |
| 360 | hist_add_tool_result=lambda *args, **kwargs: tool_results.append((args, kwargs)), |
| 361 | hist_add_message=lambda *args, **kwargs: local_messages.append((args, kwargs)), |
| 362 | ) |
| 363 | tool = vision_load_module.VisionLoad( |
| 364 | agent=agent, |
| 365 | name="vision_load", |
| 366 | method=None, |
| 367 | args={"paths": [ref]}, |
| 368 | message="", |
| 369 | loop_data=None, |
| 370 | ) |
| 371 | tool.log = SimpleNamespace(id="vision-log", update=lambda **kwargs: None) |
| 372 | |
| 373 | response = await tool.execute(paths=[ref]) |
| 374 | await tool.after_execution(response) |
| 375 | |
| 376 | assert tool._context_id() == parent_id |
| 377 | assert tool.loaded_paths == ["shot.png"] |
| 378 | assert vision_load_module.ephemeral_images.get_image(ref, context_id=parent_id) is None |
| 379 | stored_ref = tool.images_dict["shot.png"] |
| 380 | assert stored_ref.startswith("/a0/usr/chats/parent-vision/images/vision-load/shot-") |
| 381 | assert local_messages == [] |
| 382 | assert queued[0]["tokens"] == vision_load_module.TOKENS_ESTIMATE |
| 383 | raw_content = queued[0]["content"]["raw_content"] |
| 384 | assert raw_content == [ |
| 385 | {"type": "image_url", "image_url": {"url": stored_ref}} |
| 386 | ] |
| 387 | |
| 388 | |
| 389 | @pytest.mark.anyio |
| 390 | async def test_independent_vision_model_calls_can_run_concurrently(monkeypatch, tmp_path): |
| 391 | _install_tool_stub(monkeypatch) |
| 392 | import tools.vision_load as vision_load_module |
| 393 | |
| 394 | active = 0 |
| 395 | max_active = 0 |
| 396 | call_count = 0 |
| 397 | |
| 398 | class FakeVisionModel: |
| 399 | async def unified_call(self, **kwargs): |
| 400 | nonlocal active, max_active, call_count |
| 401 | active += 1 |
| 402 | call_count += 1 |
| 403 | max_active = max(max_active, active) |
| 404 | await asyncio.sleep(0.02) |
| 405 | active -= 1 |
| 406 | return "done", "" |
| 407 | |
| 408 | async def direct_call(func, *args, **kwargs): |
| 409 | return func(*args, **kwargs) |
| 410 | |
| 411 | monkeypatch.setattr(vision_load_module.runtime, "call_development_function", direct_call) |
| 412 | monkeypatch.setattr(vision_load_module, "build_vision_model", lambda _agent: FakeVisionModel()) |
| 413 | monkeypatch.setattr(vision_load_module, "get_chat_model_config", lambda _agent: {"vision": False}) |
| 414 | monkeypatch.setattr( |
| 415 | vision_load_module, |
| 416 | "get_vision_model_config", |
| 417 | lambda _agent: {"provider": "test", "name": "vision", "max_embeds": 10}, |
| 418 | ) |
| 419 | |
| 420 | image_paths = [tmp_path / "one.png", tmp_path / "two.png"] |
| 421 | for path in image_paths: |
| 422 | path.write_bytes(b"png-data") |
| 423 | |
| 424 | def make_tool(index): |
| 425 | agent = SimpleNamespace( |
| 426 | context=SimpleNamespace(id=""), |
| 427 | agent_name=f"Agent {index}", |
| 428 | last_user_message=SimpleNamespace( |
| 429 | output_text=lambda: f"inspection {index}" |
| 430 | ), |
| 431 | read_prompt=lambda _name, request, query: f"{request}\n{query}", |
| 432 | ) |
| 433 | return vision_load_module.VisionLoad( |
| 434 | agent=agent, |
| 435 | name="vision_load", |
| 436 | method=None, |
| 437 | args={"paths": [str(path) for path in image_paths]}, |
| 438 | message="", |
| 439 | loop_data=None, |
| 440 | ) |
| 441 | |
| 442 | responses = await asyncio.gather( |
| 443 | *( |
| 444 | make_tool(index).execute(paths=[str(path) for path in image_paths]) |
| 445 | for index in range(4) |
| 446 | ) |
| 447 | ) |
| 448 | |
| 449 | assert call_count == 4 |
| 450 | assert max_active == 4 |
| 451 | assert all("done" in response.message for response in responses) |