@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
py 1,023 lines 33.6 KB
Raw
1 """Provider-boundary tests for validated tool argument serialization."""
2
3 import ast
4 import asyncio
5 import importlib
6 import json
7 import os
8 import typing as t
9
10 import pydantic
11 import pytest
12
13 from composio.client.types import Tool, tool_list_response
14 from composio.utils.shared import (
15 json_schema_to_model,
16 validate_and_serialize_tool_arguments,
17 )
18
19
20 PROVIDERS = ("crewai", "langchain", "langgraph")
21
22
23 @pytest.fixture(autouse=True)
24 def _sandbox_provider_home(tmp_path, monkeypatch):
25 """Keep provider initialization from writing caches into the real home."""
26 monkeypatch.setenv("HOME", str(tmp_path))
27
28
29 def _make_tool(input_parameters: t.Dict[str, t.Any]) -> Tool:
30 return Tool(
31 name="Test tool",
32 slug="TEST_TOOL",
33 description="Test tool for provider argument serialization",
34 input_parameters=input_parameters,
35 output_parameters={},
36 available_versions=["12012025_00"],
37 version="12012025_00",
38 scopes=[],
39 toolkit=tool_list_response.ItemToolkit(name="Test", slug="test", logo=""),
40 deprecated=tool_list_response.ItemDeprecated(
41 available_versions=["12012025_00"],
42 displayName="Test tool",
43 version="12012025_00",
44 toolkit=tool_list_response.ItemDeprecatedToolkit(logo=""),
45 is_deprecated=False,
46 ),
47 is_deprecated=False,
48 no_auth=True,
49 tags=[],
50 )
51
52
53 def _provider(name: str):
54 module = pytest.importorskip(f"composio_{name}")
55 class_name = {
56 "crewai": "CrewAIProvider",
57 "langchain": "LangchainProvider",
58 "langgraph": "LanggraphProvider",
59 }[name]
60 return getattr(module, class_name)()
61
62
63 def _wrap(name: str, input_parameters: t.Dict[str, t.Any]):
64 received: t.List[t.Dict[str, t.Any]] = []
65
66 def execute_tool(slug: str, arguments: t.Dict[str, t.Any]):
67 assert slug == "TEST_TOOL"
68 received.append(arguments)
69 return {"successful": True, "data": arguments, "error": None}
70
71 wrapped = _provider(name).wrap_tool(_make_tool(input_parameters), execute_tool)
72 return wrapped, received
73
74
75 def _run(name: str, wrapped: t.Any, arguments: t.Dict[str, t.Any]):
76 if name == "crewai":
77 return wrapped.run(**arguments)
78 return wrapped.run(arguments)
79
80
81 @pytest.mark.parametrize("name", PROVIDERS)
82 def test_provider_serialization_preserves_argument_presence_and_aliases(name: str):
83 wrapped, received = _wrap(
84 name,
85 {
86 "type": "object",
87 "title": "TestArguments",
88 "properties": {
89 "query": {"type": "string"},
90 "limit": {"type": "integer"},
91 "note": {
92 "anyOf": [{"type": "string"}, {"type": "null"}],
93 },
94 "page": {"type": "integer", "default": 5},
95 "validate": {"type": "string"},
96 },
97 "required": ["query"],
98 },
99 )
100
101 _run(
102 name,
103 wrapped,
104 {"query": "agents", "note": None, "validate": "yes"},
105 )
106
107 assert received == [
108 {
109 "query": "agents",
110 "note": None,
111 "page": 5,
112 "validate": "yes",
113 }
114 ]
115
116
117 @pytest.mark.parametrize("name", PROVIDERS)
118 def test_provider_serialization_preserves_nested_argument_presence(name: str):
119 wrapped, received = _wrap(
120 name,
121 {
122 "type": "object",
123 "title": "NestedArguments",
124 "properties": {
125 "payload": {
126 "type": "object",
127 "title": "Payload",
128 "properties": {
129 "query": {"type": "string"},
130 "note": {
131 "anyOf": [{"type": "string"}, {"type": "null"}],
132 },
133 "explicit_null": {
134 "anyOf": [{"type": "string"}, {"type": "null"}],
135 },
136 "null_default": {
137 "anyOf": [{"type": "string"}, {"type": "null"}],
138 "default": None,
139 },
140 "page": {"type": "integer", "default": 5},
141 "items": {
142 "type": "array",
143 "items": {
144 "type": "object",
145 "title": "Item",
146 "properties": {
147 "name": {"type": "string"},
148 "label": {
149 "anyOf": [
150 {"type": "string"},
151 {"type": "null"},
152 ],
153 },
154 "enabled": {
155 "type": "boolean",
156 "default": True,
157 },
158 },
159 "required": ["name"],
160 },
161 },
162 "mapping": {
163 "type": "object",
164 "additionalProperties": {
165 "type": "object",
166 "title": "Entry",
167 "properties": {
168 "name": {"type": "string"},
169 "label": {
170 "anyOf": [
171 {"type": "string"},
172 {"type": "null"},
173 ],
174 },
175 "null_default": {
176 "anyOf": [
177 {"type": "string"},
178 {"type": "null"},
179 ],
180 "default": None,
181 },
182 "config": {
183 "type": "object",
184 "properties": {
185 "implicit": {
186 "anyOf": [
187 {"type": "string"},
188 {"type": "null"},
189 ],
190 },
191 "null_default": {
192 "anyOf": [
193 {"type": "string"},
194 {"type": "null"},
195 ],
196 "default": None,
197 },
198 },
199 },
200 "score": {"type": "integer", "default": 2},
201 },
202 "required": ["name", "config"],
203 },
204 },
205 },
206 "required": ["query", "items", "mapping"],
207 },
208 },
209 "required": ["payload"],
210 },
211 )
212
213 _run(
214 name,
215 wrapped,
216 {
217 "payload": {
218 "query": "agents",
219 "explicit_null": None,
220 "items": [
221 {"name": "first"},
222 {"name": "second", "label": None},
223 ],
224 "mapping": {
225 "one": {"name": "one", "config": {}},
226 "two": {"name": "two", "label": None, "config": {}},
227 },
228 },
229 },
230 )
231
232 assert received == [
233 {
234 "payload": {
235 "query": "agents",
236 "explicit_null": None,
237 "null_default": None,
238 "page": 5,
239 "items": [
240 {"name": "first", "enabled": True},
241 {"name": "second", "label": None, "enabled": True},
242 ],
243 "mapping": {
244 "one": {
245 "name": "one",
246 "null_default": None,
247 "config": {"null_default": None},
248 "score": 2,
249 },
250 "two": {
251 "name": "two",
252 "label": None,
253 "null_default": None,
254 "config": {"null_default": None},
255 "score": 2,
256 },
257 },
258 },
259 }
260 ]
261
262
263 @pytest.mark.parametrize("name", PROVIDERS)
264 def test_provider_serialization_preserves_dynamic_array_item_defaults(name: str):
265 wrapped, received = _wrap(
266 name,
267 {
268 "type": "object",
269 "title": "DynamicArrayArguments",
270 "patternProperties": {
271 "^items_": {
272 "type": "array",
273 "items": {
274 "type": "object",
275 "properties": {
276 "implicit": {
277 "anyOf": [
278 {"type": "string"},
279 {"type": "null"},
280 ],
281 },
282 "null_default": {
283 "anyOf": [
284 {"type": "string"},
285 {"type": "null"},
286 ],
287 "default": None,
288 },
289 },
290 },
291 },
292 },
293 "additionalProperties": False,
294 },
295 )
296
297 _run(name, wrapped, {"items_a": [{}, {"implicit": None}]})
298
299 assert received == [
300 {
301 "items_a": [
302 {"null_default": None},
303 {"implicit": None, "null_default": None},
304 ]
305 }
306 ]
307
308
309 @pytest.mark.parametrize("name", PROVIDERS)
310 def test_provider_serialization_preserves_presence_across_overlapping_patterns(
311 name: str,
312 ):
313 object_schema = {
314 "type": "object",
315 "properties": {
316 "implicit": {
317 "anyOf": [{"type": "string"}, {"type": "null"}],
318 },
319 "defaulted": {"type": "integer", "default": 1},
320 },
321 }
322 wrapped, received = _wrap(
323 name,
324 {
325 "type": "object",
326 "title": "OverlappingPatternArguments",
327 "patternProperties": {
328 "^x": {**object_schema, "title": "First"},
329 "x$": {**object_schema, "title": "Second"},
330 },
331 "additionalProperties": False,
332 },
333 )
334
335 _run(name, wrapped, {"x": {}})
336
337 assert received == [{"x": {"defaulted": 1}}]
338
339
340 @pytest.mark.parametrize("name", PROVIDERS)
341 @pytest.mark.parametrize("combiner", ("anyOf", "oneOf"))
342 def test_provider_serialization_uses_selected_composed_object_defaults(
343 name: str,
344 combiner: str,
345 ):
346 wrapped, received = _wrap(
347 name,
348 {
349 "type": "object",
350 "title": "ComposedArguments",
351 "properties": {
352 "payload": {
353 combiner: [
354 {
355 "type": "object",
356 "properties": {
357 "kind": {"type": "string", "enum": ["defaulted"]},
358 "value": {"type": "integer", "default": 5},
359 "config": {
360 "type": "object",
361 "properties": {
362 "nested": {
363 "type": "integer",
364 "default": 7,
365 },
366 },
367 },
368 },
369 "required": ["kind", "config"],
370 },
371 {
372 "type": "object",
373 "properties": {
374 "kind": {"type": "string", "enum": ["plain"]},
375 "value": {
376 "anyOf": [
377 {"type": "integer"},
378 {"type": "null"},
379 ],
380 },
381 "config": {
382 "type": "object",
383 "properties": {
384 "nested": {
385 "anyOf": [
386 {"type": "integer"},
387 {"type": "null"},
388 ],
389 },
390 },
391 },
392 },
393 "required": ["kind", "config"],
394 },
395 ],
396 },
397 },
398 "required": ["payload"],
399 },
400 )
401
402 _run(name, wrapped, {"payload": {"kind": "plain", "config": {}}})
403 _run(name, wrapped, {"payload": {"kind": "defaulted", "config": {}}})
404
405 assert received == [
406 {"payload": {"kind": "plain", "config": {}}},
407 {
408 "payload": {
409 "kind": "defaulted",
410 "value": 5,
411 "config": {"nested": 7},
412 }
413 },
414 ]
415
416
417 @pytest.mark.parametrize("name", PROVIDERS)
418 @pytest.mark.parametrize("combiner", ("anyOf", "oneOf"))
419 def test_provider_serialization_preserves_selected_composed_null_defaults(
420 name: str,
421 combiner: str,
422 ):
423 nullable_string = {
424 "anyOf": [{"type": "string"}, {"type": "null"}],
425 }
426 wrapped, received = _wrap(
427 name,
428 {
429 "type": "object",
430 "title": "ComposedNullDefaultArguments",
431 "properties": {
432 "payload": {
433 combiner: [
434 {
435 "type": "object",
436 "properties": {
437 "kind": {"type": "string", "enum": ["defaulted"]},
438 "value": {**nullable_string, "default": None},
439 },
440 "required": ["kind"],
441 },
442 {
443 "type": "object",
444 "properties": {
445 "kind": {"type": "string", "enum": ["plain"]},
446 "value": nullable_string,
447 },
448 "required": ["kind"],
449 },
450 ],
451 },
452 },
453 "required": ["payload"],
454 },
455 )
456
457 _run(name, wrapped, {"payload": {"kind": "plain"}})
458 _run(name, wrapped, {"payload": {"kind": "defaulted"}})
459
460 assert received == [
461 {"payload": {"kind": "plain"}},
462 {"payload": {"kind": "defaulted", "value": None}},
463 ]
464
465
466 def test_provider_serialization_preserves_ref_shaped_object_defaults():
467 schema = {
468 "$defs": {
469 "RefTarget": {
470 "type": "object",
471 "properties": {"value": {"type": "string"}},
472 },
473 },
474 "type": "object",
475 "title": "RefShapedDefaultArguments",
476 "properties": {
477 "payload": {
478 "type": "object",
479 "additionalProperties": True,
480 "default": {"$ref": "#/$defs/RefTarget"},
481 },
482 },
483 }
484 model = json_schema_to_model(schema)
485
486 assert model.model_validate({}).model_dump(mode="python", by_alias=True) == {
487 "payload": {"$ref": "#/$defs/RefTarget"}
488 }
489 assert validate_and_serialize_tool_arguments(model, {}) == {
490 "payload": {"$ref": "#/$defs/RefTarget"}
491 }
492
493
494 @pytest.mark.parametrize("name", PROVIDERS)
495 def test_provider_serialization_preserves_free_form_objects(name: str):
496 wrapped, received = _wrap(
497 name,
498 {
499 "type": "object",
500 "title": "FreeFormArguments",
501 "properties": {},
502 },
503 )
504
505 _run(name, wrapped, {"custom": {"nested": True}, "items": [1, "two"]})
506
507 assert received == [{"custom": {"nested": True}, "items": [1, "two"]}]
508
509
510 @pytest.mark.parametrize("name", PROVIDERS)
511 def test_provider_serialization_validates_pattern_only_schemas(name: str):
512 wrapped, received = _wrap(
513 name,
514 {
515 "type": "object",
516 "title": "PatternArguments",
517 "properties": {},
518 "patternProperties": {
519 "^count_": {"type": "integer", "minimum": 1},
520 },
521 "additionalProperties": False,
522 },
523 )
524
525 _run(name, wrapped, {"count_valid": 2})
526 invalid_result = _run(name, wrapped, {"count_invalid": "not-an-integer"})
527
528 assert received == [{"count_valid": 2}]
529 if name == "crewai":
530 assert invalid_result["successful"] is False
531 assert invalid_result["data"] is None
532
533
534 @pytest.mark.parametrize(
535 ("arguments", "parameter"),
536 [
537 ({"query": "agents", "limit": "not-an-integer"}, "limit"),
538 ({"limit": 5}, "query"),
539 ],
540 )
541 def test_crewai_returns_structured_validation_errors(
542 arguments: t.Dict[str, t.Any], parameter: str
543 ):
544 wrapped, received = _wrap(
545 "crewai",
546 {
547 "type": "object",
548 "title": "RequiredArguments",
549 "properties": {
550 "query": {"type": "string"},
551 "limit": {"type": "integer"},
552 },
553 "required": ["query"],
554 },
555 )
556
557 result = _run("crewai", wrapped, arguments)
558
559 assert result["successful"] is False
560 assert result["data"] is None
561 assert parameter in result["error"]
562 assert received == []
563
564
565 # ---------------------------------------------------------------------------
566 # Cross-provider matrix for free-form object arguments (SUP-1643).
567 #
568 # Each row drives a provider through the entry point its framework really
569 # calls, because the frameworks drop keys before our wrapper code runs.
570 # ---------------------------------------------------------------------------
571
572 # Providers this environment must have. CI and nox set it so a missing
573 # install fails instead of skipping; locally it is usually empty.
574 EXPECTED_PROVIDERS = frozenset(
575 os.environ.get("COMPOSIO_EXPECTED_PROVIDERS", "").replace(",", " ").split()
576 )
577
578 # Row name -> (provider package, provider class).
579 MATRIX_PROVIDERS: t.Dict[str, t.Tuple[str, str]] = {
580 "crewai": ("crewai", "CrewAIProvider"),
581 "langchain": ("langchain", "LangchainProvider"),
582 "langchain_async": ("langchain", "LangchainProvider"),
583 "langgraph": ("langgraph", "LanggraphProvider"),
584 "gemini": ("gemini", "GeminiProvider"),
585 "autogen": ("autogen", "AutogenProvider"),
586 "autogen_ag2": ("autogen", "AutogenProvider"),
587 "google_adk": ("google_adk", "GoogleAdkProvider"),
588 "llamaindex": ("llamaindex", "LlamaIndexProvider"),
589 "openai_agents": ("openai_agents", "OpenAIAgentsProvider"),
590 }
591
592 # Providers that validate arguments on the client. LlamaIndex and OpenAI
593 # Agents pass the model's arguments through, so they only get preservation
594 # rows.
595 VALIDATING_PROVIDERS = frozenset(
596 {
597 "crewai",
598 "langchain",
599 "langchain_async",
600 "langgraph",
601 "gemini",
602 "autogen",
603 "autogen_ag2",
604 "google_adk",
605 }
606 )
607
608
609 def _matrix_provider(row: str):
610 package, class_name = MATRIX_PROVIDERS[row]
611 try:
612 module = importlib.import_module(f"composio_{package}")
613 except ImportError as error:
614 if package in EXPECTED_PROVIDERS:
615 pytest.fail(f"composio_{package} is expected here but missing: {error}")
616 pytest.skip(f"composio_{package} is not installed")
617 return getattr(module, class_name)()
618
619
620 def _matrix_call(row: str, input_parameters: t.Dict[str, t.Any], arguments):
621 """Wrap a tool, call it through the framework, and record what executes."""
622 received: t.List[t.Dict[str, t.Any]] = []
623
624 def execute_tool(slug: str, arguments: t.Dict[str, t.Any]):
625 assert slug == "TEST_TOOL"
626 received.append(arguments)
627 return {"successful": True, "data": {}, "error": None}
628
629 provider = _matrix_provider(row)
630 tool = _make_tool(input_parameters)
631 wrapped = provider.wrap_tool(tool, execute_tool)
632 try:
633 result = _matrix_invoke(row, provider, wrapped, arguments)
634 except pydantic.ValidationError as error:
635 # Gemini's callable raises; google-genai reports it back to the model.
636 result = error
637 return received, result
638
639
640 def _matrix_invoke(row: str, provider, wrapped, arguments):
641 if row == "crewai":
642 result = wrapped.run(**arguments)
643 elif row in ("langchain", "langgraph"):
644 result = wrapped.invoke(arguments)
645 elif row == "langchain_async":
646 result = asyncio.run(wrapped.ainvoke(arguments))
647 elif row == "gemini":
648 # Gemini's automatic function calling invokes the callable directly.
649 result = wrapped(**arguments)
650 elif row == "autogen":
651 from autogen_core import CancellationToken
652
653 result = asyncio.run(wrapped.run_json(arguments, CancellationToken()))
654 elif row == "autogen_ag2":
655 from autogen import ConversableAgent
656
657 caller = ConversableAgent(
658 "caller",
659 llm_config={"config_list": [{"model": "gpt-4o-mini", "api_key": "x"}]},
660 human_input_mode="NEVER",
661 )
662 executor = ConversableAgent(
663 "executor", llm_config=False, human_input_mode="NEVER"
664 )
665 provider.register_tools(caller=caller, executor=executor, tools=[wrapped])
666 _, result = executor.execute_function(
667 {"name": wrapped.name, "arguments": json.dumps(arguments)}
668 )
669 elif row == "google_adk":
670 from unittest.mock import MagicMock
671
672 context = MagicMock()
673 context.tool_confirmation = None
674 result = asyncio.run(wrapped.run_async(args=arguments, tool_context=context))
675 elif row == "llamaindex":
676 result = wrapped.call(**arguments)
677 elif row == "openai_agents":
678 result = asyncio.run(wrapped.on_invoke_tool(None, json.dumps(arguments)))
679 else: # pragma: no cover - guarded by MATRIX_PROVIDERS
680 raise AssertionError(row)
681 return result
682
683
684 # Cases whose arguments every provider must deliver unchanged.
685 PRESERVATION_CASES: t.Dict[str, t.Tuple[t.Dict[str, t.Any], t.Dict[str, t.Any]]] = {
686 # The customer-reported shape: a free-form object under a declared property.
687 "nested_free_form": (
688 {
689 "type": "object",
690 "title": "NestedFreeForm",
691 "properties": {
692 "title": {"type": "string"},
693 "body": {"type": "object"},
694 },
695 "required": ["title", "body"],
696 },
697 {"title": "t", "body": {"unit": {"id": 7}, "tags": ["a"]}},
698 ),
699 "nested_additional_properties_schema": (
700 {
701 "type": "object",
702 "title": "NestedMap",
703 "properties": {
704 "body": {
705 "type": "object",
706 "additionalProperties": {"type": "string"},
707 },
708 },
709 "required": ["body"],
710 },
711 {"body": {"a": "x", "b": "y"}},
712 ),
713 "array_of_free_form_objects": (
714 {
715 "type": "object",
716 "title": "FreeFormItems",
717 "properties": {
718 "items": {"type": "array", "items": {"type": "object"}},
719 },
720 "required": ["items"],
721 },
722 {"items": [{"a": 1}, {"b": {"c": 2}}]},
723 ),
724 "root_free_form": (
725 {"type": "object", "title": "RootFreeForm", "properties": {}},
726 {"custom": {"nested": True}},
727 ),
728 "root_additional_properties": (
729 {
730 "type": "object",
731 "title": "RootOpen",
732 "properties": {"title": {"type": "string"}},
733 "required": ["title"],
734 "additionalProperties": True,
735 },
736 {"title": "t", "extra": {"k": 1}},
737 ),
738 }
739
740 # Cases every validating provider must reject without executing the tool.
741 REJECTION_CASES: t.Dict[str, t.Tuple[t.Dict[str, t.Any], t.Dict[str, t.Any]]] = {
742 "nested_map_value_type": (
743 PRESERVATION_CASES["nested_additional_properties_schema"][0],
744 {"body": {"a": 1}},
745 ),
746 "pattern_property_type": (
747 {
748 "type": "object",
749 "title": "PatternArguments",
750 "properties": {},
751 "patternProperties": {"^count_": {"type": "integer", "minimum": 1}},
752 "additionalProperties": False,
753 },
754 {"count_x": "nope"},
755 ),
756 # Unknown root keys on a schema that declares properties and omits
757 # additionalProperties are rejected, not dropped.
758 "closed_root_unknown_key": (
759 {
760 "type": "object",
761 "title": "ClosedRoot",
762 "properties": {"title": {"type": "string"}},
763 "required": ["title"],
764 },
765 {"title": "t", "extra": 1},
766 ),
767 # String-encoded scalars are rejected, not coerced.
768 "string_for_integer": (
769 {
770 "type": "object",
771 "title": "IntegerArgument",
772 "properties": {"limit": {"type": "integer"}},
773 "required": ["limit"],
774 },
775 {"limit": "5"},
776 ),
777 }
778
779
780 def _matrix_params(rows, cases):
781 return [
782 pytest.param(row, case, id=f"{row}-{case}") for row in rows for case in cases
783 ]
784
785
786 @pytest.mark.parametrize(
787 ("row", "case"), _matrix_params(MATRIX_PROVIDERS, PRESERVATION_CASES)
788 )
789 def test_matrix_preserves_free_form_arguments(row: str, case: str):
790 input_parameters, arguments = PRESERVATION_CASES[case]
791
792 received, _ = _matrix_call(row, input_parameters, arguments)
793
794 assert received == [arguments]
795
796
797 @pytest.mark.parametrize(
798 ("row", "case"),
799 _matrix_params(
800 [row for row in MATRIX_PROVIDERS if row in VALIDATING_PROVIDERS],
801 REJECTION_CASES,
802 ),
803 )
804 def test_matrix_rejects_invalid_dynamic_arguments(row: str, case: str):
805 input_parameters, arguments = REJECTION_CASES[case]
806
807 received, result = _matrix_call(row, input_parameters, arguments)
808
809 assert received == []
810 error = _rejection_error(row, result)
811 if error is not None:
812 assert REJECTION_FIELDS[case] in error
813
814
815 # The argument each rejection case's error must name.
816 REJECTION_FIELDS = {
817 "nested_map_value_type": "body.a",
818 "pattern_property_type": "count_x",
819 "closed_root_unknown_key": "extra",
820 "string_for_integer": "limit",
821 }
822
823
824 def _rejection_error(row: str, result: t.Any) -> t.Optional[str]:
825 """The validation error a rejected call reported, in its framework's shape.
826
827 Fails on any other result, so a crash cannot pass as a rejection. Returns
828 ``None`` where the framework hides the detail.
829 """
830 if row in ("langchain", "langchain_async", "langgraph"):
831 # `handle_validation_error=True` replaces the detail with this string.
832 assert result == "Tool input validation error"
833 return None
834 if row == "gemini":
835 assert isinstance(result, pydantic.ValidationError)
836 return str(result)
837 if row == "autogen_ag2":
838 # AG2 returns the tool result serialized into the function message.
839 result = ast.literal_eval(result["content"])
840 assert isinstance(result, dict), result
841 assert result["successful"] is False
842 assert result["data"] is None
843 return result["error"]
844
845
846 # ---------------------------------------------------------------------------
847 # Google ADK: validation errors reach the agent as results (SUP-1643).
848 # ---------------------------------------------------------------------------
849
850
851 def test_google_adk_returns_validation_errors_without_executing():
852 input_parameters, arguments = REJECTION_CASES["pattern_property_type"]
853
854 received, result = _matrix_call("google_adk", input_parameters, arguments)
855
856 assert received == []
857 assert result["successful"] is False
858 assert result["data"] is None
859 assert "count_x" in result["error"]
860
861
862 def test_google_adk_keeps_tool_context_out_of_arguments():
863 received, _ = _matrix_call(
864 "google_adk",
865 {"type": "object", "title": "RootFreeForm", "properties": {}},
866 {"custom": {"nested": True}},
867 )
868
869 assert received == [{"custom": {"nested": True}}]
870
871
872 def test_google_adk_does_not_forward_reserved_argument_names():
873 # ADK reserves `tool_context` and `input_stream`; a model-supplied value
874 # for either must not reach the tool backend on a free-form root.
875 received, _ = _matrix_call(
876 "google_adk",
877 {"type": "object", "title": "RootFreeForm", "properties": {}},
878 {"tool_context": {"forged": True}, "input_stream": "x", "data": 1},
879 )
880
881 assert received == [{"data": 1}]
882
883
884 def test_google_adk_never_sends_its_context_object_to_the_backend():
885 # A schema property named like ADK's context parameter makes ADK inject
886 # its own ToolContext under that name; it must never be executed with.
887 received, result = _matrix_call(
888 "google_adk",
889 {
890 "type": "object",
891 "title": "ReservedProperty",
892 "properties": {
893 "query": {"type": "string"},
894 # Untyped, so validation alone would accept any object.
895 "tool_context": {"description": "A tool argument"},
896 },
897 "required": ["query", "tool_context"],
898 },
899 {"query": "q", "tool_context": "model value"},
900 )
901
902 assert received == []
903 assert "tool_context" in result["error"]
904
905
906 # ---------------------------------------------------------------------------
907 # AutoGen: both runtime paths share one validated execution (SUP-1643).
908 # ---------------------------------------------------------------------------
909
910 AUTOGEN_ROWS = ("autogen", "autogen_ag2")
911
912
913 def _autogen_result(row: str, result: t.Any) -> t.Any:
914 """The tool's return value, unwrapped from AG2's function message."""
915 if row == "autogen_ag2":
916 # AG2 serializes a dict result with ``str()``.
917 return ast.literal_eval(result["content"])
918 return result
919
920
921 @pytest.mark.parametrize("row", AUTOGEN_ROWS)
922 def test_autogen_returns_structured_validation_error(row: str):
923 input_parameters, arguments = REJECTION_CASES["pattern_property_type"]
924
925 received, result = _matrix_call(row, input_parameters, arguments)
926
927 result = _autogen_result(row, result)
928 assert received == []
929 assert result["successful"] is False
930 assert result["data"] is None
931 assert "count_x" in result["error"]
932
933
934 @pytest.mark.parametrize("row", AUTOGEN_ROWS)
935 def test_autogen_omits_optional_arguments_the_model_left_out(row: str):
936 received, _ = _matrix_call(
937 row,
938 {
939 "type": "object",
940 "title": "OptionalArguments",
941 "properties": {
942 "query": {"type": "string"},
943 "note": {"type": "string"},
944 },
945 "required": ["query"],
946 },
947 {"query": "agents"},
948 )
949
950 assert received == [{"query": "agents"}]
951
952
953 @pytest.mark.parametrize("row", AUTOGEN_ROWS)
954 def test_autogen_restores_reserved_keyword_arguments(row: str):
955 received, _ = _matrix_call(
956 row,
957 {
958 "type": "object",
959 "title": "ReservedArguments",
960 "properties": {"from": {"type": "string"}},
961 "required": ["from"],
962 },
963 {"from_rs": "alice"},
964 )
965
966 assert received == [{"from": "alice"}]
967
968
969 def test_autogen_run_json_normalizes_string_payload():
970 received, _ = _matrix_call(
971 "autogen",
972 PRESERVATION_CASES["nested_free_form"][0] | {"required": ["body"]},
973 '{"body": {"a": 1}}',
974 )
975
976 assert received == [{"body": {"a": 1}}]
977
978
979 def test_autogen_run_json_logs_the_tool_call(caplog: pytest.LogCaptureFixture):
980 input_parameters, arguments = PRESERVATION_CASES["root_free_form"]
981
982 with caplog.at_level("INFO"):
983 received, _ = _matrix_call("autogen", input_parameters, arguments)
984
985 from autogen_core import EVENT_LOGGER_NAME
986 from autogen_core.logging import ToolCallEvent
987
988 events = [
989 record.msg
990 for record in caplog.records
991 if record.name == EVENT_LOGGER_NAME and isinstance(record.msg, ToolCallEvent)
992 ]
993 assert received == [arguments]
994 assert [json.loads(str(event))["arguments"] for event in events] == [arguments]
995
996
997 @pytest.mark.parametrize(
998 ("input_parameters", "expected"),
999 [
1000 (
1001 PRESERVATION_CASES["root_free_form"][0],
1002 {"additionalProperties": True},
1003 ),
1004 (
1005 REJECTION_CASES["pattern_property_type"][0],
1006 {
1007 "additionalProperties": False,
1008 "patternProperties": {"^count_": {"type": "integer", "minimum": 1}},
1009 },
1010 ),
1011 ],
1012 ids=["root_free_form", "pattern_only"],
1013 )
1014 def test_autogen_schema_mirrors_root_openness(
1015 input_parameters: t.Dict[str, t.Any], expected: t.Dict[str, t.Any]
1016 ):
1017 wrapped = _matrix_provider("autogen").wrap_tool(
1018 _make_tool(input_parameters), lambda slug, arguments: {}
1019 )
1020
1021 parameters = wrapped.schema["parameters"]
1022
1023 assert {key: parameters.get(key) for key in expected} == expected