1
"""Provider-boundary tests for validated tool argument serialization."""
2
3
+import ast
4
+import asyncio
5
+import importlib
6
+import json
7
+import os
8
import typing as t
9
10
+import pydantic
11
import pytest
12
13
from composio.client.types import Tool, tool_list_response
560
assert result["data"] is None
561
assert parameter in result["error"]
562
assert received == []
563
+
564
+
565
+# ---------------------------------------------------------------------------
566
+# Cross-provider matrix for free-form object arguments (SUP-1643).
567
+#
568
+# Each row drives a provider through the entry point its framework really
569
+# calls, because the frameworks drop keys before our wrapper code runs.
570
+# ---------------------------------------------------------------------------
571
+
572
+# Providers this environment must have. CI and nox set it so a missing
573
+# install fails instead of skipping; locally it is usually empty.
574
+EXPECTED_PROVIDERS = frozenset(
575
+ os.environ.get("COMPOSIO_EXPECTED_PROVIDERS", "").replace(",", " ").split()
576
+)
577
+
578
+# Row name -> (provider package, provider class).
579
+MATRIX_PROVIDERS: t.Dict[str, t.Tuple[str, str]] = {
580
+ "crewai": ("crewai", "CrewAIProvider"),
581
+ "langchain": ("langchain", "LangchainProvider"),
582
+ "langchain_async": ("langchain", "LangchainProvider"),
583
+ "langgraph": ("langgraph", "LanggraphProvider"),
584
+ "gemini": ("gemini", "GeminiProvider"),
585
+ "autogen": ("autogen", "AutogenProvider"),
586
+ "autogen_ag2": ("autogen", "AutogenProvider"),
587
+ "google_adk": ("google_adk", "GoogleAdkProvider"),
588
+ "llamaindex": ("llamaindex", "LlamaIndexProvider"),
589
+ "openai_agents": ("openai_agents", "OpenAIAgentsProvider"),
590
+}
591
+
592
+# Providers that validate arguments on the client. LlamaIndex and OpenAI
593
+# Agents pass the model's arguments through, so they only get preservation
594
+# rows.
595
+VALIDATING_PROVIDERS = frozenset(
596
+ {
597
+ "crewai",
598
+ "langchain",
599
+ "langchain_async",
600
+ "langgraph",
601
+ "gemini",
602
+ "autogen",
603
+ "autogen_ag2",
604
+ "google_adk",
605
+ }
606
+)
607
+
608
+
609
+def _matrix_provider(row: str):
610
+ package, class_name = MATRIX_PROVIDERS[row]
611
+ try:
612
+ module = importlib.import_module(f"composio_{package}")
613
+ except ImportError as error:
614
+ if package in EXPECTED_PROVIDERS:
615
+ pytest.fail(f"composio_{package} is expected here but missing: {error}")
616
+ pytest.skip(f"composio_{package} is not installed")
617
+ return getattr(module, class_name)()
618
+
619
+
620
+def _matrix_call(row: str, input_parameters: t.Dict[str, t.Any], arguments):
621
+ """Wrap a tool, call it through the framework, and record what executes."""
622
+ received: t.List[t.Dict[str, t.Any]] = []
623
+
624
+ def execute_tool(slug: str, arguments: t.Dict[str, t.Any]):
625
+ assert slug == "TEST_TOOL"
626
+ received.append(arguments)
627
+ return {"successful": True, "data": {}, "error": None}
628
+
629
+ provider = _matrix_provider(row)
630
+ tool = _make_tool(input_parameters)
631
+ wrapped = provider.wrap_tool(tool, execute_tool)
632
+ try:
633
+ result = _matrix_invoke(row, provider, wrapped, arguments)
634
+ except pydantic.ValidationError as error:
635
+ # Gemini's callable raises; google-genai reports it back to the model.
636
+ result = error
637
+ return received, result
638
+
639
+
640
+def _matrix_invoke(row: str, provider, wrapped, arguments):
641
+ if row == "crewai":
642
+ result = wrapped.run(**arguments)
643
+ elif row in ("langchain", "langgraph"):
644
+ result = wrapped.invoke(arguments)
645
+ elif row == "langchain_async":
646
+ result = asyncio.run(wrapped.ainvoke(arguments))
647
+ elif row == "gemini":
648
+ # Gemini's automatic function calling invokes the callable directly.
649
+ result = wrapped(**arguments)
650
+ elif row == "autogen":
651
+ from autogen_core import CancellationToken
652
+
653
+ result = asyncio.run(wrapped.run_json(arguments, CancellationToken()))
654
+ elif row == "autogen_ag2":
655
+ from autogen import ConversableAgent
656
+
657
+ caller = ConversableAgent(
658
+ "caller",
659
+ llm_config={"config_list": [{"model": "gpt-4o-mini", "api_key": "x"}]},
660
+ human_input_mode="NEVER",
661
+ )
662
+ executor = ConversableAgent(
663
+ "executor", llm_config=False, human_input_mode="NEVER"
664
+ )
665
+ provider.register_tools(caller=caller, executor=executor, tools=[wrapped])
666
+ _, result = executor.execute_function(
667
+ {"name": wrapped.name, "arguments": json.dumps(arguments)}
668
+ )
669
+ elif row == "google_adk":
670
+ from unittest.mock import MagicMock
671
+
672
+ context = MagicMock()
673
+ context.tool_confirmation = None
674
+ result = asyncio.run(wrapped.run_async(args=arguments, tool_context=context))
675
+ elif row == "llamaindex":
676
+ result = wrapped.call(**arguments)
677
+ elif row == "openai_agents":
678
+ result = asyncio.run(wrapped.on_invoke_tool(None, json.dumps(arguments)))
679
+ else: # pragma: no cover - guarded by MATRIX_PROVIDERS
680
+ raise AssertionError(row)
681
+ return result
682
+
683
+
684
+# Cases whose arguments every provider must deliver unchanged.
685
+PRESERVATION_CASES: t.Dict[str, t.Tuple[t.Dict[str, t.Any], t.Dict[str, t.Any]]] = {
686
+ # The customer-reported shape: a free-form object under a declared property.
687
+ "nested_free_form": (
688
+ {
689
+ "type": "object",
690
+ "title": "NestedFreeForm",
691
+ "properties": {
692
+ "title": {"type": "string"},
693
+ "body": {"type": "object"},
694
+ },
695
+ "required": ["title", "body"],
696
+ },
697
+ {"title": "t", "body": {"unit": {"id": 7}, "tags": ["a"]}},
698
+ ),
699
+ "nested_additional_properties_schema": (
700
+ {
701
+ "type": "object",
702
+ "title": "NestedMap",
703
+ "properties": {
704
+ "body": {
705
+ "type": "object",
706
+ "additionalProperties": {"type": "string"},
707
+ },
708
+ },
709
+ "required": ["body"],
710
+ },
711
+ {"body": {"a": "x", "b": "y"}},
712
+ ),
713
+ "array_of_free_form_objects": (
714
+ {
715
+ "type": "object",
716
+ "title": "FreeFormItems",
717
+ "properties": {
718
+ "items": {"type": "array", "items": {"type": "object"}},
719
+ },
720
+ "required": ["items"],
721
+ },
722
+ {"items": [{"a": 1}, {"b": {"c": 2}}]},
723
+ ),
724
+ "root_free_form": (
725
+ {"type": "object", "title": "RootFreeForm", "properties": {}},
726
+ {"custom": {"nested": True}},
727
+ ),
728
+ "root_additional_properties": (
729
+ {
730
+ "type": "object",
731
+ "title": "RootOpen",
732
+ "properties": {"title": {"type": "string"}},
733
+ "required": ["title"],
734
+ "additionalProperties": True,
735
+ },
736
+ {"title": "t", "extra": {"k": 1}},
737
+ ),
738
+}
739
+
740
+# Cases every validating provider must reject without executing the tool.
741
+REJECTION_CASES: t.Dict[str, t.Tuple[t.Dict[str, t.Any], t.Dict[str, t.Any]]] = {
742
+ "nested_map_value_type": (
743
+ PRESERVATION_CASES["nested_additional_properties_schema"][0],
744
+ {"body": {"a": 1}},
745
+ ),
746
+ "pattern_property_type": (
747
+ {
748
+ "type": "object",
749
+ "title": "PatternArguments",
750
+ "properties": {},
751
+ "patternProperties": {"^count_": {"type": "integer", "minimum": 1}},
752
+ "additionalProperties": False,
753
+ },
754
+ {"count_x": "nope"},
755
+ ),
756
+ # Unknown root keys on a schema that declares properties and omits
757
+ # additionalProperties are rejected, not dropped.
758
+ "closed_root_unknown_key": (
759
+ {
760
+ "type": "object",
761
+ "title": "ClosedRoot",
762
+ "properties": {"title": {"type": "string"}},
763
+ "required": ["title"],
764
+ },
765
+ {"title": "t", "extra": 1},
766
+ ),
767
+ # String-encoded scalars are rejected, not coerced.
768
+ "string_for_integer": (
769
+ {
770
+ "type": "object",
771
+ "title": "IntegerArgument",
772
+ "properties": {"limit": {"type": "integer"}},
773
+ "required": ["limit"],
774
+ },
775
+ {"limit": "5"},
776
+ ),
777
+}
778
+
779
+
780
+def _matrix_params(rows, cases):
781
+ return [
782
+ pytest.param(row, case, id=f"{row}-{case}") for row in rows for case in cases
783
+ ]
784
+
785
+
786
+@pytest.mark.parametrize(
787
+ ("row", "case"), _matrix_params(MATRIX_PROVIDERS, PRESERVATION_CASES)
788
+)
789
+def test_matrix_preserves_free_form_arguments(row: str, case: str):
790
+ input_parameters, arguments = PRESERVATION_CASES[case]
791
+
792
+ received, _ = _matrix_call(row, input_parameters, arguments)
793
+
794
+ assert received == [arguments]
795
+
796
+
797
+@pytest.mark.parametrize(
798
+ ("row", "case"),
799
+ _matrix_params(
800
+ [row for row in MATRIX_PROVIDERS if row in VALIDATING_PROVIDERS],
801
+ REJECTION_CASES,
802
+ ),
803
+)
804
+def test_matrix_rejects_invalid_dynamic_arguments(row: str, case: str):
805
+ input_parameters, arguments = REJECTION_CASES[case]
806
+
807
+ received, result = _matrix_call(row, input_parameters, arguments)
808
+
809
+ assert received == []
810
+ error = _rejection_error(row, result)
811
+ if error is not None:
812
+ assert REJECTION_FIELDS[case] in error
813
+
814
+
815
+# The argument each rejection case's error must name.
816
+REJECTION_FIELDS = {
817
+ "nested_map_value_type": "body.a",
818
+ "pattern_property_type": "count_x",
819
+ "closed_root_unknown_key": "extra",
820
+ "string_for_integer": "limit",
821
+}
822
+
823
+
824
+def _rejection_error(row: str, result: t.Any) -> t.Optional[str]:
825
+ """The validation error a rejected call reported, in its framework's shape.
826
+
827
+ Fails on any other result, so a crash cannot pass as a rejection. Returns
828
+ ``None`` where the framework hides the detail.
829
+ """
830
+ if row in ("langchain", "langchain_async", "langgraph"):
831
+ # `handle_validation_error=True` replaces the detail with this string.
832
+ assert result == "Tool input validation error"
833
+ return None
834
+ if row == "gemini":
835
+ assert isinstance(result, pydantic.ValidationError)
836
+ return str(result)
837
+ if row == "autogen_ag2":
838
+ # AG2 returns the tool result serialized into the function message.
839
+ result = ast.literal_eval(result["content"])
840
+ assert isinstance(result, dict), result
841
+ assert result["successful"] is False
842
+ assert result["data"] is None
843
+ return result["error"]
844
+
845
+
846
+# ---------------------------------------------------------------------------
847
+# Google ADK: validation errors reach the agent as results (SUP-1643).
848
+# ---------------------------------------------------------------------------
849
+
850
+
851
+def test_google_adk_returns_validation_errors_without_executing():
852
+ input_parameters, arguments = REJECTION_CASES["pattern_property_type"]
853
+
854
+ received, result = _matrix_call("google_adk", input_parameters, arguments)
855
+
856
+ assert received == []
857
+ assert result["successful"] is False
858
+ assert result["data"] is None
859
+ assert "count_x" in result["error"]
860
+
861
+
862
+def test_google_adk_keeps_tool_context_out_of_arguments():
863
+ received, _ = _matrix_call(
864
+ "google_adk",
865
+ {"type": "object", "title": "RootFreeForm", "properties": {}},
866
+ {"custom": {"nested": True}},
867
+ )
868
+
869
+ assert received == [{"custom": {"nested": True}}]
870
+
871
+
872
+def test_google_adk_does_not_forward_reserved_argument_names():
873
+ # ADK reserves `tool_context` and `input_stream`; a model-supplied value
874
+ # for either must not reach the tool backend on a free-form root.
875
+ received, _ = _matrix_call(
876
+ "google_adk",
877
+ {"type": "object", "title": "RootFreeForm", "properties": {}},
878
+ {"tool_context": {"forged": True}, "input_stream": "x", "data": 1},
879
+ )
880
+
881
+ assert received == [{"data": 1}]
882
+
883
+
884
+def test_google_adk_never_sends_its_context_object_to_the_backend():
885
+ # A schema property named like ADK's context parameter makes ADK inject
886
+ # its own ToolContext under that name; it must never be executed with.
887
+ received, result = _matrix_call(
888
+ "google_adk",
889
+ {
890
+ "type": "object",
891
+ "title": "ReservedProperty",
892
+ "properties": {
893
+ "query": {"type": "string"},
894
+ # Untyped, so validation alone would accept any object.
895
+ "tool_context": {"description": "A tool argument"},
896
+ },
897
+ "required": ["query", "tool_context"],
898
+ },
899
+ {"query": "q", "tool_context": "model value"},
900
+ )
901
+
902
+ assert received == []
903
+ assert "tool_context" in result["error"]
904
+
905
+
906
+# ---------------------------------------------------------------------------
907
+# AutoGen: both runtime paths share one validated execution (SUP-1643).
908
+# ---------------------------------------------------------------------------
909
+
910
+AUTOGEN_ROWS = ("autogen", "autogen_ag2")
911
+
912
+
913
+def _autogen_result(row: str, result: t.Any) -> t.Any:
914
+ """The tool's return value, unwrapped from AG2's function message."""
915
+ if row == "autogen_ag2":
916
+ # AG2 serializes a dict result with ``str()``.
917
+ return ast.literal_eval(result["content"])
918
+ return result
919
+
920
+
921
+@pytest.mark.parametrize("row", AUTOGEN_ROWS)
922
+def test_autogen_returns_structured_validation_error(row: str):
923
+ input_parameters, arguments = REJECTION_CASES["pattern_property_type"]
924
+
925
+ received, result = _matrix_call(row, input_parameters, arguments)
926
+
927
+ result = _autogen_result(row, result)
928
+ assert received == []
929
+ assert result["successful"] is False
930
+ assert result["data"] is None
931
+ assert "count_x" in result["error"]
932
+
933
+
934
+@pytest.mark.parametrize("row", AUTOGEN_ROWS)
935
+def test_autogen_omits_optional_arguments_the_model_left_out(row: str):
936
+ received, _ = _matrix_call(
937
+ row,
938
+ {
939
+ "type": "object",
940
+ "title": "OptionalArguments",
941
+ "properties": {
942
+ "query": {"type": "string"},
943
+ "note": {"type": "string"},
944
+ },
945
+ "required": ["query"],
946
+ },
947
+ {"query": "agents"},
948
+ )
949
+
950
+ assert received == [{"query": "agents"}]
951
+
952
+
953
+@pytest.mark.parametrize("row", AUTOGEN_ROWS)
954
+def test_autogen_restores_reserved_keyword_arguments(row: str):
955
+ received, _ = _matrix_call(
956
+ row,
957
+ {
958
+ "type": "object",
959
+ "title": "ReservedArguments",
960
+ "properties": {"from": {"type": "string"}},
961
+ "required": ["from"],
962
+ },
963
+ {"from_rs": "alice"},
964
+ )
965
+
966
+ assert received == [{"from": "alice"}]
967
+
968
+
969
+def test_autogen_run_json_normalizes_string_payload():
970
+ received, _ = _matrix_call(
971
+ "autogen",
972
+ PRESERVATION_CASES["nested_free_form"][0] | {"required": ["body"]},
973
+ '{"body": {"a": 1}}',
974
+ )
975
+
976
+ assert received == [{"body": {"a": 1}}]
977
+
978
+
979
+def test_autogen_run_json_logs_the_tool_call(caplog: pytest.LogCaptureFixture):
980
+ input_parameters, arguments = PRESERVATION_CASES["root_free_form"]
981
+
982
+ with caplog.at_level("INFO"):
983
+ received, _ = _matrix_call("autogen", input_parameters, arguments)
984
+
985
+ from autogen_core import EVENT_LOGGER_NAME
986
+ from autogen_core.logging import ToolCallEvent
987
+
988
+ events = [
989
+ record.msg
990
+ for record in caplog.records
991
+ if record.name == EVENT_LOGGER_NAME and isinstance(record.msg, ToolCallEvent)
992
+ ]
993
+ assert received == [arguments]
994
+ assert [json.loads(str(event))["arguments"] for event in events] == [arguments]
995
+
996
+
997
+@pytest.mark.parametrize(
998
+ ("input_parameters", "expected"),
999
+ [
1000
+ (
1001
+ PRESERVATION_CASES["root_free_form"][0],
1002
+ {"additionalProperties": True},
1003
+ ),
1004
+ (
1005
+ REJECTION_CASES["pattern_property_type"][0],
1006
+ {
1007
+ "additionalProperties": False,
1008
+ "patternProperties": {"^count_": {"type": "integer", "minimum": 1}},
1009
+ },
1010
+ ),
1011
+ ],
1012
+ ids=["root_free_form", "pattern_only"],
1013
+)
1014
+def test_autogen_schema_mirrors_root_openness(
1015
+ input_parameters: t.Dict[str, t.Any], expected: t.Dict[str, t.Any]
1016
+):
1017
+ wrapped = _matrix_provider("autogen").wrap_tool(
1018
+ _make_tool(input_parameters), lambda slug, arguments: {}
1019
+ )
1020
+
1021
+ parameters = wrapped.schema["parameters"]
1022
+
1023
+ assert {key: parameters.get(key) for key in expected} == expected