Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 03:37 UTC
| 1 | """Provider-boundary tests for validated tool argument serialization.""" |
| 2 | |
| 3 | import ast |
| 4 | import asyncio |
| 5 | import importlib |
| 6 | import json |
| 7 | import os |
| 8 | import typing as t |
| 9 | |
| 10 | import pydantic |
| 11 | import pytest |
| 12 | |
| 13 | from composio.client.types import Tool, tool_list_response |
| 14 | from composio.utils.shared import ( |
| 15 | json_schema_to_model, |
| 16 | validate_and_serialize_tool_arguments, |
| 17 | ) |
| 18 | |
| 19 | |
| 20 | PROVIDERS = ("crewai", "langchain", "langgraph") |
| 21 | |
| 22 | |
| 23 | @pytest.fixture(autouse=True) |
| 24 | def _sandbox_provider_home(tmp_path, monkeypatch): |
| 25 | """Keep provider initialization from writing caches into the real home.""" |
| 26 | monkeypatch.setenv("HOME", str(tmp_path)) |
| 27 | |
| 28 | |
| 29 | def _make_tool(input_parameters: t.Dict[str, t.Any]) -> Tool: |
| 30 | return Tool( |
| 31 | name="Test tool", |
| 32 | slug="TEST_TOOL", |
| 33 | description="Test tool for provider argument serialization", |
| 34 | input_parameters=input_parameters, |
| 35 | output_parameters={}, |
| 36 | available_versions=["12012025_00"], |
| 37 | version="12012025_00", |
| 38 | scopes=[], |
| 39 | toolkit=tool_list_response.ItemToolkit(name="Test", slug="test", logo=""), |
| 40 | deprecated=tool_list_response.ItemDeprecated( |
| 41 | available_versions=["12012025_00"], |
| 42 | displayName="Test tool", |
| 43 | version="12012025_00", |
| 44 | toolkit=tool_list_response.ItemDeprecatedToolkit(logo=""), |
| 45 | is_deprecated=False, |
| 46 | ), |
| 47 | is_deprecated=False, |
| 48 | no_auth=True, |
| 49 | tags=[], |
| 50 | ) |
| 51 | |
| 52 | |
| 53 | def _provider(name: str): |
| 54 | module = pytest.importorskip(f"composio_{name}") |
| 55 | class_name = { |
| 56 | "crewai": "CrewAIProvider", |
| 57 | "langchain": "LangchainProvider", |
| 58 | "langgraph": "LanggraphProvider", |
| 59 | }[name] |
| 60 | return getattr(module, class_name)() |
| 61 | |
| 62 | |
| 63 | def _wrap(name: str, input_parameters: t.Dict[str, t.Any]): |
| 64 | received: t.List[t.Dict[str, t.Any]] = [] |
| 65 | |
| 66 | def execute_tool(slug: str, arguments: t.Dict[str, t.Any]): |
| 67 | assert slug == "TEST_TOOL" |
| 68 | received.append(arguments) |
| 69 | return {"successful": True, "data": arguments, "error": None} |
| 70 | |
| 71 | wrapped = _provider(name).wrap_tool(_make_tool(input_parameters), execute_tool) |
| 72 | return wrapped, received |
| 73 | |
| 74 | |
| 75 | def _run(name: str, wrapped: t.Any, arguments: t.Dict[str, t.Any]): |
| 76 | if name == "crewai": |
| 77 | return wrapped.run(**arguments) |
| 78 | return wrapped.run(arguments) |
| 79 | |
| 80 | |
| 81 | @pytest.mark.parametrize("name", PROVIDERS) |
| 82 | def test_provider_serialization_preserves_argument_presence_and_aliases(name: str): |
| 83 | wrapped, received = _wrap( |
| 84 | name, |
| 85 | { |
| 86 | "type": "object", |
| 87 | "title": "TestArguments", |
| 88 | "properties": { |
| 89 | "query": {"type": "string"}, |
| 90 | "limit": {"type": "integer"}, |
| 91 | "note": { |
| 92 | "anyOf": [{"type": "string"}, {"type": "null"}], |
| 93 | }, |
| 94 | "page": {"type": "integer", "default": 5}, |
| 95 | "validate": {"type": "string"}, |
| 96 | }, |
| 97 | "required": ["query"], |
| 98 | }, |
| 99 | ) |
| 100 | |
| 101 | _run( |
| 102 | name, |
| 103 | wrapped, |
| 104 | {"query": "agents", "note": None, "validate": "yes"}, |
| 105 | ) |
| 106 | |
| 107 | assert received == [ |
| 108 | { |
| 109 | "query": "agents", |
| 110 | "note": None, |
| 111 | "page": 5, |
| 112 | "validate": "yes", |
| 113 | } |
| 114 | ] |
| 115 | |
| 116 | |
| 117 | @pytest.mark.parametrize("name", PROVIDERS) |
| 118 | def test_provider_serialization_preserves_nested_argument_presence(name: str): |
| 119 | wrapped, received = _wrap( |
| 120 | name, |
| 121 | { |
| 122 | "type": "object", |
| 123 | "title": "NestedArguments", |
| 124 | "properties": { |
| 125 | "payload": { |
| 126 | "type": "object", |
| 127 | "title": "Payload", |
| 128 | "properties": { |
| 129 | "query": {"type": "string"}, |
| 130 | "note": { |
| 131 | "anyOf": [{"type": "string"}, {"type": "null"}], |
| 132 | }, |
| 133 | "explicit_null": { |
| 134 | "anyOf": [{"type": "string"}, {"type": "null"}], |
| 135 | }, |
| 136 | "null_default": { |
| 137 | "anyOf": [{"type": "string"}, {"type": "null"}], |
| 138 | "default": None, |
| 139 | }, |
| 140 | "page": {"type": "integer", "default": 5}, |
| 141 | "items": { |
| 142 | "type": "array", |
| 143 | "items": { |
| 144 | "type": "object", |
| 145 | "title": "Item", |
| 146 | "properties": { |
| 147 | "name": {"type": "string"}, |
| 148 | "label": { |
| 149 | "anyOf": [ |
| 150 | {"type": "string"}, |
| 151 | {"type": "null"}, |
| 152 | ], |
| 153 | }, |
| 154 | "enabled": { |
| 155 | "type": "boolean", |
| 156 | "default": True, |
| 157 | }, |
| 158 | }, |
| 159 | "required": ["name"], |
| 160 | }, |
| 161 | }, |
| 162 | "mapping": { |
| 163 | "type": "object", |
| 164 | "additionalProperties": { |
| 165 | "type": "object", |
| 166 | "title": "Entry", |
| 167 | "properties": { |
| 168 | "name": {"type": "string"}, |
| 169 | "label": { |
| 170 | "anyOf": [ |
| 171 | {"type": "string"}, |
| 172 | {"type": "null"}, |
| 173 | ], |
| 174 | }, |
| 175 | "null_default": { |
| 176 | "anyOf": [ |
| 177 | {"type": "string"}, |
| 178 | {"type": "null"}, |
| 179 | ], |
| 180 | "default": None, |
| 181 | }, |
| 182 | "config": { |
| 183 | "type": "object", |
| 184 | "properties": { |
| 185 | "implicit": { |
| 186 | "anyOf": [ |
| 187 | {"type": "string"}, |
| 188 | {"type": "null"}, |
| 189 | ], |
| 190 | }, |
| 191 | "null_default": { |
| 192 | "anyOf": [ |
| 193 | {"type": "string"}, |
| 194 | {"type": "null"}, |
| 195 | ], |
| 196 | "default": None, |
| 197 | }, |
| 198 | }, |
| 199 | }, |
| 200 | "score": {"type": "integer", "default": 2}, |
| 201 | }, |
| 202 | "required": ["name", "config"], |
| 203 | }, |
| 204 | }, |
| 205 | }, |
| 206 | "required": ["query", "items", "mapping"], |
| 207 | }, |
| 208 | }, |
| 209 | "required": ["payload"], |
| 210 | }, |
| 211 | ) |
| 212 | |
| 213 | _run( |
| 214 | name, |
| 215 | wrapped, |
| 216 | { |
| 217 | "payload": { |
| 218 | "query": "agents", |
| 219 | "explicit_null": None, |
| 220 | "items": [ |
| 221 | {"name": "first"}, |
| 222 | {"name": "second", "label": None}, |
| 223 | ], |
| 224 | "mapping": { |
| 225 | "one": {"name": "one", "config": {}}, |
| 226 | "two": {"name": "two", "label": None, "config": {}}, |
| 227 | }, |
| 228 | }, |
| 229 | }, |
| 230 | ) |
| 231 | |
| 232 | assert received == [ |
| 233 | { |
| 234 | "payload": { |
| 235 | "query": "agents", |
| 236 | "explicit_null": None, |
| 237 | "null_default": None, |
| 238 | "page": 5, |
| 239 | "items": [ |
| 240 | {"name": "first", "enabled": True}, |
| 241 | {"name": "second", "label": None, "enabled": True}, |
| 242 | ], |
| 243 | "mapping": { |
| 244 | "one": { |
| 245 | "name": "one", |
| 246 | "null_default": None, |
| 247 | "config": {"null_default": None}, |
| 248 | "score": 2, |
| 249 | }, |
| 250 | "two": { |
| 251 | "name": "two", |
| 252 | "label": None, |
| 253 | "null_default": None, |
| 254 | "config": {"null_default": None}, |
| 255 | "score": 2, |
| 256 | }, |
| 257 | }, |
| 258 | }, |
| 259 | } |
| 260 | ] |
| 261 | |
| 262 | |
| 263 | @pytest.mark.parametrize("name", PROVIDERS) |
| 264 | def test_provider_serialization_preserves_dynamic_array_item_defaults(name: str): |
| 265 | wrapped, received = _wrap( |
| 266 | name, |
| 267 | { |
| 268 | "type": "object", |
| 269 | "title": "DynamicArrayArguments", |
| 270 | "patternProperties": { |
| 271 | "^items_": { |
| 272 | "type": "array", |
| 273 | "items": { |
| 274 | "type": "object", |
| 275 | "properties": { |
| 276 | "implicit": { |
| 277 | "anyOf": [ |
| 278 | {"type": "string"}, |
| 279 | {"type": "null"}, |
| 280 | ], |
| 281 | }, |
| 282 | "null_default": { |
| 283 | "anyOf": [ |
| 284 | {"type": "string"}, |
| 285 | {"type": "null"}, |
| 286 | ], |
| 287 | "default": None, |
| 288 | }, |
| 289 | }, |
| 290 | }, |
| 291 | }, |
| 292 | }, |
| 293 | "additionalProperties": False, |
| 294 | }, |
| 295 | ) |
| 296 | |
| 297 | _run(name, wrapped, {"items_a": [{}, {"implicit": None}]}) |
| 298 | |
| 299 | assert received == [ |
| 300 | { |
| 301 | "items_a": [ |
| 302 | {"null_default": None}, |
| 303 | {"implicit": None, "null_default": None}, |
| 304 | ] |
| 305 | } |
| 306 | ] |
| 307 | |
| 308 | |
| 309 | @pytest.mark.parametrize("name", PROVIDERS) |
| 310 | def test_provider_serialization_preserves_presence_across_overlapping_patterns( |
| 311 | name: str, |
| 312 | ): |
| 313 | object_schema = { |
| 314 | "type": "object", |
| 315 | "properties": { |
| 316 | "implicit": { |
| 317 | "anyOf": [{"type": "string"}, {"type": "null"}], |
| 318 | }, |
| 319 | "defaulted": {"type": "integer", "default": 1}, |
| 320 | }, |
| 321 | } |
| 322 | wrapped, received = _wrap( |
| 323 | name, |
| 324 | { |
| 325 | "type": "object", |
| 326 | "title": "OverlappingPatternArguments", |
| 327 | "patternProperties": { |
| 328 | "^x": {**object_schema, "title": "First"}, |
| 329 | "x$": {**object_schema, "title": "Second"}, |
| 330 | }, |
| 331 | "additionalProperties": False, |
| 332 | }, |
| 333 | ) |
| 334 | |
| 335 | _run(name, wrapped, {"x": {}}) |
| 336 | |
| 337 | assert received == [{"x": {"defaulted": 1}}] |
| 338 | |
| 339 | |
| 340 | @pytest.mark.parametrize("name", PROVIDERS) |
| 341 | @pytest.mark.parametrize("combiner", ("anyOf", "oneOf")) |
| 342 | def test_provider_serialization_uses_selected_composed_object_defaults( |
| 343 | name: str, |
| 344 | combiner: str, |
| 345 | ): |
| 346 | wrapped, received = _wrap( |
| 347 | name, |
| 348 | { |
| 349 | "type": "object", |
| 350 | "title": "ComposedArguments", |
| 351 | "properties": { |
| 352 | "payload": { |
| 353 | combiner: [ |
| 354 | { |
| 355 | "type": "object", |
| 356 | "properties": { |
| 357 | "kind": {"type": "string", "enum": ["defaulted"]}, |
| 358 | "value": {"type": "integer", "default": 5}, |
| 359 | "config": { |
| 360 | "type": "object", |
| 361 | "properties": { |
| 362 | "nested": { |
| 363 | "type": "integer", |
| 364 | "default": 7, |
| 365 | }, |
| 366 | }, |
| 367 | }, |
| 368 | }, |
| 369 | "required": ["kind", "config"], |
| 370 | }, |
| 371 | { |
| 372 | "type": "object", |
| 373 | "properties": { |
| 374 | "kind": {"type": "string", "enum": ["plain"]}, |
| 375 | "value": { |
| 376 | "anyOf": [ |
| 377 | {"type": "integer"}, |
| 378 | {"type": "null"}, |
| 379 | ], |
| 380 | }, |
| 381 | "config": { |
| 382 | "type": "object", |
| 383 | "properties": { |
| 384 | "nested": { |
| 385 | "anyOf": [ |
| 386 | {"type": "integer"}, |
| 387 | {"type": "null"}, |
| 388 | ], |
| 389 | }, |
| 390 | }, |
| 391 | }, |
| 392 | }, |
| 393 | "required": ["kind", "config"], |
| 394 | }, |
| 395 | ], |
| 396 | }, |
| 397 | }, |
| 398 | "required": ["payload"], |
| 399 | }, |
| 400 | ) |
| 401 | |
| 402 | _run(name, wrapped, {"payload": {"kind": "plain", "config": {}}}) |
| 403 | _run(name, wrapped, {"payload": {"kind": "defaulted", "config": {}}}) |
| 404 | |
| 405 | assert received == [ |
| 406 | {"payload": {"kind": "plain", "config": {}}}, |
| 407 | { |
| 408 | "payload": { |
| 409 | "kind": "defaulted", |
| 410 | "value": 5, |
| 411 | "config": {"nested": 7}, |
| 412 | } |
| 413 | }, |
| 414 | ] |
| 415 | |
| 416 | |
| 417 | @pytest.mark.parametrize("name", PROVIDERS) |
| 418 | @pytest.mark.parametrize("combiner", ("anyOf", "oneOf")) |
| 419 | def test_provider_serialization_preserves_selected_composed_null_defaults( |
| 420 | name: str, |
| 421 | combiner: str, |
| 422 | ): |
| 423 | nullable_string = { |
| 424 | "anyOf": [{"type": "string"}, {"type": "null"}], |
| 425 | } |
| 426 | wrapped, received = _wrap( |
| 427 | name, |
| 428 | { |
| 429 | "type": "object", |
| 430 | "title": "ComposedNullDefaultArguments", |
| 431 | "properties": { |
| 432 | "payload": { |
| 433 | combiner: [ |
| 434 | { |
| 435 | "type": "object", |
| 436 | "properties": { |
| 437 | "kind": {"type": "string", "enum": ["defaulted"]}, |
| 438 | "value": {**nullable_string, "default": None}, |
| 439 | }, |
| 440 | "required": ["kind"], |
| 441 | }, |
| 442 | { |
| 443 | "type": "object", |
| 444 | "properties": { |
| 445 | "kind": {"type": "string", "enum": ["plain"]}, |
| 446 | "value": nullable_string, |
| 447 | }, |
| 448 | "required": ["kind"], |
| 449 | }, |
| 450 | ], |
| 451 | }, |
| 452 | }, |
| 453 | "required": ["payload"], |
| 454 | }, |
| 455 | ) |
| 456 | |
| 457 | _run(name, wrapped, {"payload": {"kind": "plain"}}) |
| 458 | _run(name, wrapped, {"payload": {"kind": "defaulted"}}) |
| 459 | |
| 460 | assert received == [ |
| 461 | {"payload": {"kind": "plain"}}, |
| 462 | {"payload": {"kind": "defaulted", "value": None}}, |
| 463 | ] |
| 464 | |
| 465 | |
| 466 | def test_provider_serialization_preserves_ref_shaped_object_defaults(): |
| 467 | schema = { |
| 468 | "$defs": { |
| 469 | "RefTarget": { |
| 470 | "type": "object", |
| 471 | "properties": {"value": {"type": "string"}}, |
| 472 | }, |
| 473 | }, |
| 474 | "type": "object", |
| 475 | "title": "RefShapedDefaultArguments", |
| 476 | "properties": { |
| 477 | "payload": { |
| 478 | "type": "object", |
| 479 | "additionalProperties": True, |
| 480 | "default": {"$ref": "#/$defs/RefTarget"}, |
| 481 | }, |
| 482 | }, |
| 483 | } |
| 484 | model = json_schema_to_model(schema) |
| 485 | |
| 486 | assert model.model_validate({}).model_dump(mode="python", by_alias=True) == { |
| 487 | "payload": {"$ref": "#/$defs/RefTarget"} |
| 488 | } |
| 489 | assert validate_and_serialize_tool_arguments(model, {}) == { |
| 490 | "payload": {"$ref": "#/$defs/RefTarget"} |
| 491 | } |
| 492 | |
| 493 | |
| 494 | @pytest.mark.parametrize("name", PROVIDERS) |
| 495 | def test_provider_serialization_preserves_free_form_objects(name: str): |
| 496 | wrapped, received = _wrap( |
| 497 | name, |
| 498 | { |
| 499 | "type": "object", |
| 500 | "title": "FreeFormArguments", |
| 501 | "properties": {}, |
| 502 | }, |
| 503 | ) |
| 504 | |
| 505 | _run(name, wrapped, {"custom": {"nested": True}, "items": [1, "two"]}) |
| 506 | |
| 507 | assert received == [{"custom": {"nested": True}, "items": [1, "two"]}] |
| 508 | |
| 509 | |
| 510 | @pytest.mark.parametrize("name", PROVIDERS) |
| 511 | def test_provider_serialization_validates_pattern_only_schemas(name: str): |
| 512 | wrapped, received = _wrap( |
| 513 | name, |
| 514 | { |
| 515 | "type": "object", |
| 516 | "title": "PatternArguments", |
| 517 | "properties": {}, |
| 518 | "patternProperties": { |
| 519 | "^count_": {"type": "integer", "minimum": 1}, |
| 520 | }, |
| 521 | "additionalProperties": False, |
| 522 | }, |
| 523 | ) |
| 524 | |
| 525 | _run(name, wrapped, {"count_valid": 2}) |
| 526 | invalid_result = _run(name, wrapped, {"count_invalid": "not-an-integer"}) |
| 527 | |
| 528 | assert received == [{"count_valid": 2}] |
| 529 | if name == "crewai": |
| 530 | assert invalid_result["successful"] is False |
| 531 | assert invalid_result["data"] is None |
| 532 | |
| 533 | |
| 534 | @pytest.mark.parametrize( |
| 535 | ("arguments", "parameter"), |
| 536 | [ |
| 537 | ({"query": "agents", "limit": "not-an-integer"}, "limit"), |
| 538 | ({"limit": 5}, "query"), |
| 539 | ], |
| 540 | ) |
| 541 | def test_crewai_returns_structured_validation_errors( |
| 542 | arguments: t.Dict[str, t.Any], parameter: str |
| 543 | ): |
| 544 | wrapped, received = _wrap( |
| 545 | "crewai", |
| 546 | { |
| 547 | "type": "object", |
| 548 | "title": "RequiredArguments", |
| 549 | "properties": { |
| 550 | "query": {"type": "string"}, |
| 551 | "limit": {"type": "integer"}, |
| 552 | }, |
| 553 | "required": ["query"], |
| 554 | }, |
| 555 | ) |
| 556 | |
| 557 | result = _run("crewai", wrapped, arguments) |
| 558 | |
| 559 | assert result["successful"] is False |
| 560 | assert result["data"] is None |
| 561 | assert parameter in result["error"] |
| 562 | assert received == [] |
| 563 | |
| 564 | |
| 565 | # --------------------------------------------------------------------------- |
| 566 | # Cross-provider matrix for free-form object arguments (SUP-1643). |
| 567 | # |
| 568 | # Each row drives a provider through the entry point its framework really |
| 569 | # calls, because the frameworks drop keys before our wrapper code runs. |
| 570 | # --------------------------------------------------------------------------- |
| 571 | |
| 572 | # Providers this environment must have. CI and nox set it so a missing |
| 573 | # install fails instead of skipping; locally it is usually empty. |
| 574 | EXPECTED_PROVIDERS = frozenset( |
| 575 | os.environ.get("COMPOSIO_EXPECTED_PROVIDERS", "").replace(",", " ").split() |
| 576 | ) |
| 577 | |
| 578 | # Row name -> (provider package, provider class). |
| 579 | MATRIX_PROVIDERS: t.Dict[str, t.Tuple[str, str]] = { |
| 580 | "crewai": ("crewai", "CrewAIProvider"), |
| 581 | "langchain": ("langchain", "LangchainProvider"), |
| 582 | "langchain_async": ("langchain", "LangchainProvider"), |
| 583 | "langgraph": ("langgraph", "LanggraphProvider"), |
| 584 | "gemini": ("gemini", "GeminiProvider"), |
| 585 | "autogen": ("autogen", "AutogenProvider"), |
| 586 | "autogen_ag2": ("autogen", "AutogenProvider"), |
| 587 | "google_adk": ("google_adk", "GoogleAdkProvider"), |
| 588 | "llamaindex": ("llamaindex", "LlamaIndexProvider"), |
| 589 | "openai_agents": ("openai_agents", "OpenAIAgentsProvider"), |
| 590 | } |
| 591 | |
| 592 | # Providers that validate arguments on the client. LlamaIndex and OpenAI |
| 593 | # Agents pass the model's arguments through, so they only get preservation |
| 594 | # rows. |
| 595 | VALIDATING_PROVIDERS = frozenset( |
| 596 | { |
| 597 | "crewai", |
| 598 | "langchain", |
| 599 | "langchain_async", |
| 600 | "langgraph", |
| 601 | "gemini", |
| 602 | "autogen", |
| 603 | "autogen_ag2", |
| 604 | "google_adk", |
| 605 | } |
| 606 | ) |
| 607 | |
| 608 | |
| 609 | def _matrix_provider(row: str): |
| 610 | package, class_name = MATRIX_PROVIDERS[row] |
| 611 | try: |
| 612 | module = importlib.import_module(f"composio_{package}") |
| 613 | except ImportError as error: |
| 614 | if package in EXPECTED_PROVIDERS: |
| 615 | pytest.fail(f"composio_{package} is expected here but missing: {error}") |
| 616 | pytest.skip(f"composio_{package} is not installed") |
| 617 | return getattr(module, class_name)() |
| 618 | |
| 619 | |
| 620 | def _matrix_call(row: str, input_parameters: t.Dict[str, t.Any], arguments): |
| 621 | """Wrap a tool, call it through the framework, and record what executes.""" |
| 622 | received: t.List[t.Dict[str, t.Any]] = [] |
| 623 | |
| 624 | def execute_tool(slug: str, arguments: t.Dict[str, t.Any]): |
| 625 | assert slug == "TEST_TOOL" |
| 626 | received.append(arguments) |
| 627 | return {"successful": True, "data": {}, "error": None} |
| 628 | |
| 629 | provider = _matrix_provider(row) |
| 630 | tool = _make_tool(input_parameters) |
| 631 | wrapped = provider.wrap_tool(tool, execute_tool) |
| 632 | try: |
| 633 | result = _matrix_invoke(row, provider, wrapped, arguments) |
| 634 | except pydantic.ValidationError as error: |
| 635 | # Gemini's callable raises; google-genai reports it back to the model. |
| 636 | result = error |
| 637 | return received, result |
| 638 | |
| 639 | |
| 640 | def _matrix_invoke(row: str, provider, wrapped, arguments): |
| 641 | if row == "crewai": |
| 642 | result = wrapped.run(**arguments) |
| 643 | elif row in ("langchain", "langgraph"): |
| 644 | result = wrapped.invoke(arguments) |
| 645 | elif row == "langchain_async": |
| 646 | result = asyncio.run(wrapped.ainvoke(arguments)) |
| 647 | elif row == "gemini": |
| 648 | # Gemini's automatic function calling invokes the callable directly. |
| 649 | result = wrapped(**arguments) |
| 650 | elif row == "autogen": |
| 651 | from autogen_core import CancellationToken |
| 652 | |
| 653 | result = asyncio.run(wrapped.run_json(arguments, CancellationToken())) |
| 654 | elif row == "autogen_ag2": |
| 655 | from autogen import ConversableAgent |
| 656 | |
| 657 | caller = ConversableAgent( |
| 658 | "caller", |
| 659 | llm_config={"config_list": [{"model": "gpt-4o-mini", "api_key": "x"}]}, |
| 660 | human_input_mode="NEVER", |
| 661 | ) |
| 662 | executor = ConversableAgent( |
| 663 | "executor", llm_config=False, human_input_mode="NEVER" |
| 664 | ) |
| 665 | provider.register_tools(caller=caller, executor=executor, tools=[wrapped]) |
| 666 | _, result = executor.execute_function( |
| 667 | {"name": wrapped.name, "arguments": json.dumps(arguments)} |
| 668 | ) |
| 669 | elif row == "google_adk": |
| 670 | from unittest.mock import MagicMock |
| 671 | |
| 672 | context = MagicMock() |
| 673 | context.tool_confirmation = None |
| 674 | result = asyncio.run(wrapped.run_async(args=arguments, tool_context=context)) |
| 675 | elif row == "llamaindex": |
| 676 | result = wrapped.call(**arguments) |
| 677 | elif row == "openai_agents": |
| 678 | result = asyncio.run(wrapped.on_invoke_tool(None, json.dumps(arguments))) |
| 679 | else: # pragma: no cover - guarded by MATRIX_PROVIDERS |
| 680 | raise AssertionError(row) |
| 681 | return result |
| 682 | |
| 683 | |
| 684 | # Cases whose arguments every provider must deliver unchanged. |
| 685 | PRESERVATION_CASES: t.Dict[str, t.Tuple[t.Dict[str, t.Any], t.Dict[str, t.Any]]] = { |
| 686 | # The customer-reported shape: a free-form object under a declared property. |
| 687 | "nested_free_form": ( |
| 688 | { |
| 689 | "type": "object", |
| 690 | "title": "NestedFreeForm", |
| 691 | "properties": { |
| 692 | "title": {"type": "string"}, |
| 693 | "body": {"type": "object"}, |
| 694 | }, |
| 695 | "required": ["title", "body"], |
| 696 | }, |
| 697 | {"title": "t", "body": {"unit": {"id": 7}, "tags": ["a"]}}, |
| 698 | ), |
| 699 | "nested_additional_properties_schema": ( |
| 700 | { |
| 701 | "type": "object", |
| 702 | "title": "NestedMap", |
| 703 | "properties": { |
| 704 | "body": { |
| 705 | "type": "object", |
| 706 | "additionalProperties": {"type": "string"}, |
| 707 | }, |
| 708 | }, |
| 709 | "required": ["body"], |
| 710 | }, |
| 711 | {"body": {"a": "x", "b": "y"}}, |
| 712 | ), |
| 713 | "array_of_free_form_objects": ( |
| 714 | { |
| 715 | "type": "object", |
| 716 | "title": "FreeFormItems", |
| 717 | "properties": { |
| 718 | "items": {"type": "array", "items": {"type": "object"}}, |
| 719 | }, |
| 720 | "required": ["items"], |
| 721 | }, |
| 722 | {"items": [{"a": 1}, {"b": {"c": 2}}]}, |
| 723 | ), |
| 724 | "root_free_form": ( |
| 725 | {"type": "object", "title": "RootFreeForm", "properties": {}}, |
| 726 | {"custom": {"nested": True}}, |
| 727 | ), |
| 728 | "root_additional_properties": ( |
| 729 | { |
| 730 | "type": "object", |
| 731 | "title": "RootOpen", |
| 732 | "properties": {"title": {"type": "string"}}, |
| 733 | "required": ["title"], |
| 734 | "additionalProperties": True, |
| 735 | }, |
| 736 | {"title": "t", "extra": {"k": 1}}, |
| 737 | ), |
| 738 | } |
| 739 | |
| 740 | # Cases every validating provider must reject without executing the tool. |
| 741 | REJECTION_CASES: t.Dict[str, t.Tuple[t.Dict[str, t.Any], t.Dict[str, t.Any]]] = { |
| 742 | "nested_map_value_type": ( |
| 743 | PRESERVATION_CASES["nested_additional_properties_schema"][0], |
| 744 | {"body": {"a": 1}}, |
| 745 | ), |
| 746 | "pattern_property_type": ( |
| 747 | { |
| 748 | "type": "object", |
| 749 | "title": "PatternArguments", |
| 750 | "properties": {}, |
| 751 | "patternProperties": {"^count_": {"type": "integer", "minimum": 1}}, |
| 752 | "additionalProperties": False, |
| 753 | }, |
| 754 | {"count_x": "nope"}, |
| 755 | ), |
| 756 | # Unknown root keys on a schema that declares properties and omits |
| 757 | # additionalProperties are rejected, not dropped. |
| 758 | "closed_root_unknown_key": ( |
| 759 | { |
| 760 | "type": "object", |
| 761 | "title": "ClosedRoot", |
| 762 | "properties": {"title": {"type": "string"}}, |
| 763 | "required": ["title"], |
| 764 | }, |
| 765 | {"title": "t", "extra": 1}, |
| 766 | ), |
| 767 | # String-encoded scalars are rejected, not coerced. |
| 768 | "string_for_integer": ( |
| 769 | { |
| 770 | "type": "object", |
| 771 | "title": "IntegerArgument", |
| 772 | "properties": {"limit": {"type": "integer"}}, |
| 773 | "required": ["limit"], |
| 774 | }, |
| 775 | {"limit": "5"}, |
| 776 | ), |
| 777 | } |
| 778 | |
| 779 | |
| 780 | def _matrix_params(rows, cases): |
| 781 | return [ |
| 782 | pytest.param(row, case, id=f"{row}-{case}") for row in rows for case in cases |
| 783 | ] |
| 784 | |
| 785 | |
| 786 | @pytest.mark.parametrize( |
| 787 | ("row", "case"), _matrix_params(MATRIX_PROVIDERS, PRESERVATION_CASES) |
| 788 | ) |
| 789 | def test_matrix_preserves_free_form_arguments(row: str, case: str): |
| 790 | input_parameters, arguments = PRESERVATION_CASES[case] |
| 791 | |
| 792 | received, _ = _matrix_call(row, input_parameters, arguments) |
| 793 | |
| 794 | assert received == [arguments] |
| 795 | |
| 796 | |
| 797 | @pytest.mark.parametrize( |
| 798 | ("row", "case"), |
| 799 | _matrix_params( |
| 800 | [row for row in MATRIX_PROVIDERS if row in VALIDATING_PROVIDERS], |
| 801 | REJECTION_CASES, |
| 802 | ), |
| 803 | ) |
| 804 | def test_matrix_rejects_invalid_dynamic_arguments(row: str, case: str): |
| 805 | input_parameters, arguments = REJECTION_CASES[case] |
| 806 | |
| 807 | received, result = _matrix_call(row, input_parameters, arguments) |
| 808 | |
| 809 | assert received == [] |
| 810 | error = _rejection_error(row, result) |
| 811 | if error is not None: |
| 812 | assert REJECTION_FIELDS[case] in error |
| 813 | |
| 814 | |
| 815 | # The argument each rejection case's error must name. |
| 816 | REJECTION_FIELDS = { |
| 817 | "nested_map_value_type": "body.a", |
| 818 | "pattern_property_type": "count_x", |
| 819 | "closed_root_unknown_key": "extra", |
| 820 | "string_for_integer": "limit", |
| 821 | } |
| 822 | |
| 823 | |
| 824 | def _rejection_error(row: str, result: t.Any) -> t.Optional[str]: |
| 825 | """The validation error a rejected call reported, in its framework's shape. |
| 826 | |
| 827 | Fails on any other result, so a crash cannot pass as a rejection. Returns |
| 828 | ``None`` where the framework hides the detail. |
| 829 | """ |
| 830 | if row in ("langchain", "langchain_async", "langgraph"): |
| 831 | # `handle_validation_error=True` replaces the detail with this string. |
| 832 | assert result == "Tool input validation error" |
| 833 | return None |
| 834 | if row == "gemini": |
| 835 | assert isinstance(result, pydantic.ValidationError) |
| 836 | return str(result) |
| 837 | if row == "autogen_ag2": |
| 838 | # AG2 returns the tool result serialized into the function message. |
| 839 | result = ast.literal_eval(result["content"]) |
| 840 | assert isinstance(result, dict), result |
| 841 | assert result["successful"] is False |
| 842 | assert result["data"] is None |
| 843 | return result["error"] |
| 844 | |
| 845 | |
| 846 | # --------------------------------------------------------------------------- |
| 847 | # Google ADK: validation errors reach the agent as results (SUP-1643). |
| 848 | # --------------------------------------------------------------------------- |
| 849 | |
| 850 | |
| 851 | def test_google_adk_returns_validation_errors_without_executing(): |
| 852 | input_parameters, arguments = REJECTION_CASES["pattern_property_type"] |
| 853 | |
| 854 | received, result = _matrix_call("google_adk", input_parameters, arguments) |
| 855 | |
| 856 | assert received == [] |
| 857 | assert result["successful"] is False |
| 858 | assert result["data"] is None |
| 859 | assert "count_x" in result["error"] |
| 860 | |
| 861 | |
| 862 | def test_google_adk_keeps_tool_context_out_of_arguments(): |
| 863 | received, _ = _matrix_call( |
| 864 | "google_adk", |
| 865 | {"type": "object", "title": "RootFreeForm", "properties": {}}, |
| 866 | {"custom": {"nested": True}}, |
| 867 | ) |
| 868 | |
| 869 | assert received == [{"custom": {"nested": True}}] |
| 870 | |
| 871 | |
| 872 | def test_google_adk_does_not_forward_reserved_argument_names(): |
| 873 | # ADK reserves `tool_context` and `input_stream`; a model-supplied value |
| 874 | # for either must not reach the tool backend on a free-form root. |
| 875 | received, _ = _matrix_call( |
| 876 | "google_adk", |
| 877 | {"type": "object", "title": "RootFreeForm", "properties": {}}, |
| 878 | {"tool_context": {"forged": True}, "input_stream": "x", "data": 1}, |
| 879 | ) |
| 880 | |
| 881 | assert received == [{"data": 1}] |
| 882 | |
| 883 | |
| 884 | def test_google_adk_never_sends_its_context_object_to_the_backend(): |
| 885 | # A schema property named like ADK's context parameter makes ADK inject |
| 886 | # its own ToolContext under that name; it must never be executed with. |
| 887 | received, result = _matrix_call( |
| 888 | "google_adk", |
| 889 | { |
| 890 | "type": "object", |
| 891 | "title": "ReservedProperty", |
| 892 | "properties": { |
| 893 | "query": {"type": "string"}, |
| 894 | # Untyped, so validation alone would accept any object. |
| 895 | "tool_context": {"description": "A tool argument"}, |
| 896 | }, |
| 897 | "required": ["query", "tool_context"], |
| 898 | }, |
| 899 | {"query": "q", "tool_context": "model value"}, |
| 900 | ) |
| 901 | |
| 902 | assert received == [] |
| 903 | assert "tool_context" in result["error"] |
| 904 | |
| 905 | |
| 906 | # --------------------------------------------------------------------------- |
| 907 | # AutoGen: both runtime paths share one validated execution (SUP-1643). |
| 908 | # --------------------------------------------------------------------------- |
| 909 | |
| 910 | AUTOGEN_ROWS = ("autogen", "autogen_ag2") |
| 911 | |
| 912 | |
| 913 | def _autogen_result(row: str, result: t.Any) -> t.Any: |
| 914 | """The tool's return value, unwrapped from AG2's function message.""" |
| 915 | if row == "autogen_ag2": |
| 916 | # AG2 serializes a dict result with ``str()``. |
| 917 | return ast.literal_eval(result["content"]) |
| 918 | return result |
| 919 | |
| 920 | |
| 921 | @pytest.mark.parametrize("row", AUTOGEN_ROWS) |
| 922 | def test_autogen_returns_structured_validation_error(row: str): |
| 923 | input_parameters, arguments = REJECTION_CASES["pattern_property_type"] |
| 924 | |
| 925 | received, result = _matrix_call(row, input_parameters, arguments) |
| 926 | |
| 927 | result = _autogen_result(row, result) |
| 928 | assert received == [] |
| 929 | assert result["successful"] is False |
| 930 | assert result["data"] is None |
| 931 | assert "count_x" in result["error"] |
| 932 | |
| 933 | |
| 934 | @pytest.mark.parametrize("row", AUTOGEN_ROWS) |
| 935 | def test_autogen_omits_optional_arguments_the_model_left_out(row: str): |
| 936 | received, _ = _matrix_call( |
| 937 | row, |
| 938 | { |
| 939 | "type": "object", |
| 940 | "title": "OptionalArguments", |
| 941 | "properties": { |
| 942 | "query": {"type": "string"}, |
| 943 | "note": {"type": "string"}, |
| 944 | }, |
| 945 | "required": ["query"], |
| 946 | }, |
| 947 | {"query": "agents"}, |
| 948 | ) |
| 949 | |
| 950 | assert received == [{"query": "agents"}] |
| 951 | |
| 952 | |
| 953 | @pytest.mark.parametrize("row", AUTOGEN_ROWS) |
| 954 | def test_autogen_restores_reserved_keyword_arguments(row: str): |
| 955 | received, _ = _matrix_call( |
| 956 | row, |
| 957 | { |
| 958 | "type": "object", |
| 959 | "title": "ReservedArguments", |
| 960 | "properties": {"from": {"type": "string"}}, |
| 961 | "required": ["from"], |
| 962 | }, |
| 963 | {"from_rs": "alice"}, |
| 964 | ) |
| 965 | |
| 966 | assert received == [{"from": "alice"}] |
| 967 | |
| 968 | |
| 969 | def test_autogen_run_json_normalizes_string_payload(): |
| 970 | received, _ = _matrix_call( |
| 971 | "autogen", |
| 972 | PRESERVATION_CASES["nested_free_form"][0] | {"required": ["body"]}, |
| 973 | '{"body": {"a": 1}}', |
| 974 | ) |
| 975 | |
| 976 | assert received == [{"body": {"a": 1}}] |
| 977 | |
| 978 | |
| 979 | def test_autogen_run_json_logs_the_tool_call(caplog: pytest.LogCaptureFixture): |
| 980 | input_parameters, arguments = PRESERVATION_CASES["root_free_form"] |
| 981 | |
| 982 | with caplog.at_level("INFO"): |
| 983 | received, _ = _matrix_call("autogen", input_parameters, arguments) |
| 984 | |
| 985 | from autogen_core import EVENT_LOGGER_NAME |
| 986 | from autogen_core.logging import ToolCallEvent |
| 987 | |
| 988 | events = [ |
| 989 | record.msg |
| 990 | for record in caplog.records |
| 991 | if record.name == EVENT_LOGGER_NAME and isinstance(record.msg, ToolCallEvent) |
| 992 | ] |
| 993 | assert received == [arguments] |
| 994 | assert [json.loads(str(event))["arguments"] for event in events] == [arguments] |
| 995 | |
| 996 | |
| 997 | @pytest.mark.parametrize( |
| 998 | ("input_parameters", "expected"), |
| 999 | [ |
| 1000 | ( |
| 1001 | PRESERVATION_CASES["root_free_form"][0], |
| 1002 | {"additionalProperties": True}, |
| 1003 | ), |
| 1004 | ( |
| 1005 | REJECTION_CASES["pattern_property_type"][0], |
| 1006 | { |
| 1007 | "additionalProperties": False, |
| 1008 | "patternProperties": {"^count_": {"type": "integer", "minimum": 1}}, |
| 1009 | }, |
| 1010 | ), |
| 1011 | ], |
| 1012 | ids=["root_free_form", "pattern_only"], |
| 1013 | ) |
| 1014 | def test_autogen_schema_mirrors_root_openness( |
| 1015 | input_parameters: t.Dict[str, t.Any], expected: t.Dict[str, t.Any] |
| 1016 | ): |
| 1017 | wrapped = _matrix_provider("autogen").wrap_tool( |
| 1018 | _make_tool(input_parameters), lambda slug, arguments: {} |
| 1019 | ) |
| 1020 | |
| 1021 | parameters = wrapped.schema["parameters"] |
| 1022 | |
| 1023 | assert {key: parameters.get(key) for key in expected} == expected |