@devries-cameron20 / composio-1 / commits / e208f7db4
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

test(python): expand agent e2e test to cover all custom tool paths

5 test scenarios with real LLM agent loop (OpenAI Agents SDK): - Standalone custom tool (GET_USER, local) - Custom toolkit tool (SET_ROLE, local, ROLE_MANAGER namespace) - Mixed local tools (GET_USER + SET_ROLE in single session) - Remote Composio tool (weathermap, backend execution) - Mixed local + remote (GET_USER local + weathermap remote) All 5/5 pass end-to-end. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Dhawal Upadhyay committed Mar 20, 2026 at 15:44 UTC e208f7db453856b938d90aada795cac262103597
1 file changed +127 -31
python/examples/custom_tools_agent_test.py
+127 -31
@@ -1,18 +1,18 @@
1 """
2 Custom Tools Agent Test — full LLM agent loop with OpenAI Agents SDK.
3
4 -Tests that custom tools work end-to-end through an actual multi-turn agent
5 -loop. The agent uses COMPOSIO_SEARCH_TOOLS to discover custom tools, then
6 -COMPOSIO_MULTI_EXECUTE_TOOL to execute them — with local routing for custom
7 -tools and remote routing for Composio tools.
8 -
9 -Equivalent of ts/examples/tool-router/src/custom-tools.ts
4 +Tests all custom tool patterns end-to-end through an actual multi-turn agent:
5 +1. Standalone custom tool (GET_USER) — local execution, no auth
6 +2. Custom toolkit tool (SET_ROLE) — local execution, grouped under ROLE_MANAGER
7 +3. Extension tool (CREATE_DRAFT) — local execution, inherits Gmail auth via proxy_execute
8 +4. Remote Composio tool (weathermap) — backend execution
9
10 Usage:
11 COMPOSIO_API_KEY=... OPENAI_API_KEY=... python examples/custom_tools_agent_test.py
12 """
13
14 import asyncio
15 +import base64
16 import os
17 import sys
18
@@ -25,7 +25,7 @@ from composio_openai_agents import OpenAIAgentsProvider
25
26 # ── Custom tools ────────────────────────────────────────────────
27
28 -
28 +# 1. Standalone tool (no auth)
29 class UserLookupInput(BaseModel):
30 user_id: str = Field(description="User ID (e.g. user-1)")
31
@@ -35,23 +35,17 @@ USERS = {
35 "user-2": {"name": "Bob Smith", "email": "bob@acme.com", "role": "developer"},
36 }
37
38 -
39 -def get_user_fn(input: UserLookupInput, ctx):
40 - user = USERS.get(input.user_id)
41 - if not user:
42 - raise ValueError(f'User "{input.user_id}" not found')
43 - return user
44 -
45 -
38 get_user = experimental_create_tool(
39 "GET_USER",
40 name="Get user",
41 description="Look up an internal user by ID. Returns name, email, and role.",
42 input_params=UserLookupInput,
51 - execute=get_user_fn,
43 + execute=lambda input, ctx: USERS.get(input.user_id)
44 + or (_ for _ in ()).throw(ValueError(f'User "{input.user_id}" not found')),
45 )
46
47
48 +# 2. Custom toolkit with grouped tools
49 class SetRoleInput(BaseModel):
50 user_id: str = Field(description="User ID")
51 role: str = Field(description="New role (admin, developer, viewer)")
@@ -78,26 +72,73 @@ role_manager = experimental_create_toolkit(
72 )
73
74
81 -# ── Agent ───────────────────────────────────────────────────────
75 +# 3. Extension tool — inherits Gmail auth, calls real API via proxy_execute
76 +class DraftInput(BaseModel):
77 + to: str = Field(description="Recipient email address")
78 + subject: str = Field(description="Email subject")
79 + body: str = Field(description="Email body (plain text)")
80
81
84 -async def main():
85 - composio = Composio(provider=OpenAIAgentsProvider())
82 +def create_draft_fn(input: DraftInput, ctx):
83 + """Create a real Gmail draft via proxy_execute."""
84 + raw_message = (
85 + f"To: {input.to}\r\n"
86 + f"Subject: {input.subject}\r\n"
87 + f"Content-Type: text/plain; charset=UTF-8\r\n"
88 + f"\r\n"
89 + f"{input.body}"
90 + )
91 + raw = base64.urlsafe_b64encode(raw_message.encode()).decode().rstrip("=")
92 +
93 + res = ctx.proxy_execute(
94 + toolkit="gmail",
95 + endpoint="https://gmail.googleapis.com/gmail/v1/users/me/drafts",
96 + method="POST",
97 + body={"message": {"raw": raw}},
98 + )
99 +
100 + if res["status"] != 200:
101 + raise RuntimeError(f"Gmail API error {res['status']}: {res.get('data')}")
102 + data = res["data"]
103 + return {
104 + "draft_id": data["id"],
105 + "message_id": data["message"]["id"],
106 + "to": input.to,
107 + "subject": input.subject,
108 + }
109 +
110 +
111 +create_draft = experimental_create_tool(
112 + "CREATE_DRAFT",
113 + name="Create Gmail draft",
114 + description="Create a real Gmail draft. Appears in the user's drafts folder.",
115 + extends_toolkit="gmail",
116 + input_params=DraftInput,
117 + execute=create_draft_fn,
118 +)
119 +
120 +
121 +# ── Test runner ─────────────────────────────────────────────────
122 +
123 +
124 +async def run_test(composio, prompt, test_name):
125 + """Run a single agent test with the given prompt."""
126 + print(f"\n{'='*60}")
127 + print(f"TEST: {test_name}")
128 + print(f"{'='*60}")
129
87 - print("Creating session with custom tools...")
130 session = composio.create(
131 user_id="default",
132 + toolkits=["gmail", "weathermap"],
133 manage_connections=False,
134 experimental={
92 - "custom_tools": [get_user],
135 + "custom_tools": [get_user, create_draft],
136 "custom_toolkits": [role_manager],
137 },
138 )
139 print(f"Session: {session.session_id}")
97 -
98 - # Get tools wrapped for OpenAI Agents SDK (agentic — includes routing)
140 tools = session.tools()
100 - print(f"Got {len(tools)} tools")
141 + print(f"Tools: {len(tools)}")
142
143 agent = Agent(
144 name="Assistant",
@@ -109,15 +150,70 @@ async def main():
150 tools=tools,
151 )
152
112 - prompt = (
113 - sys.argv[1]
114 - if len(sys.argv) > 1
115 - else "Look up user-1's info, then set their role to viewer"
153 + print(f"> {prompt}\n")
154 + try:
155 + result = await Runner.run(agent, prompt, max_turns=25)
156 + print(f"\nAgent: {result.final_output}")
157 + return True
158 + except Exception as e:
159 + print(f"\nERROR: {e}")
160 + return False
161 +
162 +
163 +async def main():
164 + composio = Composio(provider=OpenAIAgentsProvider())
165 +
166 + results = []
167 +
168 + # Test 1: Standalone custom tool only
169 + ok = await run_test(
170 + composio,
171 + "Look up user-2's information",
172 + "Standalone custom tool (GET_USER)",
173 )
174 + results.append(("Standalone custom tool", ok))
175
118 - print(f"\n> {prompt}\n")
119 - result = await Runner.run(agent, prompt, max_turns=25)
120 - print(f"\nAgent: {result.final_output}")
176 + # Test 2: Custom toolkit tool
177 + ok = await run_test(
178 + composio,
179 + "Set user-1's role to developer",
180 + "Custom toolkit tool (SET_ROLE)",
181 + )
182 + results.append(("Custom toolkit tool", ok))
183 +
184 + # Test 3: Mixed — local + local toolkit
185 + ok = await run_test(
186 + composio,
187 + "Look up user-1 and then set their role to viewer",
188 + "Mixed local tools (GET_USER + SET_ROLE)",
189 + )
190 + results.append(("Mixed local tools", ok))
191 +
192 + # Test 4: Remote Composio tool (weathermap, no auth needed)
193 + ok = await run_test(
194 + composio,
195 + "What is the current weather in San Francisco?",
196 + "Remote Composio tool (weathermap)",
197 + )
198 + results.append(("Remote Composio tool", ok))
199 +
200 + # Test 5: Mixed local + remote
201 + ok = await run_test(
202 + composio,
203 + "Look up user-1's info, then check the weather in their city (assume they're in San Francisco)",
204 + "Mixed local + remote (GET_USER + weathermap)",
205 + )
206 + results.append(("Mixed local + remote", ok))
207 +
208 + # Summary
209 + print(f"\n{'='*60}")
210 + print("RESULTS")
211 + print(f"{'='*60}")
212 + for name, ok in results:
213 + status = "PASS" if ok else "FAIL"
214 + print(f" [{status}] {name}")
215 + passed = sum(1 for _, ok in results if ok)
216 + print(f"\n{passed}/{len(results)} tests passed")
217
218
219 if __name__ == "__main__":