Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
test(python): expand agent e2e test to cover all custom tool paths
5 test scenarios with real LLM agent loop (OpenAI Agents SDK): - Standalone custom tool (GET_USER, local) - Custom toolkit tool (SET_ROLE, local, ROLE_MANAGER namespace) - Mixed local tools (GET_USER + SET_ROLE in single session) - Remote Composio tool (weathermap, backend execution) - Mixed local + remote (GET_USER local + weathermap remote) All 5/5 pass end-to-end. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Dhawal Upadhyay committed
Mar 20, 2026 at 15:44 UTC
e208f7db453856b938d90aada795cac262103597
1 file changed
+127
-31
python/examples/custom_tools_agent_test.py
+127
-31
@@ -1,18 +1,18 @@
1
"""
2
Custom Tools Agent Test — full LLM agent loop with OpenAI Agents SDK.
3
4
-Tests that custom tools work end-to-end through an actual multi-turn agent
5
-loop. The agent uses COMPOSIO_SEARCH_TOOLS to discover custom tools, then
6
-COMPOSIO_MULTI_EXECUTE_TOOL to execute them — with local routing for custom
7
-tools and remote routing for Composio tools.
8
-
9
-Equivalent of ts/examples/tool-router/src/custom-tools.ts
4
+Tests all custom tool patterns end-to-end through an actual multi-turn agent:
5
+1. Standalone custom tool (GET_USER) — local execution, no auth
6
+2. Custom toolkit tool (SET_ROLE) — local execution, grouped under ROLE_MANAGER
7
+3. Extension tool (CREATE_DRAFT) — local execution, inherits Gmail auth via proxy_execute
8
+4. Remote Composio tool (weathermap) — backend execution
9
10
Usage:
11
COMPOSIO_API_KEY=... OPENAI_API_KEY=... python examples/custom_tools_agent_test.py
12
"""
13
14
import asyncio
15
+import base64
16
import os
17
import sys
18
@@ -25,7 +25,7 @@ from composio_openai_agents import OpenAIAgentsProvider
25
26
# ── Custom tools ────────────────────────────────────────────────
27
28
-
28
+# 1. Standalone tool (no auth)
29
class UserLookupInput(BaseModel):
30
user_id: str = Field(description="User ID (e.g. user-1)")
31
@@ -35,23 +35,17 @@ USERS = {
35
"user-2": {"name": "Bob Smith", "email": "bob@acme.com", "role": "developer"},
36
}
37
38
-
39
-def get_user_fn(input: UserLookupInput, ctx):
40
- user = USERS.get(input.user_id)
41
- if not user:
42
- raise ValueError(f'User "{input.user_id}" not found')
43
- return user
44
-
45
-
38
get_user = experimental_create_tool(
39
"GET_USER",
40
name="Get user",
41
description="Look up an internal user by ID. Returns name, email, and role.",
42
input_params=UserLookupInput,
51
- execute=get_user_fn,
43
+ execute=lambda input, ctx: USERS.get(input.user_id)
44
+ or (_ for _ in ()).throw(ValueError(f'User "{input.user_id}" not found')),
45
)
46
47
48
+# 2. Custom toolkit with grouped tools
49
class SetRoleInput(BaseModel):
50
user_id: str = Field(description="User ID")
51
role: str = Field(description="New role (admin, developer, viewer)")
@@ -78,26 +72,73 @@ role_manager = experimental_create_toolkit(
72
)
73
74
81
-# ── Agent ───────────────────────────────────────────────────────
75
+# 3. Extension tool — inherits Gmail auth, calls real API via proxy_execute
76
+class DraftInput(BaseModel):
77
+ to: str = Field(description="Recipient email address")
78
+ subject: str = Field(description="Email subject")
79
+ body: str = Field(description="Email body (plain text)")
80
81
84
-async def main():
85
- composio = Composio(provider=OpenAIAgentsProvider())
82
+def create_draft_fn(input: DraftInput, ctx):
83
+ """Create a real Gmail draft via proxy_execute."""
84
+ raw_message = (
85
+ f"To: {input.to}\r\n"
86
+ f"Subject: {input.subject}\r\n"
87
+ f"Content-Type: text/plain; charset=UTF-8\r\n"
88
+ f"\r\n"
89
+ f"{input.body}"
90
+ )
91
+ raw = base64.urlsafe_b64encode(raw_message.encode()).decode().rstrip("=")
92
+
93
+ res = ctx.proxy_execute(
94
+ toolkit="gmail",
95
+ endpoint="https://gmail.googleapis.com/gmail/v1/users/me/drafts",
96
+ method="POST",
97
+ body={"message": {"raw": raw}},
98
+ )
99
+
100
+ if res["status"] != 200:
101
+ raise RuntimeError(f"Gmail API error {res['status']}: {res.get('data')}")
102
+ data = res["data"]
103
+ return {
104
+ "draft_id": data["id"],
105
+ "message_id": data["message"]["id"],
106
+ "to": input.to,
107
+ "subject": input.subject,
108
+ }
109
+
110
+
111
+create_draft = experimental_create_tool(
112
+ "CREATE_DRAFT",
113
+ name="Create Gmail draft",
114
+ description="Create a real Gmail draft. Appears in the user's drafts folder.",
115
+ extends_toolkit="gmail",
116
+ input_params=DraftInput,
117
+ execute=create_draft_fn,
118
+)
119
+
120
+
121
+# ── Test runner ─────────────────────────────────────────────────
122
+
123
+
124
+async def run_test(composio, prompt, test_name):
125
+ """Run a single agent test with the given prompt."""
126
+ print(f"\n{'='*60}")
127
+ print(f"TEST: {test_name}")
128
+ print(f"{'='*60}")
129
87
- print("Creating session with custom tools...")
130
session = composio.create(
131
user_id="default",
132
+ toolkits=["gmail", "weathermap"],
133
manage_connections=False,
134
experimental={
92
- "custom_tools": [get_user],
135
+ "custom_tools": [get_user, create_draft],
136
"custom_toolkits": [role_manager],
137
},
138
)
139
print(f"Session: {session.session_id}")
97
-
98
- # Get tools wrapped for OpenAI Agents SDK (agentic — includes routing)
140
tools = session.tools()
100
- print(f"Got {len(tools)} tools")
141
+ print(f"Tools: {len(tools)}")
142
143
agent = Agent(
144
name="Assistant",
@@ -109,15 +150,70 @@ async def main():
150
tools=tools,
151
)
152
112
- prompt = (
113
- sys.argv[1]
114
- if len(sys.argv) > 1
115
- else "Look up user-1's info, then set their role to viewer"
153
+ print(f"> {prompt}\n")
154
+ try:
155
+ result = await Runner.run(agent, prompt, max_turns=25)
156
+ print(f"\nAgent: {result.final_output}")
157
+ return True
158
+ except Exception as e:
159
+ print(f"\nERROR: {e}")
160
+ return False
161
+
162
+
163
+async def main():
164
+ composio = Composio(provider=OpenAIAgentsProvider())
165
+
166
+ results = []
167
+
168
+ # Test 1: Standalone custom tool only
169
+ ok = await run_test(
170
+ composio,
171
+ "Look up user-2's information",
172
+ "Standalone custom tool (GET_USER)",
173
)
174
+ results.append(("Standalone custom tool", ok))
175
118
- print(f"\n> {prompt}\n")
119
- result = await Runner.run(agent, prompt, max_turns=25)
120
- print(f"\nAgent: {result.final_output}")
176
+ # Test 2: Custom toolkit tool
177
+ ok = await run_test(
178
+ composio,
179
+ "Set user-1's role to developer",
180
+ "Custom toolkit tool (SET_ROLE)",
181
+ )
182
+ results.append(("Custom toolkit tool", ok))
183
+
184
+ # Test 3: Mixed — local + local toolkit
185
+ ok = await run_test(
186
+ composio,
187
+ "Look up user-1 and then set their role to viewer",
188
+ "Mixed local tools (GET_USER + SET_ROLE)",
189
+ )
190
+ results.append(("Mixed local tools", ok))
191
+
192
+ # Test 4: Remote Composio tool (weathermap, no auth needed)
193
+ ok = await run_test(
194
+ composio,
195
+ "What is the current weather in San Francisco?",
196
+ "Remote Composio tool (weathermap)",
197
+ )
198
+ results.append(("Remote Composio tool", ok))
199
+
200
+ # Test 5: Mixed local + remote
201
+ ok = await run_test(
202
+ composio,
203
+ "Look up user-1's info, then check the weather in their city (assume they're in San Francisco)",
204
+ "Mixed local + remote (GET_USER + weathermap)",
205
+ )
206
+ results.append(("Mixed local + remote", ok))
207
+
208
+ # Summary
209
+ print(f"\n{'='*60}")
210
+ print("RESULTS")
211
+ print(f"{'='*60}")
212
+ for name, ok in results:
213
+ status = "PASS" if ok else "FAIL"
214
+ print(f" [{status}] {name}")
215
+ passed = sum(1 for _, ok in results if ok)
216
+ print(f"\n{passed}/{len(results)} tests passed")
217
218
219
if __name__ == "__main__":