@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 04:45 UTC

next
ts 409 lines 15 KB
Raw
1 import { execFileSync } from 'node:child_process';
2 import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
3 import { tmpdir } from 'node:os';
4 import { join } from 'node:path';
5 import { describe, expect, it } from 'vitest';
6 import { experimental_createPythonWorkbenchHelperSource } from '../../src/workbench';
7
8 describe('experimental_createPythonWorkbenchHelperSource', () => {
9 it('emits the Apollo-parity Python helpers without remote workbench-only helpers', () => {
10 const source = experimental_createPythonWorkbenchHelperSource();
11
12 expect(source).toContain('def run_composio_tool(');
13 expect(source).toContain('def invoke_llm(');
14 expect(source).toContain('def web_search(');
15 expect(source).toContain('def proxy_execute(');
16 expect(source).toContain('"x-api-key": api_key');
17 expect(source).toContain('/api/v3/tool_router/session/%s/execute');
18 // proxy_execute uses the public session route with the project key.
19 expect(source).toContain('/api/v3/tool_router/session/%s/proxy_execute');
20 // Config is injected via an `_INTERNAL` prologue, read by the helper.
21 expect(source).toContain('_INTERNAL = _composio_internal_json.loads(');
22 expect(source).toContain('openai/gpt-oss-120b');
23 expect(source).toContain(
24 'DEFAULT_INVOKE_LLM_MODEL = _INTERNAL.get("invoke_llm_model", "openai/gpt-oss-120b")'
25 );
26 expect(source).not.toContain('__COMPOSIO_INVOKE_LLM_MODEL__');
27 expect(source).not.toContain('upload_local_file');
28 expect(source).not.toContain('smart_file_extract');
29 expect(source).not.toContain('get_mount_file_url');
30 expect(source).not.toContain('x-session-access-key');
31 expect(source).not.toContain('COMPOSIO_WORKBENCH_ACCESS_KEY');
32 expect(source).not.toContain('runComposioTool');
33 expect(source).not.toContain('export async');
34 });
35
36 it('injects a custom invoke LLM model into the _INTERNAL prologue', () => {
37 const source = experimental_createPythonWorkbenchHelperSource({
38 invokeLlmModel: 'custom/model-x',
39 });
40
41 expect(source).toContain('_INTERNAL = _composio_internal_json.loads(');
42 expect(source).toContain('custom/model-x');
43 // The injected JSON is a valid Python double-quoted string literal.
44 expect(source).toContain('loads("{\\"invoke_llm_model\\":\\"custom/model-x\\"}")');
45 // The prologue must precede the helper body so `_INTERNAL` is defined first.
46 expect(source.indexOf('_INTERNAL = _composio_internal_json.loads(')).toBeLessThan(
47 source.indexOf('DEFAULT_INVOKE_LLM_MODEL = _INTERNAL.get(')
48 );
49 });
50
51 it('checks HTTP status before parsing successful JSON responses', () => {
52 const source = experimental_createPythonWorkbenchHelperSource();
53
54 expect(source.indexOf('if status >= 400:')).toBeLessThan(
55 source.indexOf('response_data = _parse_json(text)')
56 );
57 });
58
59 it('coerces a non-numeric envelope status instead of raising', () => {
60 const source = experimental_createPythonWorkbenchHelperSource();
61 const directory = mkdtempSync(join(tmpdir(), 'composio-helper-'));
62 const scriptPath = join(directory, 'helper_status_test.py');
63 const testScript = `${source}
64
65 import json as _json
66
67 def _post_json(url, headers, payload, timeout=120):
68 # The session envelope can report status as a string; a bare ">= 400"
69 # comparison would raise TypeError and mask the successful result.
70 return 200, {}, _json.dumps({"data": {"ok": True}, "status": "200"})
71
72 ok_data, ok_error = proxy_execute("GET", "/user", "github")
73
74 def _post_json_error(url, headers, payload, timeout=120):
75 return 200, {}, _json.dumps({"data": {"message": "nope"}, "status": "404"})
76
77 _post_json = _post_json_error
78 err_data, err_error = proxy_execute("GET", "/user", "github")
79
80 print(_json.dumps({
81 "ok_data": ok_data,
82 "ok_error": ok_error,
83 "err_error": err_error,
84 }))
85 `;
86
87 try {
88 writeFileSync(scriptPath, testScript);
89 const output = execFileSync('python3', [scriptPath], {
90 env: {
91 ...process.env,
92 BACKEND_URL: 'https://backend.test/',
93 COMPOSIO_TOOLROUTER_SESSION_ID: 'session_123',
94 COMPOSIO_API_KEY: 'project_key',
95 },
96 encoding: 'utf8',
97 });
98 const parsed = JSON.parse(output);
99
100 // String status "200" is treated as success, not a "Failed to execute" crash.
101 expect(parsed.ok_data).toEqual({ ok: true });
102 expect(parsed.ok_error).toBe('');
103 // String status "404" is surfaced as a non-empty API error.
104 expect(parsed.err_error).toContain('API returned status 404');
105 } finally {
106 rmSync(directory, { recursive: true, force: true });
107 }
108 });
109
110 it('never re-sends a tool execution after a network failure', () => {
111 const source = experimental_createPythonWorkbenchHelperSource();
112 const directory = mkdtempSync(join(tmpdir(), 'composio-helper-'));
113 const scriptPath = join(directory, 'helper_retry_test.py');
114 const testScript = `${source}
115
116 import json as _json
117
118 _timeout_calls = []
119
120 def _post_json_timeout(url, headers, payload, timeout=120):
121 # The backend may already have run the tool when the read times out.
122 _timeout_calls.append(payload)
123 raise TimeoutError("timed out")
124
125 _post_json = _post_json_timeout
126 timeout_data, timeout_error = run_composio_tool(
127 "gmail_send_email", {"to": "a@example.com"}, {"delay_ms": 0}, False
128 )
129
130 _rate_limit_calls = []
131
132 def _post_json_rate_limited(url, headers, payload, timeout=120):
133 # A 429 is rejected before the tool runs, so it stays safe to retry.
134 _rate_limit_calls.append(payload)
135 if len(_rate_limit_calls) == 1:
136 return 429, {}, _json.dumps({"error": "rate limited"})
137 return 200, {}, _json.dumps({"data": {"ok": True}})
138
139 _post_json = _post_json_rate_limited
140 rate_limit_data, rate_limit_error = run_composio_tool(
141 "gmail_send_email", {"to": "a@example.com"}, {"delay_ms": 0}, False
142 )
143
144 print(_json.dumps({
145 "timeout_calls": len(_timeout_calls),
146 "timeout_data": timeout_data,
147 "timeout_error": timeout_error,
148 "rate_limit_calls": len(_rate_limit_calls),
149 "rate_limit_data": rate_limit_data,
150 "rate_limit_error": rate_limit_error,
151 }))
152 `;
153
154 try {
155 writeFileSync(scriptPath, testScript);
156 const output = execFileSync('python3', [scriptPath], {
157 env: {
158 ...process.env,
159 BACKEND_URL: 'https://backend.test/',
160 COMPOSIO_TOOLROUTER_SESSION_ID: 'session_123',
161 COMPOSIO_API_KEY: 'project_key',
162 },
163 encoding: 'utf8',
164 });
165 const parsed = JSON.parse(output);
166
167 expect(parsed.timeout_calls).toBe(1);
168 expect(parsed.timeout_data).toEqual({});
169 expect(parsed.timeout_error).toContain('Composio tool request failed');
170 expect(parsed.rate_limit_calls).toBe(2);
171 expect(parsed.rate_limit_data).toEqual({ data: { ok: true } });
172 expect(parsed.rate_limit_error).toBe('');
173 } finally {
174 rmSync(directory, { recursive: true, force: true });
175 }
176 });
177
178 it('reports a failed execution and an input request through the error channel', () => {
179 const source = experimental_createPythonWorkbenchHelperSource();
180 const directory = mkdtempSync(join(tmpdir(), 'composio-helper-'));
181 const scriptPath = join(directory, 'helper_result_type_test.py');
182 const testScript = `${source}
183
184 import json as _json
185
186 _INPUT_REQUIRED = {
187 "result_type": "input_required",
188 "input_requests": {
189 "approval_1": {
190 "type": "elicitation",
191 "mode": "form",
192 "message": "Allow GMAIL_SEND_EMAIL to send this email?",
193 "requested_schema": {"type": "object"},
194 }
195 },
196 "request_state": "opaque-state-token",
197 }
198
199 _answers = {
200 # A failed execution can carry a null or empty error.
201 "failed_null": {"result_type": "failed", "data": {}, "error": None, "log_id": "log"},
202 "failed_empty": {"result_type": "failed", "data": {}, "error": "", "log_id": "log"},
203 "failed_message": {"result_type": "failed", "data": {}, "error": "Boom", "log_id": "log"},
204 "completed": {"result_type": "completed", "data": {"ok": True}, "error": None, "log_id": "log"},
205 "input_required": _INPUT_REQUIRED,
206 }
207 _calls = []
208 _results = {}
209
210 for _name, _answer in _answers.items():
211 def _post_json(url, headers, payload, timeout=120, _answer=_answer):
212 _calls.append(_name)
213 return 200, {}, _json.dumps(_answer)
214
215 _data, _error = run_composio_tool("gmail_send_email", {}, {"delay_ms": 0}, False)
216 _results[_name] = {"data": _data, "error": _error}
217
218 def _post_json(url, headers, payload, timeout=120):
219 return 200, {}, _json.dumps(_INPUT_REQUIRED)
220
221 _proxy_data, _proxy_error = proxy_execute("POST", "/user/repos", "GitHub", body={"name": "repo"})
222 _results["proxy_input_required"] = {"data": _proxy_data, "error": _proxy_error}
223
224 def _post_json(url, headers, payload, timeout=120):
225 return 200, {}, _json.dumps({"result_type": "completed", "data": {"id": 1}, "status": 201})
226
227 _proxy_data, _proxy_error = proxy_execute("POST", "/user/repos", "github", body={"name": "repo"})
228 _results["proxy_completed"] = {"data": _proxy_data, "error": _proxy_error}
229
230 print(_json.dumps({"calls": _calls, "results": _results}))
231 `;
232
233 try {
234 writeFileSync(scriptPath, testScript);
235 const output = execFileSync('python3', [scriptPath], {
236 env: {
237 ...process.env,
238 BACKEND_URL: 'https://backend.test/',
239 COMPOSIO_TOOLROUTER_SESSION_ID: 'session_123',
240 COMPOSIO_API_KEY: 'project_key',
241 },
242 encoding: 'utf8',
243 });
244 const { calls, results } = JSON.parse(output);
245
246 expect(results.failed_null.error).toBe(
247 'Composio tool execution failed without an error message'
248 );
249 expect(results.failed_null.data).toMatchObject({ result_type: 'failed', log_id: 'log' });
250 expect(results.failed_empty.error).toBe(
251 'Composio tool execution failed without an error message'
252 );
253 expect(results.failed_message.error).toBe('Boom');
254 expect(results.completed).toEqual({
255 data: { result_type: 'completed', data: { ok: true }, error: null, log_id: 'log' },
256 error: '',
257 });
258
259 expect(results.input_required.error).toBe(
260 'Tool GMAIL_SEND_EMAIL requires user input before it can run (1 input request) and was not executed'
261 );
262 expect(results.input_required.data).toEqual({
263 result_type: 'input_required',
264 input_requests: {
265 approval_1: {
266 type: 'elicitation',
267 mode: 'form',
268 message: 'Allow GMAIL_SEND_EMAIL to send this email?',
269 requested_schema: { type: 'object' },
270 },
271 },
272 });
273 expect(results.proxy_input_required).toEqual({
274 data: null,
275 error:
276 'POST proxy call for toolkit github requires user input before it can run (1 input request) and was not executed',
277 });
278 expect(results.proxy_completed).toEqual({ data: { id: 1 }, error: '' });
279
280 // The opaque continuation state reaches neither the data nor the message.
281 expect(output).not.toContain('opaque-state-token');
282 // Nothing is re-sent: a failure or an input request is not a retry signal.
283 expect(calls).toEqual([
284 'failed_null',
285 'failed_empty',
286 'failed_message',
287 'completed',
288 'input_required',
289 ]);
290 } finally {
291 rmSync(directory, { recursive: true, force: true });
292 }
293 });
294
295 it('round-trips helper calls through the session execute endpoint shape', () => {
296 const source = experimental_createPythonWorkbenchHelperSource({
297 invokeLlmModel: 'test/model',
298 });
299 const directory = mkdtempSync(join(tmpdir(), 'composio-helper-'));
300 const scriptPath = join(directory, 'helper_test.py');
301 const testScript = `${source}
302
303 import json as _json
304
305 _calls = []
306
307 def _post_json(url, headers, payload, timeout=120):
308 _calls.append({"url": url, "headers": headers, "payload": payload})
309 if "toolkit_slug" in payload:
310 return 200, {}, _json.dumps({"data": {"login": "octocat"}, "status": 200})
311 if payload["tool_slug"] == "COMPOSIO_SEARCH_GROQ_CHAT":
312 _content = (chr(96) * 3) + 'json\\n{"ok": true}\\n' + (chr(96) * 3)
313 return 200, {}, _json.dumps({
314 "data": {
315 "choices": [
316 {"message": {"content": _content}}
317 ]
318 }
319 })
320 if payload["tool_slug"] == "COMPOSIO_SEARCH_EXA_ANSWER":
321 return 200, {}, _json.dumps({"data": {"answer": "answer text"}})
322 return 200, {}, _json.dumps({"data": {"ok": True}})
323
324 tool_result, tool_error = run_composio_tool(
325 "github_get_repo",
326 {"owner": "composio"},
327 {"max_retries": 0, "delay_ms": 0},
328 False,
329 account="acct_123",
330 )
331 llm_result, llm_error = invoke_llm("return JSON")
332 search_result, search_error = web_search("what is Composio?")
333 proxy_result, proxy_error = proxy_execute(
334 "GET", "/user", "github", query_params={"per_page": "1"}
335 )
336
337 print(_json.dumps({
338 "calls": _calls,
339 "tool_result": tool_result,
340 "tool_error": tool_error,
341 "llm_result": llm_result,
342 "llm_error": llm_error,
343 "search_result": search_result,
344 "search_error": search_error,
345 "proxy_result": proxy_result,
346 "proxy_error": proxy_error,
347 }))
348 `;
349
350 try {
351 writeFileSync(scriptPath, testScript);
352 const output = execFileSync('python3', [scriptPath], {
353 env: {
354 ...process.env,
355 BACKEND_URL: 'https://backend.test/',
356 COMPOSIO_TOOLROUTER_SESSION_ID: 'session_123',
357 COMPOSIO_API_KEY: 'project_key',
358 },
359 encoding: 'utf8',
360 });
361 const parsed = JSON.parse(output);
362
363 expect(parsed.tool_result).toEqual({ data: { ok: true } });
364 expect(parsed.tool_error).toBe('');
365 expect(parsed.llm_result).toBe('{"ok": true}');
366 expect(parsed.llm_error).toBe('');
367 expect(parsed.search_result).toBe('answer text');
368 expect(parsed.search_error).toBe('');
369 expect(parsed.proxy_result).toEqual({ login: 'octocat' });
370 expect(parsed.proxy_error).toBe('');
371 expect(parsed.calls).toHaveLength(4);
372 expect(parsed.calls[0]).toMatchObject({
373 url: 'https://backend.test/api/v3/tool_router/session/session_123/execute',
374 headers: {
375 'Content-Type': 'application/json',
376 'x-api-key': 'project_key',
377 },
378 payload: {
379 tool_slug: 'GITHUB_GET_REPO',
380 arguments: { owner: 'composio' },
381 account: 'acct_123',
382 },
383 });
384 expect(parsed.calls[1].payload).toMatchObject({
385 tool_slug: 'COMPOSIO_SEARCH_GROQ_CHAT',
386 arguments: {
387 model: 'test/model',
388 temperature: 0.5,
389 },
390 });
391 expect(parsed.calls[2].payload).toEqual({
392 tool_slug: 'COMPOSIO_SEARCH_EXA_ANSWER',
393 arguments: { content: 'what is Composio?' },
394 });
395 expect(parsed.calls[3]).toMatchObject({
396 url: 'https://backend.test/api/v3/tool_router/session/session_123/proxy_execute',
397 headers: { 'x-api-key': 'project_key' },
398 payload: {
399 toolkit_slug: 'github',
400 endpoint: '/user',
401 method: 'GET',
402 parameters: [{ name: 'per_page', value: '1', type: 'query' }],
403 },
404 });
405 } finally {
406 rmSync(directory, { recursive: true, force: true });
407 }
408 });
409 });