1
import fs from 'node:fs';
2
import path from 'node:path';
3
import { spawn, spawnSync } from 'node:child_process';
4
+import type { IncomingMessage, ServerResponse } from 'node:http';
5
+import { Schema } from 'effect';
6
import { describe, expect, it } from 'vitest';
7
import * as tempy from 'tempy';
8
import { withHttpServer } from 'test/__utils__/http-server';
290
);
291
});
292
});
293
+
294
+// A stand-in for the backend and the Dashboard: the backend calls `composio execute` makes before
295
+// it runs a tool get minimal answers, and the Dashboard's execute procedures answer per tool slug.
296
+const DashboardRequest = Schema.fromJsonString(
297
+ Schema.Struct({
298
+ json: Schema.Struct({
299
+ tool_slug: Schema.optional(Schema.String),
300
+ slug: Schema.optional(Schema.String),
301
+ }),
302
+ })
303
+);
304
+const decodeDashboardRequest = Schema.decodeUnknownSync(DashboardRequest);
305
+
306
+const sendJson = (res: ServerResponse, status: number, body: unknown) => {
307
+ res.writeHead(status, { 'content-type': 'application/json' });
308
+ res.end(JSON.stringify(body));
309
+};
310
+
311
+const dashboardReply = (response: Record<string, unknown>) => ({
312
+ result: { data: { json: { ok: true, response } } },
313
+});
314
+
315
+const fakeComposio = (req: IncomingMessage, res: ServerResponse) => {
316
+ const { pathname } = new URL(req.url ?? '/', 'http://127.0.0.1');
317
+ const chunks: Array<Buffer> = [];
318
+ req.on('data', (chunk: Buffer) => chunks.push(chunk));
319
+ req.on('end', () => {
320
+ if (pathname.startsWith('/api/cli/trpc/')) {
321
+ const { json } = decodeDashboardRequest(Buffer.concat(chunks).toString('utf8'));
322
+ const slug = json.tool_slug ?? json.slug;
323
+ if (slug === 'HACKERNEWS_GET_FRONTPAGE') {
324
+ return sendJson(res, 200, dashboardReply({ result_type: 'completed', data: { ok: 1 } }));
325
+ }
326
+ if (slug === 'HACKERNEWS_GET_USER' || slug === 'COMPOSIO_SEARCH_TOOLS') {
327
+ return sendJson(
328
+ res,
329
+ 200,
330
+ dashboardReply({ result_type: 'failed', data: {}, error: 'User not found' })
331
+ );
332
+ }
333
+ return sendJson(res, 503, { error: { message: 'Service unavailable' } });
334
+ }
335
+ if (pathname.endsWith('/org/consumer/project/resolve')) {
336
+ return sendJson(res, 200, {
337
+ org_id: 'org_test',
338
+ project_id: 'proj_test',
339
+ project_nano_id: 'pr_test',
340
+ project_name: 'consumer',
341
+ consumer_user_id: 'consumer-user-test',
342
+ });
343
+ }
344
+ const toolSlug = /^\/api\/v3\.1\/tools\/([A-Z_]+)$/u.exec(pathname)?.[1];
345
+ if (toolSlug !== undefined) {
346
+ return sendJson(res, 200, {
347
+ slug: toolSlug,
348
+ name: toolSlug,
349
+ description: '',
350
+ available_versions: ['20260101_00'],
351
+ input_parameters: { type: 'object', properties: {} },
352
+ output_parameters: {},
353
+ tags: [],
354
+ });
355
+ }
356
+ if (pathname.endsWith('/toolkits') || pathname.endsWith('/connected_accounts')) {
357
+ return sendJson(res, 200, { items: [], next_cursor: null, total_pages: 1, current_page: 1 });
358
+ }
359
+ return sendJson(res, 404, { error: { message: `Not found: ${pathname}` } });
360
+ });
361
+};
362
+
363
+/** Shorter than the tests' 30s timeout. */
364
+const CLI_DEADLINE_MS = 20_000;
365
+
366
+const runCliAgainst = (baseUrl: string, args: ReadonlyArray<string>) => {
367
+ const configDirectory = tempy.temporaryDirectory();
368
+ fs.writeFileSync(
369
+ path.join(configDirectory, 'user_data.json'),
370
+ JSON.stringify({
371
+ api_key: 'uak_test_api_key',
372
+ base_url: baseUrl,
373
+ web_url: baseUrl,
374
+ org_id: 'org_test',
375
+ test_user_id: null,
376
+ })
377
+ );
378
+ // A host key would replace the stored one.
379
+ const {
380
+ COMPOSIO_USER_API_KEY: _apiKey,
381
+ COMPOSIO_ENVIRONMENT: _environment,
382
+ ...host
383
+ } = process.env;
384
+ // Asynchronous: the fake server answers from this process's event loop.
385
+ return new Promise<{ status: number | null; stdout: string; stderr: string }>(
386
+ (resolve, reject) => {
387
+ const child = spawn('bun', ['run', 'src/bin.ts', ...args], {
388
+ cwd: process.cwd(),
389
+ env: {
390
+ ...host,
391
+ CI: '1',
392
+ NO_COLOR: '1',
393
+ COMPOSIO_CACHE_DIR: configDirectory,
394
+ COMPOSIO_BASE_URL: baseUrl,
395
+ COMPOSIO_WEB_URL: baseUrl,
396
+ COMPOSIO_CLI_TELEMETRY_DISABLED: '1',
397
+ },
398
+ });
399
+ let stdout = '';
400
+ let stderr = '';
401
+ child.stdout.setEncoding('utf8').on('data', (chunk: string) => (stdout += chunk));
402
+ child.stderr.setEncoding('utf8').on('data', (chunk: string) => (stderr += chunk));
403
+ // Stop a hung CLI before the test times out, so the fake server can close.
404
+ const deadline = setTimeout(() => {
405
+ child.kill('SIGKILL');
406
+ reject(new Error(`CLI did not exit within ${CLI_DEADLINE_MS}ms\nstderr:\n${stderr}`));
407
+ }, CLI_DEADLINE_MS);
408
+ child.once('error', error => {
409
+ clearTimeout(deadline);
410
+ reject(error);
411
+ });
412
+ child.once('close', status => {
413
+ clearTimeout(deadline);
414
+ resolve({ status, stdout, stderr });
415
+ });
416
+ }
417
+ );
418
+};
419
+
420
+describe('reported tool execution failures', () => {
421
+ it.each([
422
+ {
423
+ name: 'a tool that reports a failure',
424
+ args: ['execute', 'HACKERNEWS_GET_USER', '-d', '{"username":"nobody"}'],
425
+ stdout: { successful: false, data: {}, error: 'User not found' },
426
+ },
427
+ {
428
+ name: 'a Dashboard outage',
429
+ args: ['execute', 'HACKERNEWS_GET_ITEM', '-d', '{"id":1}'],
430
+ stdout: { successful: false, error: expect.stringContaining('(HTTP 503)') },
431
+ },
432
+ {
433
+ name: 'a meta tool that reports a failure',
434
+ args: ['execute', 'COMPOSIO_SEARCH_TOOLS', '-d', '{"queries":[]}'],
435
+ stdout: { successful: false, data: {}, error: 'User not found' },
436
+ },
437
+ {
438
+ name: 'a parallel run with one failed call',
439
+ args: [
440
+ 'execute',
441
+ '--parallel',
442
+ 'HACKERNEWS_GET_FRONTPAGE',
443
+ '-d',
444
+ '{}',
445
+ 'HACKERNEWS_GET_USER',
446
+ '-d',
447
+ '{"username":"nobody"}',
448
+ ],
449
+ stdout: {
450
+ successful: false,
451
+ parallel: true,
452
+ results: [
453
+ { slug: 'HACKERNEWS_GET_FRONTPAGE', successful: true },
454
+ { slug: 'HACKERNEWS_GET_USER', successful: false, error: 'User not found' },
455
+ ],
456
+ },
457
+ },
458
+ ])(
459
+ '[Given] $name [Then] prints only the JSON result and exits 1',
460
+ ({ args, stdout }) =>
461
+ withHttpServer(fakeComposio, async baseUrl => {
462
+ const result = await runCliAgainst(baseUrl, args);
463
+
464
+ expect(JSON.parse(result.stdout)).toMatchObject(stdout);
465
+ expect(result.stderr).toBe('');
466
+ expect(result.status).toBe(1);
467
+ }),
468
+ 30_000
469
+ );
470
+
471
+ it.each([
472
+ {
473
+ name: 'a successful call',
474
+ args: ['execute', 'HACKERNEWS_GET_FRONTPAGE', '-d', '{}'],
475
+ },
476
+ {
477
+ name: 'a parallel run where every call succeeds',
478
+ args: [
479
+ 'execute',
480
+ '--parallel',
481
+ 'HACKERNEWS_GET_FRONTPAGE',
482
+ '-d',
483
+ '{}',
484
+ 'HACKERNEWS_GET_FRONTPAGE',
485
+ '-d',
486
+ '{}',
487
+ ],
488
+ },
489
+ ])(
490
+ '[Given] $name [Then] exits 0',
491
+ ({ args }) =>
492
+ withHttpServer(fakeComposio, async baseUrl => {
493
+ const result = await runCliAgainst(baseUrl, args);
494
+
495
+ expect(JSON.parse(result.stdout)).toMatchObject({ successful: true });
496
+ expect(result.stderr).toBe('');
497
+ expect(result.status).toBe(0);
498
+ }),
499
+ 30_000
500
+ );
501
+});