1
import { describe, expect, it } from '@effect/vitest';
2
import * as BunFileSystem from '@effect/platform-bun/BunFileSystem';
3
-import { ConfigProvider, DateTime, Effect, FileSystem, Layer, Schedule } from 'effect';
3
+import { ConfigProvider, DateTime, Deferred, Effect, FileSystem, Layer, Schedule } from 'effect';
4
import * as tempy from 'tempy';
5
import { toolkitFromToolSlug } from 'src/effects/toolkit-from-tool-slug';
6
import type { Toolkits } from 'src/models/toolkits';
7
-import { ComposioToolkitsRepository, HttpServerError } from 'src/services/composio-clients';
7
+import {
8
+ ComposioToolkitsRepository,
9
+ HttpServerError,
10
+ type ToolkitProjectScope,
11
+} from 'src/services/composio-clients';
12
import { KNOWN_TOOLKIT_SLUGS_FILE } from 'src/services/known-toolkit-slugs';
13
import { ToolkitSlugCatalog } from 'src/services/toolkit-slug-catalog';
14
import { makeToolkitFixture } from 'test/__utils__/models/toolkits';
15
import {
16
countingToolkitsRepository,
17
+ type GetProjectToolkitsError,
18
type GetToolkitsError,
19
} from 'test/__utils__/services/toolkits-repository-stub';
20
25
*/
26
const UNRELEASED_TOOLKIT = 'acme_analytics';
27
28
+/**
29
+ * A project-scoped custom toolkit. Custom toolkits never ship in the baked
30
+ * catalog, and their slugs share the `custom` prefix with nothing baked.
31
+ */
32
+const CUSTOM_TOOLKIT = 'custom_grain';
33
+
34
const failingFetch = () =>
35
Effect.fail(new HttpServerError({ cause: 'catalog unavailable', status: 503 }));
36
45
refreshedAt: DateTime.formatIso(DateTime.subtract(DateTime.nowUnsafe(), { days: daysAgo })),
46
});
47
48
+interface ResolverOptions {
49
+ readonly getToolkits?: () => Effect.Effect<Toolkits, GetToolkitsError>;
50
+ readonly getProjectToolkits?: (
51
+ scope?: ToolkitProjectScope
52
+ ) => Effect.Effect<Toolkits, GetProjectToolkitsError>;
53
+ readonly seedLearnedFile?: string;
54
+}
55
+
56
interface ResolverContext {
57
readonly calls: () => number;
58
+ readonly projectCalls: () => number;
59
readonly waitForLearnedFile: (
60
predicate: (content: string) => boolean
61
) => Effect.Effect<string, unknown, FileSystem.FileSystem>;
67
* is written verbatim — including deliberately corrupt content.
68
*/
69
const withResolver = <A>(
50
- options: {
51
- readonly getToolkits?: () => Effect.Effect<Toolkits, GetToolkitsError>;
52
- readonly seedLearnedFile?: string;
53
- },
70
+ options: ResolverOptions,
71
program: (
72
context: ResolverContext
73
) => Effect.Effect<
83
84
const runInCacheDir = <A>(
85
cacheDir: string,
69
- options: {
70
- readonly getToolkits?: () => Effect.Effect<Toolkits, GetToolkitsError>;
71
- readonly seedLearnedFile?: string;
72
- },
86
+ options: ResolverOptions,
87
program: (
88
context: ResolverContext
89
) => Effect.Effect<
101
}
102
103
const repository = countingToolkitsRepository(
90
- options.getToolkits ?? (() => Effect.succeed([] as Toolkits))
104
+ options.getToolkits ?? (() => Effect.succeed([] as Toolkits)),
105
+ options.getProjectToolkits
106
);
107
108
const readLearnedFile = fs.readFileString(learnedFile);
109
110
return yield* program({
111
calls: repository.calls,
112
+ projectCalls: repository.projectCalls,
113
// The resolver records what it learns in the background, so a test that
114
// asserts on the file has to wait for it rather than read once.
115
waitForLearnedFile: predicate =>
145
);
146
147
it.effect('gives meta tools no toolkit, even when one shadows their slug', () =>
132
- withResolver({ seedLearnedFile: learnedFileContent([]) }, ({ calls }) =>
148
+ withResolver({ seedLearnedFile: learnedFileContent([]) }, ({ calls, projectCalls }) =>
149
Effect.gen(function* () {
150
// `composio_search` is a real, linkable toolkit whose slug is a prefix
151
// of this session meta tool. Attributing the meta tool to it would
153
expect(yield* toolkitFromToolSlug('COMPOSIO_SEARCH_TOOLS')).toBeUndefined();
154
expect(yield* toolkitFromToolSlug('COMPOSIO_MANAGE_CONNECTIONS')).toBeUndefined();
155
expect(calls()).toBe(0);
156
+ expect(projectCalls()).toBe(0);
157
})
158
)
159
);
218
)
219
);
220
204
- it.live('records learned slugs at most once per run', () =>
221
+ it.live('merges later recordings in a run instead of dropping them', () =>
222
withResolver({ seedLearnedFile: learnedFileContent([]) }, ({ waitForLearnedFile }) =>
223
Effect.gen(function* () {
224
const catalog = yield* ToolkitSlugCatalog;
208
- // Every miss in a run merges the same memoized fetch, so a second
209
- // recording could only rewrite the same file. Passing a different list
210
- // here makes the gate observable: its slugs must never land.
225
+ // Misses in one run can see different catalogs — an unscoped startup
226
+ // lookup, then the command's project — so a later recording that
227
+ // brings new slugs must land alongside the first one.
228
yield* catalog.remember([UNRELEASED_TOOLKIT]);
212
- yield* catalog.remember(['slug_from_a_second_recording']);
229
+ yield* catalog.remember([UNRELEASED_TOOLKIT, CUSTOM_TOOLKIT]);
230
214
- const learned = yield* waitForLearnedFile(content => content.includes(UNRELEASED_TOOLKIT));
215
- expect(learned).not.toContain('slug_from_a_second_recording');
231
+ const learned = yield* waitForLearnedFile(content => content.includes(CUSTOM_TOOLKIT));
232
+ expect(learned).toContain(UNRELEASED_TOOLKIT);
233
})
234
)
235
);
273
const learned = yield* waitForLearnedFile(content =>
274
content.includes(UNRELEASED_TOOLKIT)
275
);
259
- expect(learned).not.toContain('stale_toolkit');
276
+ expect(learned).toContain('stale_toolkit');
277
})
278
)
279
);
314
})
315
)
316
);
317
+ describe('project-managed custom toolkits', () => {
318
+ it.live('resolves a custom toolkit from the project catalog, and remembers it', () =>
319
+ withResolver(
320
+ {
321
+ getProjectToolkits: () => Effect.succeed([makeToolkitFixture(CUSTOM_TOOLKIT)]),
322
+ seedLearnedFile: learnedFileContent([]),
323
+ },
324
+ ({ calls, projectCalls, waitForLearnedFile }) =>
325
+ Effect.gen(function* () {
326
+ expect(yield* toolkitFromToolSlug('CUSTOM_GRAIN_SEARCH_PERSONS')).toBe(CUSTOM_TOOLKIT);
327
+ expect(calls()).toBe(1);
328
+ expect(projectCalls()).toBe(1);
329
+
330
+ yield* waitForLearnedFile(content => content.includes(CUSTOM_TOOLKIT));
331
+ })
332
+ )
333
+ );
334
+
335
+ it.live('uses a remembered custom toolkit when the project catalog is unreachable', () =>
336
+ withResolver(
337
+ {
338
+ getToolkits: failingFetch,
339
+ getProjectToolkits: failingFetch,
340
+ seedLearnedFile: learnedFileContent([CUSTOM_TOOLKIT]),
341
+ },
342
+ ({ calls, projectCalls }) =>
343
+ Effect.gen(function* () {
344
+ expect(yield* toolkitFromToolSlug('CUSTOM_GRAIN_SEARCH_PERSONS')).toBe(CUSTOM_TOOLKIT);
345
+ expect(calls()).toBe(1);
346
+ expect(projectCalls()).toBe(1);
347
+ })
348
+ )
349
+ );
350
+
351
+ for (const learned of ['custom_grain', 'custom_grain_search_persons']) {
352
+ it.live(`uses the active project's prefix instead of learned ${learned}`, () =>
353
+ withResolver(
354
+ {
355
+ seedLearnedFile: learnedFileContent([learned]),
356
+ getProjectToolkits: scope =>
357
+ Effect.succeed(
358
+ scope?.projectId === 'proj_b' ? [makeToolkitFixture('custom_grain_search')] : []
359
+ ),
360
+ },
361
+ () =>
362
+ Effect.gen(function* () {
363
+ expect(
364
+ yield* toolkitFromToolSlug('CUSTOM_GRAIN_SEARCH_PERSONS_LIST', {
365
+ orgId: 'org_1',
366
+ projectId: 'proj_b',
367
+ })
368
+ ).toBe('custom_grain_search');
369
+ expect(
370
+ yield* toolkitFromToolSlug('CUSTOM_GRAIN_SEARCH_PERSONS_LIST', {
371
+ orgId: 'org_1',
372
+ projectId: 'proj_empty',
373
+ })
374
+ ).toBe('custom');
375
+ })
376
+ )
377
+ );
378
+ }
379
+
380
+ it.live('keeps a scoped discovery when an older background fetch completes later', () =>
381
+ Effect.gen(function* () {
382
+ const releaseRefresh = yield* Deferred.make<void>();
383
+ yield* withResolver(
384
+ {
385
+ seedLearnedFile: learnedFileContent([], 8),
386
+ getToolkits: () =>
387
+ Deferred.await(releaseRefresh).pipe(
388
+ Effect.as([makeToolkitFixture(UNRELEASED_TOOLKIT)])
389
+ ),
390
+ },
391
+ ({ waitForLearnedFile }) =>
392
+ Effect.gen(function* () {
393
+ expect(yield* toolkitFromToolSlug('GMAIL_SEND_EMAIL')).toBe('gmail');
394
+ const catalog = yield* ToolkitSlugCatalog;
395
+ yield* catalog.remember([CUSTOM_TOOLKIT]);
396
+ yield* waitForLearnedFile(content => content.includes(CUSTOM_TOOLKIT));
397
+ yield* Deferred.succeed(releaseRefresh, undefined);
398
+ const learned = yield* waitForLearnedFile(content =>
399
+ content.includes(UNRELEASED_TOOLKIT)
400
+ );
401
+ expect(learned).toContain(CUSTOM_TOOLKIT);
402
+ })
403
+ );
404
+ })
405
+ );
406
+
407
+ it.live('retries a stale native catalog after remembering a partial project result', () =>
408
+ Effect.gen(function* () {
409
+ const cacheDir = tempy.temporaryDirectory();
410
+ const seed = learnedFileContent([], 8);
411
+ yield* runInCacheDir(
412
+ cacheDir,
413
+ {
414
+ seedLearnedFile: seed,
415
+ getToolkits: failingFetch,
416
+ getProjectToolkits: () => Effect.succeed([makeToolkitFixture(CUSTOM_TOOLKIT)]),
417
+ },
418
+ ({ waitForLearnedFile }) =>
419
+ Effect.gen(function* () {
420
+ expect(yield* toolkitFromToolSlug('CUSTOM_GRAIN_SEARCH_PERSONS')).toBe(
421
+ CUSTOM_TOOLKIT
422
+ );
423
+ const learned = yield* waitForLearnedFile(content =>
424
+ content.includes(CUSTOM_TOOLKIT)
425
+ );
426
+ expect(JSON.parse(learned).refreshedAt).toBe(JSON.parse(seed).refreshedAt);
427
+ })
428
+ );
429
+ yield* runInCacheDir(
430
+ cacheDir,
431
+ {
432
+ getToolkits: () => Effect.succeed([makeToolkitFixture(UNRELEASED_TOOLKIT)]),
433
+ },
434
+ ({ waitForLearnedFile }) =>
435
+ Effect.gen(function* () {
436
+ expect(yield* toolkitFromToolSlug('GMAIL_SEND_EMAIL')).toBe('gmail');
437
+ const learned = yield* waitForLearnedFile(content =>
438
+ content.includes(UNRELEASED_TOOLKIT)
439
+ );
440
+ expect(learned).toContain(CUSTOM_TOOLKIT);
441
+ expect(JSON.parse(learned).refreshedAt).not.toBe(JSON.parse(seed).refreshedAt);
442
+ })
443
+ );
444
+ })
445
+ );
446
+
447
+ it.live('preserves learned custom slugs when an unscoped refresh cannot list them', () =>
448
+ withResolver(
449
+ {
450
+ seedLearnedFile: learnedFileContent([CUSTOM_TOOLKIT], 8),
451
+ getToolkits: () => Effect.succeed([makeToolkitFixture(UNRELEASED_TOOLKIT)]),
452
+ getProjectToolkits: failingFetch,
453
+ },
454
+ ({ waitForLearnedFile }) =>
455
+ Effect.gen(function* () {
456
+ expect(yield* toolkitFromToolSlug('GMAIL_SEND_EMAIL')).toBe('gmail');
457
+ const learned = yield* waitForLearnedFile(content =>
458
+ content.includes(UNRELEASED_TOOLKIT)
459
+ );
460
+ expect(learned).toContain(CUSTOM_TOOLKIT);
461
+ })
462
+ )
463
+ );
464
+
465
+ it.live('keeps resolving baked `custom*` toolkits locally', () =>
466
+ withResolver(
467
+ { seedLearnedFile: learnedFileContent([CUSTOM_TOOLKIT]) },
468
+ ({ calls, projectCalls }) =>
469
+ Effect.gen(function* () {
470
+ expect(yield* toolkitFromToolSlug('CUSTOMERIO_SEND_EVENT')).toBe('customerio');
471
+ expect(calls()).toBe(0);
472
+ expect(projectCalls()).toBe(0);
473
+ })
474
+ )
475
+ );
476
+
477
+ it.live('prefers the longest custom toolkit prefix', () =>
478
+ withResolver(
479
+ {
480
+ getProjectToolkits: () =>
481
+ Effect.succeed([
482
+ makeToolkitFixture(CUSTOM_TOOLKIT),
483
+ makeToolkitFixture(`${CUSTOM_TOOLKIT}_search`),
484
+ ]),
485
+ seedLearnedFile: learnedFileContent([]),
486
+ },
487
+ () =>
488
+ Effect.gen(function* () {
489
+ expect(yield* toolkitFromToolSlug('CUSTOM_GRAIN_SEARCH_PERSONS')).toBe(
490
+ `${CUSTOM_TOOLKIT}_search`
491
+ );
492
+ })
493
+ )
494
+ );
495
+
496
+ it.live('still resolves native toolkits when the project catalog is unreachable', () =>
497
+ withResolver(
498
+ {
499
+ getToolkits: () => Effect.succeed([makeToolkitFixture(UNRELEASED_TOOLKIT)]),
500
+ getProjectToolkits: failingFetch,
501
+ seedLearnedFile: learnedFileContent([]),
502
+ },
503
+ () =>
504
+ Effect.gen(function* () {
505
+ expect(yield* toolkitFromToolSlug('ACME_ANALYTICS_RUN_REPORT')).toBe(
506
+ UNRELEASED_TOOLKIT
507
+ );
508
+ expect(yield* toolkitFromToolSlug('CUSTOM_GRAIN_SEARCH_PERSONS')).toBe('custom');
509
+ })
510
+ )
511
+ );
512
+
513
+ it.live('resolves custom toolkits when only the native catalog is unreachable', () =>
514
+ withResolver(
515
+ {
516
+ getToolkits: failingFetch,
517
+ getProjectToolkits: () => Effect.succeed([makeToolkitFixture(CUSTOM_TOOLKIT)]),
518
+ seedLearnedFile: learnedFileContent([]),
519
+ },
520
+ () =>
521
+ Effect.gen(function* () {
522
+ expect(yield* toolkitFromToolSlug('CUSTOM_GRAIN_SEARCH_PERSONS')).toBe(CUSTOM_TOOLKIT);
523
+ })
524
+ )
525
+ );
526
+
527
+ it.live('guesses, and records nothing, when both catalogs are unreachable', () => {
528
+ const seed = learnedFileContent([]);
529
+ return withResolver(
530
+ {
531
+ getToolkits: failingFetch,
532
+ getProjectToolkits: failingFetch,
533
+ seedLearnedFile: seed,
534
+ },
535
+ ({ waitForLearnedFile }) =>
536
+ Effect.gen(function* () {
537
+ expect(yield* toolkitFromToolSlug('CUSTOM_GRAIN_SEARCH_PERSONS')).toBe('custom');
538
+ yield* Effect.sleep('50 millis');
539
+ expect(yield* waitForLearnedFile(() => true)).toBe(seed);
540
+ })
541
+ );
542
+ });
543
+
544
+ it.live('learns a custom toolkit found by a scoped lookup after an unscoped one', () =>
545
+ withResolver(
546
+ {
547
+ getProjectToolkits: scope =>
548
+ Effect.succeed(scope ? [makeToolkitFixture(CUSTOM_TOOLKIT)] : []),
549
+ seedLearnedFile: learnedFileContent([]),
550
+ },
551
+ ({ waitForLearnedFile }) =>
552
+ Effect.gen(function* () {
553
+ // Startup telemetry resolves before the command knows its project.
554
+ expect(yield* toolkitFromToolSlug('CUSTOM_GRAIN_SEARCH_PERSONS')).toBe('custom');
555
+ expect(
556
+ yield* toolkitFromToolSlug('CUSTOM_GRAIN_SEARCH_PERSONS', {
557
+ orgId: 'org_1',
558
+ projectId: 'proj_1',
559
+ })
560
+ ).toBe(CUSTOM_TOOLKIT);
561
+
562
+ yield* waitForLearnedFile(content => content.includes(CUSTOM_TOOLKIT));
563
+ })
564
+ )
565
+ );
566
+
567
+ it.live('includes project toolkits in the background refresh', () =>
568
+ withResolver(
569
+ {
570
+ getProjectToolkits: () => Effect.succeed([makeToolkitFixture(CUSTOM_TOOLKIT)]),
571
+ seedLearnedFile: learnedFileContent(['stale_toolkit'], 8),
572
+ },
573
+ ({ waitForLearnedFile }) =>
574
+ Effect.gen(function* () {
575
+ expect(yield* toolkitFromToolSlug('GMAIL_SEND_EMAIL')).toBe('gmail');
576
+ yield* waitForLearnedFile(content => content.includes(CUSTOM_TOOLKIT));
577
+ })
578
+ )
579
+ );
580
+ });
581
});