1
-import { describe, expect, layer } from '@effect/vitest';
2
-import { Effect } from 'effect';
1
+import { describe, expect, it } from '@effect/vitest';
2
+import { FileSystem } from '@effect/platform';
3
+import { BunFileSystem } from '@effect/platform-bun';
4
+import { ConfigProvider, DateTime, Effect, Layer, Schedule } from 'effect';
5
+import * as tempy from 'tempy';
6
import { toolkitFromToolSlug } from 'src/effects/toolkit-from-tool-slug';
7
import type { Toolkits } from 'src/models/toolkits';
5
-import { TestLive } from 'test/__utils__';
8
+import { ComposioToolkitsRepository, HttpServerError } from 'src/services/composio-clients';
9
+import { KNOWN_TOOLKIT_SLUGS_FILE } from 'src/services/known-toolkit-slugs';
10
+import { ToolkitSlugCatalog } from 'src/services/toolkit-slug-catalog';
11
import { makeToolkitFixture } from 'test/__utils__/models/toolkits';
12
+import {
13
+ countingToolkitsRepository,
14
+ type GetToolkitsError,
15
+} from 'test/__utils__/services/toolkits-repository-stub';
16
8
-const testToolkits: Toolkits = [
9
- makeToolkitFixture('google', 'Google'),
10
- makeToolkitFixture('google_analytics', 'Google Analytics'),
11
- makeToolkitFixture('gmail', 'Gmail'),
12
- makeToolkitFixture('microsoft', 'Microsoft'),
13
- makeToolkitFixture('microsoft_teams', 'Microsoft Teams'),
14
- // A real, linkable toolkit whose slug is also the prefix of a session meta
15
- // tool (`COMPOSIO_SEARCH_TOOLS`).
16
- makeToolkitFixture('composio_search', 'Composio Search'),
17
-];
17
+/**
18
+ * A toolkit that cannot be in the baked catalog, standing in for one released
19
+ * after the binary was built. Tests of the fetch path must use slugs like this
20
+ * — anything real resolves locally and never reaches the repository.
21
+ */
22
+const UNRELEASED_TOOLKIT = 'acme_analytics';
23
19
-describe('toolkitFromToolSlug', () => {
20
- layer(TestLive({ toolkitsData: { toolkits: testToolkits } }))('with known toolkits', it => {
21
- it.effect('resolves multi-word toolkit slugs by longest known prefix', () =>
22
- Effect.gen(function* () {
23
- expect(yield* toolkitFromToolSlug('GOOGLE_ANALYTICS_RUN_REPORT')).toBe('google_analytics');
24
- expect(yield* toolkitFromToolSlug('MICROSOFT_TEAMS_SEND_MESSAGE')).toBe('microsoft_teams');
25
- })
24
+const failingFetch = () =>
25
+ Effect.fail(new HttpServerError({ cause: 'catalog unavailable', status: 503 }));
26
+
27
+/**
28
+ * A learned-slugs file recorded `daysAgo` days ago. Anything under a week old
29
+ * keeps the resolver from starting a background refresh, which is what makes
30
+ * "no catalog fetch" assertions meaningful.
31
+ */
32
+const learnedFileContent = (slugs: ReadonlyArray<string>, daysAgo = 0) =>
33
+ JSON.stringify({
34
+ slugs,
35
+ refreshedAt: DateTime.formatIso(DateTime.subtract(DateTime.unsafeNow(), { days: daysAgo })),
36
+ });
37
+
38
+interface ResolverContext {
39
+ readonly calls: () => number;
40
+ readonly waitForLearnedFile: (
41
+ predicate: (content: string) => boolean
42
+ ) => Effect.Effect<string, unknown, FileSystem.FileSystem>;
43
+}
44
+
45
+/**
46
+ * Runs `program` against a private cache directory, so the resolver reads and
47
+ * writes a learned-slugs file belonging to this test alone. `seedLearnedFile`
48
+ * is written verbatim — including deliberately corrupt content.
49
+ */
50
+const withResolver = <A>(
51
+ options: {
52
+ readonly getToolkits?: () => Effect.Effect<Toolkits, GetToolkitsError>;
53
+ readonly seedLearnedFile?: string;
54
+ },
55
+ program: (
56
+ context: ResolverContext
57
+ ) => Effect.Effect<
58
+ A,
59
+ unknown,
60
+ FileSystem.FileSystem | ComposioToolkitsRepository | ToolkitSlugCatalog
61
+ >
62
+) =>
63
+ Effect.suspend(() => {
64
+ const cacheDir = tempy.temporaryDirectory();
65
+ return runInCacheDir(cacheDir, options, program);
66
+ });
67
+
68
+const runInCacheDir = <A>(
69
+ cacheDir: string,
70
+ options: {
71
+ readonly getToolkits?: () => Effect.Effect<Toolkits, GetToolkitsError>;
72
+ readonly seedLearnedFile?: string;
73
+ },
74
+ program: (
75
+ context: ResolverContext
76
+ ) => Effect.Effect<
77
+ A,
78
+ unknown,
79
+ FileSystem.FileSystem | ComposioToolkitsRepository | ToolkitSlugCatalog
80
+ >
81
+) =>
82
+ Effect.gen(function* () {
83
+ const fs = yield* FileSystem.FileSystem;
84
+ const learnedFile = `${cacheDir}/${KNOWN_TOOLKIT_SLUGS_FILE}`;
85
+
86
+ if (options.seedLearnedFile !== undefined) {
87
+ yield* fs.writeFileString(learnedFile, options.seedLearnedFile);
88
+ }
89
+
90
+ const repository = countingToolkitsRepository(
91
+ options.getToolkits ?? (() => Effect.succeed([] as Toolkits))
92
);
93
28
- it.effect('resolves single-word toolkit slugs', () =>
29
- Effect.gen(function* () {
30
- expect(yield* toolkitFromToolSlug('GMAIL_SEND_EMAIL')).toBe('gmail');
31
- })
94
+ const readLearnedFile = fs.readFileString(learnedFile);
95
+
96
+ return yield* program({
97
+ calls: repository.calls,
98
+ // The resolver records what it learns in the background, so a test that
99
+ // asserts on the file has to wait for it rather than read once.
100
+ waitForLearnedFile: predicate =>
101
+ readLearnedFile.pipe(
102
+ Effect.filterOrFail(predicate, () => 'learned file has not caught up yet' as const),
103
+ Effect.retry(Schedule.spaced('10 millis').pipe(Schedule.intersect(Schedule.recurs(200))))
104
+ ),
105
+ }).pipe(
106
+ // Built inside this test's cache directory, and fresh per test: the
107
+ // catalog memoizes what it reads, so a shared one would leak the first
108
+ // test's learned slugs into every later one.
109
+ Effect.provide(
110
+ Layer.mergeAll(
111
+ repository.layer,
112
+ Layer.provide(ToolkitSlugCatalog.Default, repository.layer)
113
+ )
114
+ )
115
);
116
+ }).pipe(
117
+ Effect.provide(BunFileSystem.layer),
118
+ Effect.withConfigProvider(ConfigProvider.fromMap(new Map([['CACHE_DIR', cacheDir]])))
119
+ );
120
34
- it.effect('falls back to the guess for slugs outside the known list', () =>
121
+describe('toolkitFromToolSlug', () => {
122
+ it.effect('resolves a baked multi-word toolkit without touching the network', () =>
123
+ withResolver({ seedLearnedFile: learnedFileContent([]) }, ({ calls }) =>
124
Effect.gen(function* () {
36
- expect(yield* toolkitFromToolSlug('NOTION_CREATE_PAGE')).toBe('notion');
125
+ expect(yield* toolkitFromToolSlug('GOOGLE_ANALYTICS_RUN_REPORT')).toBe('google_analytics');
126
+ expect(yield* toolkitFromToolSlug('GMAIL_SEND_EMAIL')).toBe('gmail');
127
+ expect(calls()).toBe(0);
128
})
38
- );
129
+ )
130
+ );
131
40
- it.effect('gives meta tools no toolkit, even when one shadows their slug', () =>
132
+ it.effect('gives meta tools no toolkit, even when one shadows their slug', () =>
133
+ withResolver({ seedLearnedFile: learnedFileContent([]) }, ({ calls }) =>
134
Effect.gen(function* () {
42
- // Attributing this to `composio_search` would make a failed meta call
43
- // tell the user to link an app that has nothing to do with it.
135
+ // `composio_search` is a real, linkable toolkit whose slug is a prefix
136
+ // of this session meta tool. Attributing the meta tool to it would
137
+ // send users to link an app they do not need.
138
expect(yield* toolkitFromToolSlug('COMPOSIO_SEARCH_TOOLS')).toBeUndefined();
139
expect(yield* toolkitFromToolSlug('COMPOSIO_MANAGE_CONNECTIONS')).toBeUndefined();
140
+ expect(calls()).toBe(0);
141
})
47
- );
142
+ )
143
+ );
144
49
- it.effect('still resolves ordinary tools of the shadowed toolkit', () =>
145
+ it.effect('treats a bare `composio` match as resolved, not as a miss', () =>
146
+ withResolver({ seedLearnedFile: learnedFileContent([]) }, ({ calls }) =>
147
Effect.gen(function* () {
51
- expect(yield* toolkitFromToolSlug('COMPOSIO_SEARCH_SEARCH')).toBe('composio_search');
148
+ expect(yield* toolkitFromToolSlug('COMPOSIO_ENABLE_TRIGGER')).toBeUndefined();
149
+ // A miss here would cost every `COMPOSIO_*` tool a catalog fetch.
150
+ expect(calls()).toBe(0);
151
})
53
- );
54
- });
152
+ )
153
+ );
154
+
155
+ it.live('answers from the baked list on a first run, and seeds the file behind it', () =>
156
+ withResolver(
157
+ { getToolkits: () => Effect.succeed([makeToolkitFixture(UNRELEASED_TOOLKIT)]) },
158
+ ({ waitForLearnedFile }) =>
159
+ Effect.gen(function* () {
160
+ expect(yield* toolkitFromToolSlug('GOOGLE_ANALYTICS_RUN_REPORT')).toBe(
161
+ 'google_analytics'
162
+ );
163
+
164
+ const learned = yield* waitForLearnedFile(content =>
165
+ content.includes(UNRELEASED_TOOLKIT)
166
+ );
167
+ expect(learned).toContain('google_analytics');
168
+ })
169
+ )
170
+ );
171
+
172
+ it.live('resolves a learned toolkit without touching the network', () =>
173
+ withResolver(
174
+ {
175
+ seedLearnedFile: learnedFileContent([UNRELEASED_TOOLKIT]),
176
+ },
177
+ ({ calls }) =>
178
+ Effect.gen(function* () {
179
+ expect(yield* toolkitFromToolSlug('ACME_ANALYTICS_RUN_REPORT')).toBe(UNRELEASED_TOOLKIT);
180
+ expect(calls()).toBe(0);
181
+ })
182
+ )
183
+ );
184
56
- layer(TestLive())('with an empty toolkit list', it => {
57
- it.effect('falls back to the first-underscore guess', () =>
185
+ it.live('fetches the catalog for an unknown slug, and remembers what it finds', () =>
186
+ withResolver(
187
+ {
188
+ getToolkits: () => Effect.succeed([makeToolkitFixture(UNRELEASED_TOOLKIT)]),
189
+ seedLearnedFile: learnedFileContent([]),
190
+ },
191
+ ({ calls, waitForLearnedFile }) =>
192
+ Effect.gen(function* () {
193
+ expect(yield* toolkitFromToolSlug('ACME_ANALYTICS_RUN_REPORT')).toBe(UNRELEASED_TOOLKIT);
194
+ expect(calls()).toBe(1);
195
+
196
+ const learned = yield* waitForLearnedFile(content =>
197
+ content.includes(UNRELEASED_TOOLKIT)
198
+ );
199
+ // Baked slugs are recorded alongside it, so the file stands alone.
200
+ expect(learned).toContain('google_analytics');
201
+ })
202
+ )
203
+ );
204
+
205
+ it.live('records learned slugs at most once per run', () =>
206
+ withResolver({ seedLearnedFile: learnedFileContent([]) }, ({ waitForLearnedFile }) =>
207
Effect.gen(function* () {
59
- expect(yield* toolkitFromToolSlug('GOOGLE_ANALYTICS_RUN_REPORT')).toBe('google');
208
+ const catalog = yield* ToolkitSlugCatalog;
209
+ // Every miss in a run merges the same memoized fetch, so a second
210
+ // recording could only rewrite the same file. Passing a different list
211
+ // here makes the gate observable: its slugs must never land.
212
+ yield* catalog.remember([UNRELEASED_TOOLKIT]);
213
+ yield* catalog.remember(['slug_from_a_second_recording']);
214
+
215
+ const learned = yield* waitForLearnedFile(content => content.includes(UNRELEASED_TOOLKIT));
216
+ expect(learned).not.toContain('slug_from_a_second_recording');
217
})
61
- );
62
- });
218
+ )
219
+ );
220
+
221
+ it.live('falls back to the first-underscore guess when the catalog is unreachable', () =>
222
+ withResolver(
223
+ {
224
+ getToolkits: failingFetch,
225
+ seedLearnedFile: learnedFileContent([]),
226
+ },
227
+ ({ calls }) =>
228
+ Effect.gen(function* () {
229
+ expect(yield* toolkitFromToolSlug('ACME_ANALYTICS_RUN_REPORT')).toBe('acme');
230
+ expect(calls()).toBe(1);
231
+ })
232
+ )
233
+ );
234
+
235
+ it.live('tolerates a corrupted learned-slugs file', () =>
236
+ withResolver({ seedLearnedFile: '{ not json' }, ({ calls }) =>
237
+ Effect.gen(function* () {
238
+ expect(yield* toolkitFromToolSlug('GOOGLE_ANALYTICS_RUN_REPORT')).toBe('google_analytics');
239
+ expect(calls()).toBeLessThanOrEqual(1);
240
+ })
241
+ )
242
+ );
243
+
244
+ it.live('refreshes learned slugs in the background once they go stale', () =>
245
+ withResolver(
246
+ {
247
+ getToolkits: () => Effect.succeed([makeToolkitFixture(UNRELEASED_TOOLKIT)]),
248
+ seedLearnedFile: learnedFileContent(['stale_toolkit'], 8),
249
+ },
250
+ ({ waitForLearnedFile }) =>
251
+ Effect.gen(function* () {
252
+ // Answered from the baked list — the refresh is what the fetch is for.
253
+ expect(yield* toolkitFromToolSlug('GOOGLE_ANALYTICS_RUN_REPORT')).toBe(
254
+ 'google_analytics'
255
+ );
256
+
257
+ const learned = yield* waitForLearnedFile(content =>
258
+ content.includes(UNRELEASED_TOOLKIT)
259
+ );
260
+ expect(learned).not.toContain('stale_toolkit');
261
+ })
262
+ )
263
+ );
264
+
265
+ it.live('reads local knowledge once, however many slugs a run resolves', () =>
266
+ withResolver(
267
+ {
268
+ getToolkits: () => Effect.succeed([makeToolkitFixture(UNRELEASED_TOOLKIT)]),
269
+ seedLearnedFile: learnedFileContent(['stale_toolkit'], 8),
270
+ },
271
+ ({ calls, waitForLearnedFile }) =>
272
+ Effect.gen(function* () {
273
+ expect(yield* toolkitFromToolSlug('GOOGLE_ANALYTICS_RUN_REPORT')).toBe(
274
+ 'google_analytics'
275
+ );
276
+ expect(yield* toolkitFromToolSlug('GMAIL_SEND_EMAIL')).toBe('gmail');
277
+ expect(yield* toolkitFromToolSlug('GITHUB_CREATE_AN_ISSUE')).toBe('github');
278
+
279
+ yield* waitForLearnedFile(content => content.includes(UNRELEASED_TOOLKIT));
280
+ // The staleness verdict cannot change under a running process, so the
281
+ // refresh it triggers belongs to the run, not to each resolution.
282
+ expect(calls()).toBe(1);
283
+ })
284
+ )
285
+ );
286
+
287
+ it.live('does not refresh while the learned slugs are fresh', () =>
288
+ withResolver(
289
+ {
290
+ getToolkits: failingFetch,
291
+ seedLearnedFile: learnedFileContent([UNRELEASED_TOOLKIT], 6),
292
+ },
293
+ ({ calls }) =>
294
+ Effect.gen(function* () {
295
+ expect(yield* toolkitFromToolSlug('ACME_ANALYTICS_RUN_REPORT')).toBe(UNRELEASED_TOOLKIT);
296
+ yield* Effect.sleep('50 millis');
297
+ expect(calls()).toBe(0);
298
+ })
299
+ )
300
+ );
301
});