1
+import process from 'node:process';
2
+import path from 'node:path';
3
+import { $ } from 'bun';
4
+import {
5
+ Config,
6
+ ConfigProvider,
7
+ Console,
8
+ Data,
9
+ DateTime,
10
+ Effect,
11
+ Layer,
12
+ Logger,
13
+ Option,
14
+ Schema,
15
+} from 'effect';
16
+import { FileSystem } from '@effect/platform';
17
+import { BunContext, BunFileSystem, BunRuntime } from '@effect/platform-bun';
18
+import { TOOLKIT_SLUG_PATTERN } from 'src/models/toolkits';
19
+import { teardown } from './_teardown';
20
+
21
+/**
22
+ * Usage: `pnpm --filter @composio/cli generate:toolkit-slugs`.
23
+ *
24
+ * Refreshes `src/generated/toolkit-slugs.ts` — the toolkit slugs the CLI knows
25
+ * about without asking the API. Resolving a tool slug to its toolkit needs the
26
+ * catalog (`GOOGLE_ANALYTICS_RUN_REPORT` belongs to `google_analytics`, not
27
+ * `google`), and downloading ~800 KB of toolkit metadata to learn ~11 KB of
28
+ * slugs is most of the wall time of a `composio execute`.
29
+ *
30
+ * Authentication, in order of preference:
31
+ * - `COMPOSIO_USER_API_KEY` (+ `COMPOSIO_ORG_ID`) — a `uak_` key, what a
32
+ * logged-in developer has locally.
33
+ * - `COMPOSIO_API_KEY` — a project-scoped key, what CI has.
34
+ *
35
+ * `COMPOSIO_BASE_URL` overrides the API host; it defaults to production
36
+ * because the baked list ships to users of the production catalog.
37
+ */
38
+
39
+const PRODUCTION_BASE_URL = 'https://backend.composio.dev';
40
+const PAGE_SIZE = 1000;
41
+
42
+/** Slugs the catalog cannot plausibly be missing. */
43
+const SENTINEL_SLUGS = ['github', 'gmail', 'google_analytics'] as const;
44
+
45
+/**
46
+ * A catalog this small means we are looking at a partial or wrong response —
47
+ * an empty page, a filtered org view, an error body that happened to decode.
48
+ * Today's production catalog holds ~1070 toolkits and the backend never
49
+ * removes one, so anything below this is a bad fetch, not a shrunken catalog.
50
+ */
51
+const MIN_EXPECTED_SLUGS = 1000;
52
+
53
+const ToolkitsPage = Schema.Struct({
54
+ items: Schema.Array(
55
+ Schema.Struct({
56
+ slug: Schema.String,
57
+ // `native` toolkits are the shared catalog. Anything else (an org-scoped
58
+ // custom toolkit, say) is not knowledge we can bake into a released
59
+ // binary for everyone.
60
+ type: Schema.optional(Schema.String),
61
+ })
62
+ ),
63
+ next_cursor: Schema.NullishOr(Schema.String),
64
+});
65
+
66
+class ToolkitFetchError extends Data.TaggedError('ToolkitFetchError')<{
67
+ readonly message: string;
68
+}> {}
69
+
70
+const authHeaders = Effect.gen(function* () {
71
+ const userApiKey = yield* Config.option(Config.string('COMPOSIO_USER_API_KEY'));
72
+ const orgId = yield* Config.option(Config.string('COMPOSIO_ORG_ID'));
73
+ const apiKey = yield* Config.option(Config.string('COMPOSIO_API_KEY'));
74
+
75
+ if (Option.isSome(userApiKey)) {
76
+ return {
77
+ 'x-user-api-key': userApiKey.value,
78
+ ...(Option.isSome(orgId) ? { 'x-org-id': orgId.value } : {}),
79
+ };
80
+ }
81
+
82
+ if (Option.isSome(apiKey)) {
83
+ return { 'x-api-key': apiKey.value };
84
+ }
85
+
86
+ return yield* new ToolkitFetchError({
87
+ message: 'Set COMPOSIO_API_KEY, or COMPOSIO_USER_API_KEY (+ COMPOSIO_ORG_ID), and retry.',
88
+ });
89
+});
90
+
91
+const fetchPage = (params: { baseUrl: string; headers: Record<string, string>; cursor?: string }) =>
92
+ Effect.gen(function* () {
93
+ const url = new URL('/api/v3/toolkits', params.baseUrl);
94
+ url.searchParams.set('limit', String(PAGE_SIZE));
95
+ if (params.cursor) {
96
+ url.searchParams.set('cursor', params.cursor);
97
+ }
98
+
99
+ const response = yield* Effect.tryPromise({
100
+ try: () => fetch(url, { headers: params.headers }),
101
+ catch: cause => new ToolkitFetchError({ message: `GET ${url.pathname} failed: ${cause}` }),
102
+ });
103
+
104
+ if (!response.ok) {
105
+ const body = yield* Effect.promise(() => response.text());
106
+ return yield* new ToolkitFetchError({
107
+ message: `GET ${url.pathname} returned ${response.status}: ${body.slice(0, 400)}`,
108
+ });
109
+ }
110
+
111
+ const payload = yield* Effect.tryPromise({
112
+ try: () => response.json(),
113
+ catch: cause => new ToolkitFetchError({ message: `Response was not JSON: ${cause}` }),
114
+ });
115
+
116
+ return yield* Schema.decodeUnknown(ToolkitsPage)(payload).pipe(
117
+ Effect.mapError(cause => new ToolkitFetchError({ message: `Unexpected response: ${cause}` }))
118
+ );
119
+ });
120
+
121
+/** Walks the cursor to the end of the catalog. */
122
+const fetchAllSlugs = (params: { baseUrl: string; headers: Record<string, string> }) =>
123
+ Effect.gen(function* () {
124
+ const slugs: string[] = [];
125
+ let cursor: string | undefined = undefined;
126
+ let page = 0;
127
+
128
+ do {
129
+ const result = yield* fetchPage({ ...params, cursor });
130
+ page += 1;
131
+
132
+ for (const toolkit of result.items) {
133
+ if (toolkit.type === undefined || toolkit.type === 'native') {
134
+ slugs.push(toolkit.slug.toLowerCase());
135
+ }
136
+ }
137
+
138
+ yield* Effect.logInfo(`Page ${page}: ${result.items.length} toolkits (${slugs.length} kept)`);
139
+ cursor = result.next_cursor ?? undefined;
140
+ } while (cursor);
141
+
142
+ return [...new Set(slugs)].sort();
143
+ });
144
+
145
+/**
146
+ * Refuses to overwrite a good list with a bad fetch. Every failure mode we can
147
+ * cheaply detect — truncated pagination, an org-scoped view, a decoded error
148
+ * body — shows up as too few slugs, malformed slugs, or missing staples.
149
+ */
150
+const checkSanity = (slugs: ReadonlyArray<string>) =>
151
+ Effect.gen(function* () {
152
+ if (slugs.length < MIN_EXPECTED_SLUGS) {
153
+ return yield* new ToolkitFetchError({
154
+ message: `Refusing to write ${slugs.length} slugs; expected at least ${MIN_EXPECTED_SLUGS}.`,
155
+ });
156
+ }
157
+
158
+ const malformed = slugs.filter(slug => !TOOLKIT_SLUG_PATTERN.test(slug));
159
+ if (malformed.length > 0) {
160
+ return yield* new ToolkitFetchError({
161
+ message: `Refusing to write malformed slugs: ${malformed.slice(0, 10).join(', ')}`,
162
+ });
163
+ }
164
+
165
+ const missing = SENTINEL_SLUGS.filter(sentinel => !slugs.includes(sentinel));
166
+ if (missing.length > 0) {
167
+ return yield* new ToolkitFetchError({
168
+ message: `Refusing to write a catalog without ${missing.join(', ')}.`,
169
+ });
170
+ }
171
+ });
172
+
173
+const renderModule = (params: { slugs: ReadonlyArray<string>; refreshedAt: string }) =>
174
+ `// Generated by scripts/generate-toolkit-slugs.ts — do not edit by hand.
175
+// Refresh: pnpm --filter @composio/cli generate:toolkit-slugs
176
+
177
+/**
178
+ * Toolkit slugs known at build time, sorted. Resolving a tool slug to its
179
+ * toolkit is a longest-prefix match against the known catalog, and this list
180
+ * answers that without a network call. Toolkits released after this snapshot
181
+ * are picked up at runtime; the backend never removes a toolkit, so an entry
182
+ * here never goes stale in the other direction.
183
+ */
184
+export const BAKED_TOOLKIT_SLUGS: ReadonlyArray<string> = [
185
+${params.slugs.map(slug => ` '${slug}',`).join('\n')}
186
+];
187
+
188
+/** When the list above was generated. */
189
+export const BAKED_TOOLKIT_SLUGS_REFRESHED_AT = '${params.refreshedAt}';
190
+`;
191
+
192
+export function generateToolkitSlugs() {
193
+ return Effect.gen(function* () {
194
+ const fs = yield* FileSystem.FileSystem;
195
+ const baseUrl = yield* Config.string('COMPOSIO_BASE_URL').pipe(
196
+ Config.withDefault(PRODUCTION_BASE_URL)
197
+ );
198
+ const headers = yield* authHeaders;
199
+
200
+ yield* Effect.logInfo(`Fetching the toolkit catalog from ${baseUrl}`);
201
+ const slugs = yield* fetchAllSlugs({ baseUrl, headers });
202
+ yield* checkSanity(slugs);
203
+
204
+ const refreshedAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));
205
+ const outputPath = path.join(process.cwd(), 'src', 'generated', 'toolkit-slugs.ts');
206
+
207
+ yield* fs.makeDirectory(path.dirname(outputPath), { recursive: true });
208
+ yield* fs.writeFileString(outputPath, renderModule({ slugs, refreshedAt }));
209
+ yield* Effect.promise(() => $`pnpm exec prettier --write ${outputPath}`.quiet());
210
+
211
+ yield* Console.log(`Wrote ${slugs.length} toolkit slugs to ${outputPath}`);
212
+ });
213
+}
214
+
215
+if (require.main === module) {
216
+ generateToolkitSlugs().pipe(
217
+ Effect.provide(Logger.pretty),
218
+ Effect.provide(BunContext.layer),
219
+ Effect.provide(BunFileSystem.layer),
220
+ Effect.provide(Layer.setConfigProvider(ConfigProvider.fromEnv())),
221
+ Effect.scoped,
222
+ BunRuntime.runMain({ teardown })
223
+ );
224
+}