@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
mjs 346 lines 14.6 KB
Raw
1 #!/usr/bin/env node
2 // Idempotent provisioning check for the live example runs.
3 //
4 // Verifies that the dedicated (disposable) Composio project holds the auth
5 // configs and connected accounts the tier-2/3 examples need, creates whatever
6 // can be created without a human (API-key auth config), and
7 // prints the remaining one-time OAuth authorizations.
8 //
9 // node scripts/examples-provision.mjs # report (stderr) + exports (stdout)
10 // out=$(node scripts/examples-provision.mjs) && eval "$out"
11 // load COMPOSIO_EXAMPLES_* into the shell. Do NOT collapse this into
12 // eval "$(...)": eval reports the status of the text it evaluates, so a
13 // failed provisioning run would look like success.
14 // node scripts/examples-provision.mjs --initiate-missing
15 // also starts an OAuth connection request for each missing account and
16 // prints the redirect URL to authorize it (one browser visit per toolkit)
17 // node scripts/examples-provision.mjs --gc [--dry-run]
18 // DESTRUCTIVE. Deletes resources example runs leak into the project:
19 // connected accounts that never became ACTIVE, surplus serpapi demo
20 // accounts, and MCP configs named `examples-<label>-<unix-seconds>`,
21 // which is the name every example gives the configs it creates.
22 // Configs from runs predating that convention are not matched.
23 // Connected accounts are only ever deleted when they are bound to an
24 // `examples-<slug>` auth config, which only this script creates. An
25 // account the examples did not create is never touched, whatever user
26 // owns it. Only resources older than 24h are removed, so a concurrent run
27 // is safe. Preview with --dry-run.
28 //
29 // Auth config ids and connected account ids are not secrets; no credential
30 // values are ever printed. The API-key demo value stored for serpapi is a
31 // deliberately fake placeholder, not a real key.
32 //
33 // Selection flags match harness/run.mjs: [--lang ts|py] [--ids a,b]
34 // [--tiers 1,2,3] [--exclude-toolkits a,b]. Only resources required by the
35 // selected entries are provisioned; garbage collection remains project-wide
36 // within the examples-* ownership boundary above.
37
38 import {
39 BROWSER_GRANT_TOOLKITS,
40 DEMO_TOOLKIT,
41 describeExclusions,
42 emptySelectionMessage,
43 loadManifest,
44 parseSelectionOptions,
45 requiredBrowserGrantToolkits,
46 requiresDemoToolkit,
47 selectManifestEntries,
48 } from '../harness/manifest.mjs';
49 import { resolveBackendBaseUrl } from '../harness/backend-url.mjs';
50
51 let BASE_URL;
52 try {
53 BASE_URL = resolveBackendBaseUrl();
54 } catch (error) {
55 console.error(error instanceof Error ? error.message : String(error));
56 process.exit(1);
57 }
58 const API_KEY = process.env.COMPOSIO_API_KEY;
59 const USER_ID = process.env.COMPOSIO_EXAMPLES_USER_ID ?? 'examples';
60 const INITIATE_MISSING = process.argv.includes('--initiate-missing');
61 const GC = process.argv.includes('--gc');
62 const DRY_RUN = process.argv.includes('--dry-run');
63 const report = line => console.error(line);
64
65 const selection = selectManifestEntries(loadManifest(), parseSelectionOptions(process.argv));
66 for (const line of describeExclusions(selection)) report(line);
67 if (selection.entries.length === 0) {
68 report(emptySelectionMessage(selection, 'example entries'));
69 process.exit(1);
70 }
71 const selectedBrowserGrantToolkits = requiredBrowserGrantToolkits(selection.entries);
72 const needsDemoToolkit = requiresDemoToolkit(selection.entries);
73
74 // COMPOSIO_BASE_URL selects the backend and defaults to staging;
75 // resolveBackendBaseUrl refuses a structurally unusable one before the key is
76 // read or sent. Point this at a project whose resources you are willing to have
77 // this script create — and, under --gc, delete.
78 if (!API_KEY) {
79 console.error('COMPOSIO_API_KEY is required (dedicated examples-project key)');
80 process.exit(1);
81 }
82
83 // The backend's edge rate-limits bursts of requests with a 429. Listing and
84 // deleting are safe to repeat, so wait and retry those; never retry a create.
85 const RETRYABLE_METHODS = new Set(['GET', 'DELETE']);
86 const MAX_RATE_LIMIT_RETRIES = 5;
87
88 const rateLimitDelayMs = (res, attempt) => {
89 const retryAfter = Number(res.headers.get('retry-after'));
90 if (Number.isFinite(retryAfter) && retryAfter > 0) return Math.min(retryAfter, 60) * 1000;
91 return Math.min(2 ** (attempt + 1), 32) * 1000;
92 };
93
94 async function api(method, path, body) {
95 for (let attempt = 0; ; attempt++) {
96 const res = await request(method, path, body);
97 if (res.status === 429 && RETRYABLE_METHODS.has(method) && attempt < MAX_RATE_LIMIT_RETRIES) {
98 await res.body?.cancel();
99 const delayMs = rateLimitDelayMs(res, attempt);
100 report(`rate limited on ${method} ${path}; retrying in ${delayMs / 1000}s`);
101 await new Promise(resolve => setTimeout(resolve, delayMs));
102 continue;
103 }
104 const text = await res.text();
105 if (!res.ok) {
106 throw new Error(`${method} ${path} -> ${res.status}: ${text.slice(0, 300)}`);
107 }
108 return text ? JSON.parse(text) : undefined;
109 }
110 }
111
112 function request(method, path, body) {
113 return fetch(`${BASE_URL}${path}`, {
114 method,
115 headers: { 'x-api-key': API_KEY, 'content-type': 'application/json' },
116 body: body === undefined ? undefined : JSON.stringify(body),
117 // Without this a hung backend hangs the whole run instead of failing it.
118 signal: AbortSignal.timeout(30_000),
119 // The host allowlist above only checks BASE_URL. fetch follows redirects by
120 // default and forwards x-api-key across origins (only Authorization is
121 // stripped), so a 3xx would hand the key to whatever host it names.
122 redirect: 'error',
123 });
124 }
125
126 // Two pagination shapes are in play. Most v3.1 collections (auth_configs,
127 // connected_accounts) return `next_cursor`; /mcp/servers never does and reports
128 // current_page/total_pages instead. Follow whichever the response actually
129 // offers, otherwise the MCP cleanup silently stops after one page.
130 const MAX_PAGES = 50;
131
132 async function listAll(path, key = 'items') {
133 const out = [];
134 let cursor;
135 let pageNo = 1;
136 for (let page = 0; page < MAX_PAGES; page++) {
137 const sep = path.includes('?') ? '&' : '?';
138 let query = 'limit=100';
139 if (cursor) query += `&cursor=${encodeURIComponent(cursor)}`;
140 else if (pageNo > 1) query += `&page_no=${pageNo}`;
141
142 const data = await api('GET', `${path}${sep}${query}`);
143 out.push(...(data?.[key] ?? []));
144
145 if (data?.next_cursor) {
146 if (data.next_cursor === cursor) {
147 report(`warning: ${path} repeated cursor ${cursor}; stopping to avoid a loop`);
148 return out;
149 }
150 cursor = data.next_cursor;
151 continue;
152 }
153
154 const totalPages = Number(data?.total_pages);
155 const currentPage = Number(data?.current_page ?? pageNo);
156 if (Number.isFinite(totalPages) && Number.isFinite(currentPage) && currentPage < totalPages) {
157 // Clear the cursor before falling back to page numbers. Leaving it set
158 // would resend the same cursor every iteration and repeat one page until
159 // the page cap.
160 cursor = undefined;
161 pageNo = currentPage + 1;
162 continue;
163 }
164 return out;
165 }
166 report(`warning: stopped paginating ${path} after ${MAX_PAGES} pages; results may be incomplete`);
167 return out;
168 }
169
170 const [authConfigs, accounts] = await Promise.all([
171 listAll('/api/v3.1/auth_configs'),
172 listAll(`/api/v3.1/connected_accounts?user_ids=${encodeURIComponent(USER_ID)}`),
173 ]);
174
175 if (GC) {
176 const cutoff = Date.now() - 24 * 60 * 60 * 1000;
177 // Fail closed: a missing or unparseable created_at must not read as "old
178 // enough to delete", which `new Date(0)` would.
179 const stale = r => {
180 const created = new Date(r.created_at ?? '').getTime();
181 return Number.isFinite(created) && created < cutoff;
182 };
183 const gcDelete = async (kind, path, item) => {
184 const label = `${kind} ${item.id} (${item.toolkit?.slug ?? item.name}, created ${item.created_at})`;
185 if (DRY_RUN) {
186 report(`gc: would delete ${label}`);
187 return;
188 }
189 await api('DELETE', path);
190 report(`gc: deleted ${label}`);
191 };
192
193 // Ownership marker: this script is the only thing that creates auth configs
194 // named `examples-<slug>`, so an account bound to one of them was created by
195 // an example run. Match the name exactly. findAuthConfig's fallback to any
196 // config for the toolkit is fine for reading, but must never authorise a
197 // delete, or a project's own gmail config would look example-owned.
198 // Keying on the auth config rather than the user id matters: some examples
199 // connect as 'default' rather than COMPOSIO_EXAMPLES_USER_ID.
200 const ownedNames = new Set(
201 [...BROWSER_GRANT_TOOLKITS, DEMO_TOOLKIT].map(({ slug }) => `examples-${slug}`)
202 );
203 const ownedAuthConfigIds = new Set(
204 authConfigs.filter(c => ownedNames.has(c.name)).map(c => c.id)
205 );
206 const isExampleOwned = a => ownedAuthConfigIds.has(a.auth_config?.id);
207
208 if (ownedAuthConfigIds.size === 0) {
209 report('gc: no examples-* auth configs in this project; skipping account cleanup');
210 }
211
212 // Among example-created accounts only: ones that never became ACTIVE are dead
213 // weight from OAuth-initiating runs, and surplus serpapi demo accounts pile up
214 // from api-key runs. The newest standing account per toolkit is never touched.
215 const allAccounts = (await listAll('/api/v3.1/connected_accounts')).filter(isExampleOwned);
216 const serpapiActive = allAccounts
217 .filter(a => a.toolkit?.slug === 'serpapi' && a.status === 'ACTIVE')
218 .sort((a, b) => new Date(b.created_at) - new Date(a.created_at));
219 const doomedAccounts = [
220 ...allAccounts.filter(a => a.status !== 'ACTIVE' && stale(a)),
221 ...serpapiActive.slice(1).filter(stale),
222 ];
223 for (const account of doomedAccounts) {
224 await gcDelete('connected account', `/api/v3.1/connected_accounts/${account.id}`, account);
225 }
226
227 // Example runs name their MCP configs `examples-<label>-<unix-seconds>`.
228 // Both halves are load-bearing. The reserved `examples-` prefix is what proves
229 // an example created it, and the trailing timestamp keeps a hand-made
230 // `examples-scratch` out of the delete set. Matching a bare trailing digit run
231 // instead would also claim names like `release-1754923456`. Seconds, not
232 // milliseconds: the API caps MCP names at 30 characters.
233 const EXAMPLE_MCP_NAME = /^examples-[a-z0-9-]+-\d{10}$/;
234 const mcpServers = await listAll('/api/v3.1/mcp/servers');
235 for (const server of mcpServers.filter(s => EXAMPLE_MCP_NAME.test(s.name ?? '') && stale(s))) {
236 await gcDelete('mcp config', `/api/v3.1/mcp/${server.id}`, server);
237 }
238 }
239
240 const exports = { COMPOSIO_EXAMPLES_USER_ID: USER_ID };
241 const pendingGrants = [];
242 let ok = true;
243
244 function findAuthConfig(slug) {
245 // Only the exactly-named config counts. Adopting a project's own config for
246 // the same toolkit would bind example-created accounts to a config --gc does
247 // not recognise, so they could never be cleaned up, and it would point the
248 // examples at credentials the examples do not own.
249 return authConfigs.find(
250 c => c.toolkit?.slug === slug && c.status !== 'DISABLED' && c.name === `examples-${slug}`
251 );
252 }
253 function findActiveAccount(slug, authConfigId) {
254 // Must agree with findAuthConfig. Accepting any ACTIVE account for the toolkit
255 // would pair a freshly created examples-<slug> config with an account bound to
256 // a different one, report the run complete, and leave the examples-owned
257 // config with no connection --initiate-missing would ever create.
258 return accounts.find(
259 a => a.toolkit?.slug === slug && a.status === 'ACTIVE' && a.auth_config?.id === authConfigId
260 );
261 }
262
263 for (const { exportPrefix, slug } of selectedBrowserGrantToolkits) {
264 let config = findAuthConfig(slug);
265 if (!config) {
266 const created = await api('POST', '/api/v3.1/auth_configs', {
267 toolkit: { slug },
268 auth_config: { type: 'use_composio_managed_auth', name: `examples-${slug}` },
269 });
270 config = { id: created.auth_config?.id ?? created.id };
271 report(`created auth config for ${slug}: ${config.id}`);
272 }
273 if (exportPrefix) {
274 exports[`COMPOSIO_EXAMPLES_${exportPrefix}_AUTH_CONFIG_ID`] = config.id;
275 }
276
277 const account = findActiveAccount(slug, config.id);
278 if (account) {
279 if (exportPrefix) {
280 exports[`COMPOSIO_EXAMPLES_${exportPrefix}_CONNECTED_ACCOUNT_ID`] = account.id;
281 }
282 report(`${slug}: ACTIVE connection ${account.id} (user ${USER_ID})`);
283 } else {
284 ok = false;
285 if (INITIATE_MISSING) {
286 // POST /connected_accounts is retired for Composio-managed OAuth auth
287 // configs (it 400s with a migration message); every toolkit in this loop
288 // is exactly that, so the auth-link endpoint is the only way in. It
289 // returns the redirect URL directly rather than nesting it under
290 // connectionData.
291 const created = await api('POST', '/api/v3.1/connected_accounts/link', {
292 auth_config_id: config.id,
293 user_id: USER_ID,
294 });
295 pendingGrants.push(
296 `${slug}: authorize in a browser -> ${created.redirect_url ?? '(no redirect url returned)'}`
297 );
298 } else {
299 pendingGrants.push(
300 `${slug}: no ACTIVE connection for user ${USER_ID} — rerun with --initiate-missing to get an authorization URL`
301 );
302 }
303 }
304 }
305
306 // API-key toolkit: create the auth config automatically. The examples create
307 // their own connected accounts to demonstrate that API. The stored value is a
308 // placeholder; serpapi only validates it at tool-execution time and no example
309 // executes a serpapi tool.
310 if (needsDemoToolkit) {
311 const { exportPrefix, slug, demoValue } = DEMO_TOOLKIT;
312 let config = findAuthConfig(slug);
313 if (!config) {
314 const created = await api('POST', '/api/v3.1/auth_configs', {
315 toolkit: { slug },
316 auth_config: { type: 'use_custom_auth', authScheme: 'API_KEY', name: `examples-${slug}` },
317 });
318 config = { id: created.auth_config?.id ?? created.id };
319 report(`created API-key auth config for ${slug}: ${config.id}`);
320 }
321 exports[`COMPOSIO_EXAMPLES_${exportPrefix}_AUTH_CONFIG_ID`] = config.id;
322 exports[`COMPOSIO_EXAMPLES_${exportPrefix}_PLACEHOLDER`] = demoValue;
323 }
324
325 if (pendingGrants.length) {
326 report('');
327 report('OAuth connections still needing a one-time human authorization:');
328 for (const line of pendingGrants) report(` - ${line}`);
329 }
330 report('');
331 report(ok ? 'provisioned state: complete' : 'provisioned state: INCOMPLETE (see above)');
332
333 // stdout is meant to be eval'd, so every value is single-quoted: an id the
334 // backend returned with a space or a shell metacharacter must not become code.
335 const shellQuote = value => `'${String(value).replace(/'/g, `'\\''`)}'`;
336
337 for (const [name, value] of Object.entries(exports)) {
338 if (value === undefined || value === null) {
339 ok = false;
340 report(`error: ${name} could not be resolved; not exporting it`);
341 continue;
342 }
343 console.log(`export ${name}=${shellQuote(value)}`);
344 }
345
346 process.exitCode = ok ? 0 : 1;