@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
ts 269 lines 11.1 KB
Raw
1 /**
2 * Checks that the deployed toolkit Markdown pages list the tools production
3 * serves, for the toolkits with the most tools.
4 *
5 * It also checks the toolkit list behind /api/toolkits/instant.
6 *
7 * Failures caught:
8 * - A deployed page lists a tool count matching neither production nor the
9 * committed snapshot. This is how a stale Next.js data-cache entry shows up:
10 * tableau.md listed 10 tools for days after production and the snapshot
11 * both had 436.
12 * - A production list page at TOOL_PAGE_SIZE would make a cache entry over
13 * PAGE_BYTES_BUDGET. Next.js does not cache an entry over 2 MiB, and an entry
14 * holds the body base64-encoded, so a page near 1.5 MiB will go stale once it
15 * grows past that.
16 * - The deployed /api/toolkits/instant slugs differ from production, or a
17 * production /toolkits page exceeds PAGE_BYTES_BUDGET.
18 *
19 * Usage: COMPOSIO_API_KEY=... bun scripts/check-toolkit-parity.ts [--top 30] [--markdown report.md]
20 */
21 import { writeFile, readFile } from 'fs/promises';
22 import { fileURLToPath } from 'url';
23 import { z } from 'zod';
24 import {
25 API_PAGE_LIMIT,
26 createComposioClient,
27 listToolkits,
28 listToolkitTools,
29 TOOL_PAGE_SIZE,
30 TOOLKIT_PAGE_SIZE,
31 } from '../lib/composio-api';
32 import { instantToolkitSlugs } from '../lib/instant-toolkit-api';
33 import { instantToolkitSlugsSchema } from '../lib/instant-toolkit-filter';
34 import { fetchWithRetry } from './fetch-with-retry';
35
36 /** The Next.js data-cache entry limit, measured on the serialized entry. */
37 export const CACHE_ENTRY_LIMIT = 2 * 1024 * 1024;
38
39 /** Alert when an entry reaches 85% of the limit. */
40 export const PAGE_BYTES_BUDGET = Math.floor(CACHE_ENTRY_LIMIT * 0.85);
41
42 /**
43 * The size Next.js counts for a cached response: the body is stored
44 * base64-encoded, so an entry is about 4/3 of the raw body.
45 */
46 export function cacheEntryBytes(responseBytes: number): number {
47 return Math.ceil(responseBytes / 3) * 4;
48 }
49
50 const DOCS_BASE_URL = 'https://docs.composio.dev';
51 const SNAPSHOT_PATH = fileURLToPath(new URL('../public/data/toolkits.json', import.meta.url));
52
53 export interface ToolkitObservation {
54 slug: string;
55 snapshotCount: number;
56 productionCount: number;
57 deployedCount: number;
58 largestPageBytes: number;
59 }
60
61 /** Counts the tools listed in the "## Tools" section of a toolkit Markdown page. */
62 export function countMarkdownTools(markdown: string): number {
63 const start = markdown.indexOf('\n## Tools\n');
64 if (start === -1) return 0;
65 // Tool descriptions can contain headings. The renderer puts triggers after tools.
66 const triggers = markdown.indexOf('\n## Triggers\n', start + 1);
67 const section = markdown.slice(start, triggers === -1 ? undefined : triggers);
68 return section.match(/^\*\*Slug:\*\* `/gm)?.length ?? 0;
69 }
70
71 /** Describes every way one toolkit's observation breaks parity or the page budget. */
72 export function findProblems(observation: ToolkitObservation): string[] {
73 const { slug, snapshotCount, productionCount, deployedCount, largestPageBytes } = observation;
74 const problems: string[] = [];
75 if (deployedCount !== productionCount && deployedCount !== snapshotCount) {
76 problems.push(
77 `${slug}.md lists ${deployedCount} tools; production has ${productionCount}, the snapshot has ${snapshotCount}`
78 );
79 }
80 if (snapshotCount === API_PAGE_LIMIT && productionCount > API_PAGE_LIMIT) {
81 problems.push(
82 `the ${slug} snapshot stops at ${snapshotCount} of ${productionCount} tools, the API page cap; the committed snapshot predates the paginated generator, so re-run generate:toolkits`
83 );
84 }
85 if (cacheEntryBytes(largestPageBytes) > PAGE_BYTES_BUDGET) {
86 problems.push(
87 `${slug} has a ${largestPageBytes}-byte tools page (${cacheEntryBytes(largestPageBytes)}-byte cache entry) at TOOL_PAGE_SIZE ${TOOL_PAGE_SIZE}, over the ${PAGE_BYTES_BUDGET}-byte entry budget; lower TOOL_PAGE_SIZE in lib/composio-api.ts`
88 );
89 }
90 return problems;
91 }
92
93 export interface InstantObservation {
94 productionSlugs: string[];
95 deployedSlugs: string[];
96 largestPageBytes: number;
97 }
98
99 /** Describes how the deployed Instant toolkit list differs from production. */
100 export function findInstantProblems({ productionSlugs, deployedSlugs, largestPageBytes }: InstantObservation): string[] {
101 const problems: string[] = [];
102 const deployed = new Set(deployedSlugs);
103 const production = new Set(productionSlugs);
104 const missing = productionSlugs.filter(slug => !deployed.has(slug));
105 const extra = deployedSlugs.filter(slug => !production.has(slug));
106 if (missing.length > 0 || extra.length > 0) {
107 problems.push(
108 `/api/toolkits/instant lists ${deployedSlugs.length} toolkits; production has ${productionSlugs.length} (missing: ${missing.slice(0, 5).join(', ') || 'none'}; extra: ${extra.slice(0, 5).join(', ') || 'none'})`
109 );
110 }
111 if (cacheEntryBytes(largestPageBytes) > PAGE_BYTES_BUDGET) {
112 problems.push(
113 `a ${largestPageBytes}-byte /toolkits page (${cacheEntryBytes(largestPageBytes)}-byte cache entry) at TOOLKIT_PAGE_SIZE ${TOOLKIT_PAGE_SIZE}, over the ${PAGE_BYTES_BUDGET}-byte entry budget; lower TOOLKIT_PAGE_SIZE in lib/composio-api.ts`
114 );
115 }
116 return problems;
117 }
118
119 const snapshotSchema = z.array(z.object({ slug: z.string(), toolCount: z.number() }));
120
121 type CountedToolkit = z.infer<typeof snapshotSchema>[number];
122
123 const productionToolkitsSchema = z.array(
124 z.object({ slug: z.string(), meta: z.object({ tools_count: z.number() }) })
125 );
126
127 /** Production tool counts from a /toolkits listing, keyed by lowercase slug. */
128 export function productionToolCounts(items: unknown[]): CountedToolkit[] {
129 return productionToolkitsSchema
130 .parse(items)
131 .map(toolkit => ({ slug: toolkit.slug.toLowerCase(), toolCount: toolkit.meta.tools_count }));
132 }
133
134 /**
135 * The toolkits to check: the `top` largest by production count, then the `top`
136 * largest by snapshot count, without duplicates. Ranking by production too
137 * keeps a toolkit whose snapshot count collapsed in the set. Only published
138 * toolkits (those in the snapshot) are eligible, since others have no page.
139 */
140 export function selectWatchList(
141 snapshot: CountedToolkit[],
142 production: CountedToolkit[],
143 top: number
144 ): CountedToolkit[] {
145 const published = new Map(snapshot.map(toolkit => [toolkit.slug, toolkit]));
146 const largest = (toolkits: CountedToolkit[]) =>
147 [...toolkits].sort((a, b) => b.toolCount - a.toolCount).slice(0, top);
148 const fromProduction = largest(production.filter(toolkit => published.has(toolkit.slug))).flatMap(
149 toolkit => published.get(toolkit.slug) ?? []
150 );
151 const watched = new Map<string, CountedToolkit>();
152 for (const toolkit of [...fromProduction, ...largest(snapshot)]) {
153 if (!watched.has(toolkit.slug)) watched.set(toolkit.slug, toolkit);
154 }
155 return [...watched.values()];
156 }
157
158 /** A production client that records the size of the largest response body it reads. */
159 function measuringClient(apiKey: string) {
160 let largestPageBytes = 0;
161 const client = createComposioClient(apiKey, {
162 maxRetries: 0,
163 timeout: 10 * 60_000,
164 fetch: async (input, init) => {
165 const response = await fetchWithRetry(String(input), init);
166 largestPageBytes = Math.max(largestPageBytes, Buffer.byteLength(await response.clone().text()));
167 return response;
168 },
169 });
170 return { client, largestPageBytes: () => largestPageBytes };
171 }
172
173 async function observeProduction(slug: string, apiKey: string) {
174 const { client, largestPageBytes } = measuringClient(apiKey);
175 const tools = await listToolkitTools(client, slug);
176 return { productionCount: tools.length, largestPageBytes: largestPageBytes() };
177 }
178
179 interface CatalogObservation {
180 toolkits: unknown[];
181 largestPageBytes: number;
182 }
183
184 async function observeCatalog(apiKey: string): Promise<CatalogObservation> {
185 const { client, largestPageBytes } = measuringClient(apiKey);
186 const toolkits = await listToolkits(client);
187 return { toolkits, largestPageBytes: largestPageBytes() };
188 }
189
190 async function observeInstant({ toolkits, largestPageBytes }: CatalogObservation): Promise<InstantObservation> {
191 const response = await fetchWithRetry(`${DOCS_BASE_URL}/api/toolkits/instant`);
192 if (!response.ok) throw new Error(`/api/toolkits/instant responded ${response.status}`);
193 const deployed = instantToolkitSlugsSchema.parse(await response.json());
194 return { productionSlugs: instantToolkitSlugs(toolkits), deployedSlugs: deployed.slugs, largestPageBytes };
195 }
196
197 async function observeDeployed(slug: string): Promise<number> {
198 const response = await fetchWithRetry(`${DOCS_BASE_URL}/toolkits/${slug}.md`);
199 if (!response.ok) throw new Error(`${slug}.md responded ${response.status}`);
200 return countMarkdownTools(await response.text());
201 }
202
203 function argValue(name: string): string | undefined {
204 const index = process.argv.indexOf(name);
205 return index === -1 ? undefined : process.argv[index + 1];
206 }
207
208 async function main() {
209 const apiKey = process.env.COMPOSIO_API_KEY;
210 if (!apiKey) throw new Error('COMPOSIO_API_KEY is required');
211 const top = Number(argValue('--top') ?? 30);
212 const markdownPath = argValue('--markdown');
213
214 const snapshot = snapshotSchema.parse(JSON.parse(await readFile(SNAPSHOT_PATH, 'utf-8')));
215
216 const problems: string[] = [];
217 const rows: string[] = [];
218
219 // One production catalog read feeds both the watch list and the Instant check.
220 let catalog: CatalogObservation | undefined;
221 let production: CountedToolkit[] = [];
222 try {
223 catalog = await observeCatalog(apiKey);
224 production = productionToolCounts(catalog.toolkits);
225 } catch (error) {
226 problems.push(`Production toolkit catalog: ${error instanceof Error ? error.message : String(error)}`);
227 }
228 const toolkits = selectWatchList(snapshot, production, top);
229 for (const toolkit of toolkits) {
230 try {
231 const production = await observeProduction(toolkit.slug, apiKey);
232 const deployedCount = await observeDeployed(toolkit.slug);
233 const observation = { slug: toolkit.slug, snapshotCount: toolkit.toolCount, deployedCount, ...production };
234 problems.push(...findProblems(observation));
235 rows.push(
236 `| ${toolkit.slug} | ${toolkit.toolCount} | ${production.productionCount} | ${deployedCount} | ${production.largestPageBytes} |`
237 );
238 } catch (error) {
239 problems.push(`${toolkit.slug}: ${error instanceof Error ? error.message : String(error)}`);
240 }
241 }
242
243 if (catalog) {
244 try {
245 const instant = await observeInstant(catalog);
246 problems.push(...findInstantProblems(instant));
247 rows.push(
248 `| (Instant toolkit list) | n/a | ${instant.productionSlugs.length} | ${instant.deployedSlugs.length} | ${instant.largestPageBytes} |`
249 );
250 } catch (error) {
251 problems.push(`Instant toolkit list: ${error instanceof Error ? error.message : String(error)}`);
252 }
253 }
254
255 const report = [
256 problems.length === 0 ? '✅ Toolkit pages match production.' : `❌ ${problems.length} toolkit parity problem(s):`,
257 '',
258 ...problems.map(problem => `- ${problem}`),
259 '',
260 '| Toolkit | Snapshot | Production | Deployed .md | Largest page (bytes) |',
261 '|---|---|---|---|---|',
262 ...rows,
263 ].join('\n');
264 console.log(report);
265 if (markdownPath) await writeFile(markdownPath, report);
266 process.exit(problems.length === 0 ? 0 : 1);
267 }
268
269 if (import.meta.main) await main();