Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 03:37 UTC
| 1 | /** |
| 2 | * Checks that the deployed toolkit Markdown pages list the tools production |
| 3 | * serves, for the toolkits with the most tools. |
| 4 | * |
| 5 | * It also checks the toolkit list behind /api/toolkits/instant. |
| 6 | * |
| 7 | * Failures caught: |
| 8 | * - A deployed page lists a tool count matching neither production nor the |
| 9 | * committed snapshot. This is how a stale Next.js data-cache entry shows up: |
| 10 | * tableau.md listed 10 tools for days after production and the snapshot |
| 11 | * both had 436. |
| 12 | * - A production list page at TOOL_PAGE_SIZE would make a cache entry over |
| 13 | * PAGE_BYTES_BUDGET. Next.js does not cache an entry over 2 MiB, and an entry |
| 14 | * holds the body base64-encoded, so a page near 1.5 MiB will go stale once it |
| 15 | * grows past that. |
| 16 | * - The deployed /api/toolkits/instant slugs differ from production, or a |
| 17 | * production /toolkits page exceeds PAGE_BYTES_BUDGET. |
| 18 | * |
| 19 | * Usage: COMPOSIO_API_KEY=... bun scripts/check-toolkit-parity.ts [--top 30] [--markdown report.md] |
| 20 | */ |
| 21 | import { writeFile, readFile } from 'fs/promises'; |
| 22 | import { fileURLToPath } from 'url'; |
| 23 | import { z } from 'zod'; |
| 24 | import { |
| 25 | API_PAGE_LIMIT, |
| 26 | createComposioClient, |
| 27 | listToolkits, |
| 28 | listToolkitTools, |
| 29 | TOOL_PAGE_SIZE, |
| 30 | TOOLKIT_PAGE_SIZE, |
| 31 | } from '../lib/composio-api'; |
| 32 | import { instantToolkitSlugs } from '../lib/instant-toolkit-api'; |
| 33 | import { instantToolkitSlugsSchema } from '../lib/instant-toolkit-filter'; |
| 34 | import { fetchWithRetry } from './fetch-with-retry'; |
| 35 | |
| 36 | /** The Next.js data-cache entry limit, measured on the serialized entry. */ |
| 37 | export const CACHE_ENTRY_LIMIT = 2 * 1024 * 1024; |
| 38 | |
| 39 | /** Alert when an entry reaches 85% of the limit. */ |
| 40 | export const PAGE_BYTES_BUDGET = Math.floor(CACHE_ENTRY_LIMIT * 0.85); |
| 41 | |
| 42 | /** |
| 43 | * The size Next.js counts for a cached response: the body is stored |
| 44 | * base64-encoded, so an entry is about 4/3 of the raw body. |
| 45 | */ |
| 46 | export function cacheEntryBytes(responseBytes: number): number { |
| 47 | return Math.ceil(responseBytes / 3) * 4; |
| 48 | } |
| 49 | |
| 50 | const DOCS_BASE_URL = 'https://docs.composio.dev'; |
| 51 | const SNAPSHOT_PATH = fileURLToPath(new URL('../public/data/toolkits.json', import.meta.url)); |
| 52 | |
| 53 | export interface ToolkitObservation { |
| 54 | slug: string; |
| 55 | snapshotCount: number; |
| 56 | productionCount: number; |
| 57 | deployedCount: number; |
| 58 | largestPageBytes: number; |
| 59 | } |
| 60 | |
| 61 | /** Counts the tools listed in the "## Tools" section of a toolkit Markdown page. */ |
| 62 | export function countMarkdownTools(markdown: string): number { |
| 63 | const start = markdown.indexOf('\n## Tools\n'); |
| 64 | if (start === -1) return 0; |
| 65 | // Tool descriptions can contain headings. The renderer puts triggers after tools. |
| 66 | const triggers = markdown.indexOf('\n## Triggers\n', start + 1); |
| 67 | const section = markdown.slice(start, triggers === -1 ? undefined : triggers); |
| 68 | return section.match(/^\*\*Slug:\*\* `/gm)?.length ?? 0; |
| 69 | } |
| 70 | |
| 71 | /** Describes every way one toolkit's observation breaks parity or the page budget. */ |
| 72 | export function findProblems(observation: ToolkitObservation): string[] { |
| 73 | const { slug, snapshotCount, productionCount, deployedCount, largestPageBytes } = observation; |
| 74 | const problems: string[] = []; |
| 75 | if (deployedCount !== productionCount && deployedCount !== snapshotCount) { |
| 76 | problems.push( |
| 77 | `${slug}.md lists ${deployedCount} tools; production has ${productionCount}, the snapshot has ${snapshotCount}` |
| 78 | ); |
| 79 | } |
| 80 | if (snapshotCount === API_PAGE_LIMIT && productionCount > API_PAGE_LIMIT) { |
| 81 | problems.push( |
| 82 | `the ${slug} snapshot stops at ${snapshotCount} of ${productionCount} tools, the API page cap; the committed snapshot predates the paginated generator, so re-run generate:toolkits` |
| 83 | ); |
| 84 | } |
| 85 | if (cacheEntryBytes(largestPageBytes) > PAGE_BYTES_BUDGET) { |
| 86 | problems.push( |
| 87 | `${slug} has a ${largestPageBytes}-byte tools page (${cacheEntryBytes(largestPageBytes)}-byte cache entry) at TOOL_PAGE_SIZE ${TOOL_PAGE_SIZE}, over the ${PAGE_BYTES_BUDGET}-byte entry budget; lower TOOL_PAGE_SIZE in lib/composio-api.ts` |
| 88 | ); |
| 89 | } |
| 90 | return problems; |
| 91 | } |
| 92 | |
| 93 | export interface InstantObservation { |
| 94 | productionSlugs: string[]; |
| 95 | deployedSlugs: string[]; |
| 96 | largestPageBytes: number; |
| 97 | } |
| 98 | |
| 99 | /** Describes how the deployed Instant toolkit list differs from production. */ |
| 100 | export function findInstantProblems({ productionSlugs, deployedSlugs, largestPageBytes }: InstantObservation): string[] { |
| 101 | const problems: string[] = []; |
| 102 | const deployed = new Set(deployedSlugs); |
| 103 | const production = new Set(productionSlugs); |
| 104 | const missing = productionSlugs.filter(slug => !deployed.has(slug)); |
| 105 | const extra = deployedSlugs.filter(slug => !production.has(slug)); |
| 106 | if (missing.length > 0 || extra.length > 0) { |
| 107 | problems.push( |
| 108 | `/api/toolkits/instant lists ${deployedSlugs.length} toolkits; production has ${productionSlugs.length} (missing: ${missing.slice(0, 5).join(', ') || 'none'}; extra: ${extra.slice(0, 5).join(', ') || 'none'})` |
| 109 | ); |
| 110 | } |
| 111 | if (cacheEntryBytes(largestPageBytes) > PAGE_BYTES_BUDGET) { |
| 112 | problems.push( |
| 113 | `a ${largestPageBytes}-byte /toolkits page (${cacheEntryBytes(largestPageBytes)}-byte cache entry) at TOOLKIT_PAGE_SIZE ${TOOLKIT_PAGE_SIZE}, over the ${PAGE_BYTES_BUDGET}-byte entry budget; lower TOOLKIT_PAGE_SIZE in lib/composio-api.ts` |
| 114 | ); |
| 115 | } |
| 116 | return problems; |
| 117 | } |
| 118 | |
| 119 | const snapshotSchema = z.array(z.object({ slug: z.string(), toolCount: z.number() })); |
| 120 | |
| 121 | type CountedToolkit = z.infer<typeof snapshotSchema>[number]; |
| 122 | |
| 123 | const productionToolkitsSchema = z.array( |
| 124 | z.object({ slug: z.string(), meta: z.object({ tools_count: z.number() }) }) |
| 125 | ); |
| 126 | |
| 127 | /** Production tool counts from a /toolkits listing, keyed by lowercase slug. */ |
| 128 | export function productionToolCounts(items: unknown[]): CountedToolkit[] { |
| 129 | return productionToolkitsSchema |
| 130 | .parse(items) |
| 131 | .map(toolkit => ({ slug: toolkit.slug.toLowerCase(), toolCount: toolkit.meta.tools_count })); |
| 132 | } |
| 133 | |
| 134 | /** |
| 135 | * The toolkits to check: the `top` largest by production count, then the `top` |
| 136 | * largest by snapshot count, without duplicates. Ranking by production too |
| 137 | * keeps a toolkit whose snapshot count collapsed in the set. Only published |
| 138 | * toolkits (those in the snapshot) are eligible, since others have no page. |
| 139 | */ |
| 140 | export function selectWatchList( |
| 141 | snapshot: CountedToolkit[], |
| 142 | production: CountedToolkit[], |
| 143 | top: number |
| 144 | ): CountedToolkit[] { |
| 145 | const published = new Map(snapshot.map(toolkit => [toolkit.slug, toolkit])); |
| 146 | const largest = (toolkits: CountedToolkit[]) => |
| 147 | [...toolkits].sort((a, b) => b.toolCount - a.toolCount).slice(0, top); |
| 148 | const fromProduction = largest(production.filter(toolkit => published.has(toolkit.slug))).flatMap( |
| 149 | toolkit => published.get(toolkit.slug) ?? [] |
| 150 | ); |
| 151 | const watched = new Map<string, CountedToolkit>(); |
| 152 | for (const toolkit of [...fromProduction, ...largest(snapshot)]) { |
| 153 | if (!watched.has(toolkit.slug)) watched.set(toolkit.slug, toolkit); |
| 154 | } |
| 155 | return [...watched.values()]; |
| 156 | } |
| 157 | |
| 158 | /** A production client that records the size of the largest response body it reads. */ |
| 159 | function measuringClient(apiKey: string) { |
| 160 | let largestPageBytes = 0; |
| 161 | const client = createComposioClient(apiKey, { |
| 162 | maxRetries: 0, |
| 163 | timeout: 10 * 60_000, |
| 164 | fetch: async (input, init) => { |
| 165 | const response = await fetchWithRetry(String(input), init); |
| 166 | largestPageBytes = Math.max(largestPageBytes, Buffer.byteLength(await response.clone().text())); |
| 167 | return response; |
| 168 | }, |
| 169 | }); |
| 170 | return { client, largestPageBytes: () => largestPageBytes }; |
| 171 | } |
| 172 | |
| 173 | async function observeProduction(slug: string, apiKey: string) { |
| 174 | const { client, largestPageBytes } = measuringClient(apiKey); |
| 175 | const tools = await listToolkitTools(client, slug); |
| 176 | return { productionCount: tools.length, largestPageBytes: largestPageBytes() }; |
| 177 | } |
| 178 | |
| 179 | interface CatalogObservation { |
| 180 | toolkits: unknown[]; |
| 181 | largestPageBytes: number; |
| 182 | } |
| 183 | |
| 184 | async function observeCatalog(apiKey: string): Promise<CatalogObservation> { |
| 185 | const { client, largestPageBytes } = measuringClient(apiKey); |
| 186 | const toolkits = await listToolkits(client); |
| 187 | return { toolkits, largestPageBytes: largestPageBytes() }; |
| 188 | } |
| 189 | |
| 190 | async function observeInstant({ toolkits, largestPageBytes }: CatalogObservation): Promise<InstantObservation> { |
| 191 | const response = await fetchWithRetry(`${DOCS_BASE_URL}/api/toolkits/instant`); |
| 192 | if (!response.ok) throw new Error(`/api/toolkits/instant responded ${response.status}`); |
| 193 | const deployed = instantToolkitSlugsSchema.parse(await response.json()); |
| 194 | return { productionSlugs: instantToolkitSlugs(toolkits), deployedSlugs: deployed.slugs, largestPageBytes }; |
| 195 | } |
| 196 | |
| 197 | async function observeDeployed(slug: string): Promise<number> { |
| 198 | const response = await fetchWithRetry(`${DOCS_BASE_URL}/toolkits/${slug}.md`); |
| 199 | if (!response.ok) throw new Error(`${slug}.md responded ${response.status}`); |
| 200 | return countMarkdownTools(await response.text()); |
| 201 | } |
| 202 | |
| 203 | function argValue(name: string): string | undefined { |
| 204 | const index = process.argv.indexOf(name); |
| 205 | return index === -1 ? undefined : process.argv[index + 1]; |
| 206 | } |
| 207 | |
| 208 | async function main() { |
| 209 | const apiKey = process.env.COMPOSIO_API_KEY; |
| 210 | if (!apiKey) throw new Error('COMPOSIO_API_KEY is required'); |
| 211 | const top = Number(argValue('--top') ?? 30); |
| 212 | const markdownPath = argValue('--markdown'); |
| 213 | |
| 214 | const snapshot = snapshotSchema.parse(JSON.parse(await readFile(SNAPSHOT_PATH, 'utf-8'))); |
| 215 | |
| 216 | const problems: string[] = []; |
| 217 | const rows: string[] = []; |
| 218 | |
| 219 | // One production catalog read feeds both the watch list and the Instant check. |
| 220 | let catalog: CatalogObservation | undefined; |
| 221 | let production: CountedToolkit[] = []; |
| 222 | try { |
| 223 | catalog = await observeCatalog(apiKey); |
| 224 | production = productionToolCounts(catalog.toolkits); |
| 225 | } catch (error) { |
| 226 | problems.push(`Production toolkit catalog: ${error instanceof Error ? error.message : String(error)}`); |
| 227 | } |
| 228 | const toolkits = selectWatchList(snapshot, production, top); |
| 229 | for (const toolkit of toolkits) { |
| 230 | try { |
| 231 | const production = await observeProduction(toolkit.slug, apiKey); |
| 232 | const deployedCount = await observeDeployed(toolkit.slug); |
| 233 | const observation = { slug: toolkit.slug, snapshotCount: toolkit.toolCount, deployedCount, ...production }; |
| 234 | problems.push(...findProblems(observation)); |
| 235 | rows.push( |
| 236 | `| ${toolkit.slug} | ${toolkit.toolCount} | ${production.productionCount} | ${deployedCount} | ${production.largestPageBytes} |` |
| 237 | ); |
| 238 | } catch (error) { |
| 239 | problems.push(`${toolkit.slug}: ${error instanceof Error ? error.message : String(error)}`); |
| 240 | } |
| 241 | } |
| 242 | |
| 243 | if (catalog) { |
| 244 | try { |
| 245 | const instant = await observeInstant(catalog); |
| 246 | problems.push(...findInstantProblems(instant)); |
| 247 | rows.push( |
| 248 | `| (Instant toolkit list) | n/a | ${instant.productionSlugs.length} | ${instant.deployedSlugs.length} | ${instant.largestPageBytes} |` |
| 249 | ); |
| 250 | } catch (error) { |
| 251 | problems.push(`Instant toolkit list: ${error instanceof Error ? error.message : String(error)}`); |
| 252 | } |
| 253 | } |
| 254 | |
| 255 | const report = [ |
| 256 | problems.length === 0 ? '✅ Toolkit pages match production.' : `❌ ${problems.length} toolkit parity problem(s):`, |
| 257 | '', |
| 258 | ...problems.map(problem => `- ${problem}`), |
| 259 | '', |
| 260 | '| Toolkit | Snapshot | Production | Deployed .md | Largest page (bytes) |', |
| 261 | '|---|---|---|---|---|', |
| 262 | ...rows, |
| 263 | ].join('\n'); |
| 264 | console.log(report); |
| 265 | if (markdownPath) await writeFile(markdownPath, report); |
| 266 | process.exit(problems.length === 0 ? 0 : 1); |
| 267 | } |
| 268 | |
| 269 | if (import.meta.main) await main(); |