Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 03:37 UTC
| 1 | /** |
| 2 | * Toolkit-version source guard and distribution smoke test. |
| 3 | * |
| 4 | * Companion to production-urls.test.ts. That guard catches staging *hostnames* |
| 5 | * leaking into the committed docs data; this one catches staging *versions*. |
| 6 | * |
| 7 | * The "Docs - Update Data" workflow snapshots `public/data/toolkits.json` from the |
| 8 | * backend every ~5h and auto-commits it. When it is (mis)pointed at STAGING, the |
| 9 | * `version` field leaks: staging bumps the ENTIRE catalog to a single internal |
| 10 | * version per day (e.g. `20260703_00`), so ~all toolkits report the same version. |
| 11 | * Production releases per-toolkit, so its versions are diverse and no single value |
| 12 | * dominates the catalog. |
| 13 | * |
| 14 | * This surfaced in a customer report (2026-07-08): every toolkit page showed |
| 15 | * "Latest version: 20260703_00", an internal version that does not exist on |
| 16 | * production (Gmail's real latest was 20260702_01). The hostname guard did not |
| 17 | * catch it because a version string is not a URL. |
| 18 | * |
| 19 | * Source provenance is enforced separately: the generator rejects non-production |
| 20 | * API bases, and its shared changelog fetch is pinned to production. The thresholds |
| 21 | * below are deliberately only a smoke signal for the whole-catalog staging-bump |
| 22 | * pattern; they are not proof of provenance. Empirically, production's most common |
| 23 | * version covers ~0.73 of the versioned catalog across ~25 distinct versions; a |
| 24 | * whole-catalog staging snapshot is ~0.95 across a handful. Note `00000000_00` is a |
| 25 | * legitimate production value for a few unreleased toolkits. |
| 26 | */ |
| 27 | import { describe, test, expect } from "bun:test"; |
| 28 | import { readFile } from "fs/promises"; |
| 29 | import { join } from "path"; |
| 30 | import { requireProductionApiV3Url } from "../../scripts/production-api.mjs"; |
| 31 | import { |
| 32 | applyToolkitVersions, |
| 33 | fetchProductionToolkitVersions, |
| 34 | type ToolkitVersionFetcher, |
| 35 | } from "../../scripts/toolkit-versions"; |
| 36 | |
| 37 | const DATA_DIR = join(import.meta.dir, "../../public/data"); |
| 38 | |
| 39 | /** Released toolkit versions are date-stamped: `YYYYMMDD_NN` (incl. `00000000_00`). */ |
| 40 | const VERSION_RE = /^\d{8}_\d{2}$/; |
| 41 | |
| 42 | /** |
| 43 | * Max fraction of versioned toolkits that may share one version. Production sits |
| 44 | * near 0.73; a whole-catalog staging bump sits near 0.95. 0.90 separates them with |
| 45 | * headroom on both sides. A legitimate breach is a canary worth investigating, not |
| 46 | * a value to quietly raise. |
| 47 | */ |
| 48 | const MAX_SINGLE_VERSION_SHARE = 0.9; |
| 49 | |
| 50 | /** |
| 51 | * Min distinct versions across the catalog. Production carries ~25; a staging |
| 52 | * snapshot carries a handful (the leak that prompted this guard had 7). 10 catches |
| 53 | * the staging signature with room to spare. |
| 54 | */ |
| 55 | const MIN_DISTINCT_VERSIONS = 10; |
| 56 | |
| 57 | interface Toolkit { |
| 58 | slug: string; |
| 59 | version: string | null; |
| 60 | } |
| 61 | |
| 62 | describe("Toolkit version source - production only", () => { |
| 63 | test("rejects non-production API overrides", () => { |
| 64 | expect(requireProductionApiV3Url(undefined)).toBe( |
| 65 | "https://backend.composio.dev/api/v3" |
| 66 | ); |
| 67 | expect(requireProductionApiV3Url("https://backend.composio.dev/api/v3/")).toBe( |
| 68 | "https://backend.composio.dev/api/v3" |
| 69 | ); |
| 70 | expect(() => |
| 71 | requireProductionApiV3Url("https://staging-backend.composio.dev/api/v3") |
| 72 | ).toThrow("Published docs data must be generated from"); |
| 73 | }); |
| 74 | |
| 75 | test("fetches the changelog from the fixed production endpoint", async () => { |
| 76 | const requests: string[] = []; |
| 77 | const fetcher: ToolkitVersionFetcher = async (url, options) => { |
| 78 | requests.push(url); |
| 79 | expect(new Headers(options?.headers).get("x-api-key")).toBe("test-key"); |
| 80 | return Response.json({ |
| 81 | items: [{ slug: "GMAIL", versions: [{ version: "20260702_01" }] }], |
| 82 | }); |
| 83 | }; |
| 84 | |
| 85 | const versions = await fetchProductionToolkitVersions("test-key", fetcher); |
| 86 | |
| 87 | expect(requests).toEqual([ |
| 88 | "https://backend.composio.dev/api/v3.1/toolkits/changelog", |
| 89 | ]); |
| 90 | expect(versions.get("gmail")).toBe("20260702_01"); |
| 91 | }); |
| 92 | |
| 93 | test("rejects malformed changelog data before rewriting the catalog", async () => { |
| 94 | const fetcher: ToolkitVersionFetcher = async () => Response.json({}); |
| 95 | |
| 96 | expect(fetchProductionToolkitVersions("test-key", fetcher)).rejects.toThrow( |
| 97 | "missing an items array" |
| 98 | ); |
| 99 | }); |
| 100 | |
| 101 | test("skips malformed changelog entries instead of failing the run", async () => { |
| 102 | const fetcher: ToolkitVersionFetcher = async () => |
| 103 | Response.json({ |
| 104 | items: [ |
| 105 | null, |
| 106 | { slug: 7, versions: [{ version: "20260101_01" }] }, |
| 107 | { slug: "SLACK", versions: "none" }, |
| 108 | { slug: "NOTION", versions: [{ version: 3 }] }, |
| 109 | { slug: "GMAIL", versions: [{ version: "20260702_01" }] }, |
| 110 | ], |
| 111 | }); |
| 112 | |
| 113 | const versions = await fetchProductionToolkitVersions("test-key", fetcher); |
| 114 | |
| 115 | expect([...versions]).toEqual([["gmail", "20260702_01"]]); |
| 116 | }); |
| 117 | |
| 118 | test("rejects an empty production version map before rewriting the catalog", async () => { |
| 119 | const fetcher: ToolkitVersionFetcher = async () => Response.json({ items: [] }); |
| 120 | |
| 121 | expect(fetchProductionToolkitVersions("test-key", fetcher)).rejects.toThrow( |
| 122 | "contains no toolkit versions" |
| 123 | ); |
| 124 | }); |
| 125 | |
| 126 | test("uses null when a toolkit is absent from the production changelog", () => { |
| 127 | const toolkits = [ |
| 128 | { slug: "gmail", version: "stale" }, |
| 129 | { slug: "staging-only", version: "stale" }, |
| 130 | ]; |
| 131 | |
| 132 | expect( |
| 133 | applyToolkitVersions(toolkits, new Map([["gmail", "20260702_01"]])) |
| 134 | ).toEqual({ matched: 1, missing: 1 }); |
| 135 | expect(toolkits).toEqual([ |
| 136 | { slug: "gmail", version: "20260702_01" }, |
| 137 | { slug: "staging-only", version: null }, |
| 138 | ]); |
| 139 | }); |
| 140 | }); |
| 141 | |
| 142 | async function loadToolkits(): Promise<Toolkit[]> { |
| 143 | const raw = await readFile(join(DATA_DIR, "toolkits.json"), "utf-8"); |
| 144 | return JSON.parse(raw) as Toolkit[]; |
| 145 | } |
| 146 | |
| 147 | describe("Toolkit version distribution smoke test", () => { |
| 148 | test("every non-null version is a valid date-stamped version", async () => { |
| 149 | const toolkits = await loadToolkits(); |
| 150 | const offenders = toolkits |
| 151 | .filter((t) => t.version != null && !VERSION_RE.test(t.version)) |
| 152 | .map((t) => `${t.slug}: ${t.version}`); |
| 153 | expect(offenders).toEqual([]); |
| 154 | }); |
| 155 | |
| 156 | test("versions are diverse and no single one dominates the catalog", async () => { |
| 157 | const toolkits = await loadToolkits(); |
| 158 | const versioned = toolkits |
| 159 | .map((t) => t.version) |
| 160 | .filter((v): v is string => v != null); |
| 161 | |
| 162 | // Guard needs a meaningful sample; the catalog is ~1k toolkits. |
| 163 | expect(versioned.length).toBeGreaterThan(100); |
| 164 | |
| 165 | const counts = new Map<string, number>(); |
| 166 | for (const v of versioned) counts.set(v, (counts.get(v) ?? 0) + 1); |
| 167 | |
| 168 | const topCount = Math.max(...counts.values()); |
| 169 | const share = topCount / versioned.length; |
| 170 | |
| 171 | // A staging whole-catalog bump collapses both metrics at once: one version |
| 172 | // covers ~all toolkits, and the distinct count drops to a handful. |
| 173 | expect(counts.size).toBeGreaterThanOrEqual(MIN_DISTINCT_VERSIONS); |
| 174 | expect(share).toBeLessThan(MAX_SINGLE_VERSION_SHARE); |
| 175 | }); |
| 176 | }); |