@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
ts 176 lines 6.8 KB
Raw
1 /**
2 * Toolkit-version source guard and distribution smoke test.
3 *
4 * Companion to production-urls.test.ts. That guard catches staging *hostnames*
5 * leaking into the committed docs data; this one catches staging *versions*.
6 *
7 * The "Docs - Update Data" workflow snapshots `public/data/toolkits.json` from the
8 * backend every ~5h and auto-commits it. When it is (mis)pointed at STAGING, the
9 * `version` field leaks: staging bumps the ENTIRE catalog to a single internal
10 * version per day (e.g. `20260703_00`), so ~all toolkits report the same version.
11 * Production releases per-toolkit, so its versions are diverse and no single value
12 * dominates the catalog.
13 *
14 * This surfaced in a customer report (2026-07-08): every toolkit page showed
15 * "Latest version: 20260703_00", an internal version that does not exist on
16 * production (Gmail's real latest was 20260702_01). The hostname guard did not
17 * catch it because a version string is not a URL.
18 *
19 * Source provenance is enforced separately: the generator rejects non-production
20 * API bases, and its shared changelog fetch is pinned to production. The thresholds
21 * below are deliberately only a smoke signal for the whole-catalog staging-bump
22 * pattern; they are not proof of provenance. Empirically, production's most common
23 * version covers ~0.73 of the versioned catalog across ~25 distinct versions; a
24 * whole-catalog staging snapshot is ~0.95 across a handful. Note `00000000_00` is a
25 * legitimate production value for a few unreleased toolkits.
26 */
27 import { describe, test, expect } from "bun:test";
28 import { readFile } from "fs/promises";
29 import { join } from "path";
30 import { requireProductionApiV3Url } from "../../scripts/production-api.mjs";
31 import {
32 applyToolkitVersions,
33 fetchProductionToolkitVersions,
34 type ToolkitVersionFetcher,
35 } from "../../scripts/toolkit-versions";
36
37 const DATA_DIR = join(import.meta.dir, "../../public/data");
38
39 /** Released toolkit versions are date-stamped: `YYYYMMDD_NN` (incl. `00000000_00`). */
40 const VERSION_RE = /^\d{8}_\d{2}$/;
41
42 /**
43 * Max fraction of versioned toolkits that may share one version. Production sits
44 * near 0.73; a whole-catalog staging bump sits near 0.95. 0.90 separates them with
45 * headroom on both sides. A legitimate breach is a canary worth investigating, not
46 * a value to quietly raise.
47 */
48 const MAX_SINGLE_VERSION_SHARE = 0.9;
49
50 /**
51 * Min distinct versions across the catalog. Production carries ~25; a staging
52 * snapshot carries a handful (the leak that prompted this guard had 7). 10 catches
53 * the staging signature with room to spare.
54 */
55 const MIN_DISTINCT_VERSIONS = 10;
56
57 interface Toolkit {
58 slug: string;
59 version: string | null;
60 }
61
62 describe("Toolkit version source - production only", () => {
63 test("rejects non-production API overrides", () => {
64 expect(requireProductionApiV3Url(undefined)).toBe(
65 "https://backend.composio.dev/api/v3"
66 );
67 expect(requireProductionApiV3Url("https://backend.composio.dev/api/v3/")).toBe(
68 "https://backend.composio.dev/api/v3"
69 );
70 expect(() =>
71 requireProductionApiV3Url("https://staging-backend.composio.dev/api/v3")
72 ).toThrow("Published docs data must be generated from");
73 });
74
75 test("fetches the changelog from the fixed production endpoint", async () => {
76 const requests: string[] = [];
77 const fetcher: ToolkitVersionFetcher = async (url, options) => {
78 requests.push(url);
79 expect(new Headers(options?.headers).get("x-api-key")).toBe("test-key");
80 return Response.json({
81 items: [{ slug: "GMAIL", versions: [{ version: "20260702_01" }] }],
82 });
83 };
84
85 const versions = await fetchProductionToolkitVersions("test-key", fetcher);
86
87 expect(requests).toEqual([
88 "https://backend.composio.dev/api/v3.1/toolkits/changelog",
89 ]);
90 expect(versions.get("gmail")).toBe("20260702_01");
91 });
92
93 test("rejects malformed changelog data before rewriting the catalog", async () => {
94 const fetcher: ToolkitVersionFetcher = async () => Response.json({});
95
96 expect(fetchProductionToolkitVersions("test-key", fetcher)).rejects.toThrow(
97 "missing an items array"
98 );
99 });
100
101 test("skips malformed changelog entries instead of failing the run", async () => {
102 const fetcher: ToolkitVersionFetcher = async () =>
103 Response.json({
104 items: [
105 null,
106 { slug: 7, versions: [{ version: "20260101_01" }] },
107 { slug: "SLACK", versions: "none" },
108 { slug: "NOTION", versions: [{ version: 3 }] },
109 { slug: "GMAIL", versions: [{ version: "20260702_01" }] },
110 ],
111 });
112
113 const versions = await fetchProductionToolkitVersions("test-key", fetcher);
114
115 expect([...versions]).toEqual([["gmail", "20260702_01"]]);
116 });
117
118 test("rejects an empty production version map before rewriting the catalog", async () => {
119 const fetcher: ToolkitVersionFetcher = async () => Response.json({ items: [] });
120
121 expect(fetchProductionToolkitVersions("test-key", fetcher)).rejects.toThrow(
122 "contains no toolkit versions"
123 );
124 });
125
126 test("uses null when a toolkit is absent from the production changelog", () => {
127 const toolkits = [
128 { slug: "gmail", version: "stale" },
129 { slug: "staging-only", version: "stale" },
130 ];
131
132 expect(
133 applyToolkitVersions(toolkits, new Map([["gmail", "20260702_01"]]))
134 ).toEqual({ matched: 1, missing: 1 });
135 expect(toolkits).toEqual([
136 { slug: "gmail", version: "20260702_01" },
137 { slug: "staging-only", version: null },
138 ]);
139 });
140 });
141
142 async function loadToolkits(): Promise<Toolkit[]> {
143 const raw = await readFile(join(DATA_DIR, "toolkits.json"), "utf-8");
144 return JSON.parse(raw) as Toolkit[];
145 }
146
147 describe("Toolkit version distribution smoke test", () => {
148 test("every non-null version is a valid date-stamped version", async () => {
149 const toolkits = await loadToolkits();
150 const offenders = toolkits
151 .filter((t) => t.version != null && !VERSION_RE.test(t.version))
152 .map((t) => `${t.slug}: ${t.version}`);
153 expect(offenders).toEqual([]);
154 });
155
156 test("versions are diverse and no single one dominates the catalog", async () => {
157 const toolkits = await loadToolkits();
158 const versioned = toolkits
159 .map((t) => t.version)
160 .filter((v): v is string => v != null);
161
162 // Guard needs a meaningful sample; the catalog is ~1k toolkits.
163 expect(versioned.length).toBeGreaterThan(100);
164
165 const counts = new Map<string, number>();
166 for (const v of versioned) counts.set(v, (counts.get(v) ?? 0) + 1);
167
168 const topCount = Math.max(...counts.values());
169 const share = topCount / versioned.length;
170
171 // A staging whole-catalog bump collapses both metrics at once: one version
172 // covers ~all toolkits, and the distinct count drops to a handful.
173 expect(counts.size).toBeGreaterThanOrEqual(MIN_DISTINCT_VERSIONS);
174 expect(share).toBeLessThan(MAX_SINGLE_VERSION_SHARE);
175 });
176 });