@devries-cameron20 / composio-1 / commits / 35f46c567
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

fix(review): enforce public toolkit fallback policy

jkomyno committed Aug 25, 2026 at 04:13 UTC 35f46c567bf7402d53e011ba0f501b3cccc49192
7 files changed +152 -5
docs/decisions/toolkit-page-availability.md
+2 -1
@@ -36,7 +36,8 @@ block toolkit, OpenAPI, API index, or meta-tool generation.
36
37 - A snapshot hit renders without a production catalog request.
38 - A snapshot miss may issue one production lookup for the validated slug. An
39 - invalid or unknown slug still returns 404.
39 + invalid, excluded, disabled, custom, or unknown slug still returns 404. The
40 + fallback only publishes enabled, Composio-managed toolkits.
41 - Landing, sitemap, search, KB verification, and build behavior remain
42 snapshot-backed.
43 - `/llms.mdx/toolkits/<slug>` parity is deferred. A toolkit available only
docs/lib/public-toolkit-policy.ts new
+9
@@ -0,0 +1,9 @@
1 +const EXCLUDED_PUBLIC_TOOLKIT_SLUGS = new Set(['test_app']);
2 +
3 +export function normalizeToolkitSlug(slug: string): string {
4 + return slug.toLowerCase();
5 +}
6 +
7 +export function isPublicToolkitSlug(slug: string): boolean {
8 + return !EXCLUDED_PUBLIC_TOOLKIT_SLUGS.has(normalizeToolkitSlug(slug));
9 +}
docs/lib/toolkit-api.ts
+39 -1
@@ -1,12 +1,16 @@
1 import { z } from 'zod';
2 +import { isPublicToolkitSlug, normalizeToolkitSlug } from '@/lib/public-toolkit-policy';
3 import { PRODUCTION_API_V3_URL } from '@/scripts/production-api.mjs';
4 import type { AuthConfigDetail, AuthConfigField, Toolkit } from '@/types/toolkit';
5
6 const NEGATIVE_CACHE_TTL_MS = 60_000;
7 const NEGATIVE_CACHE_MAX_ENTRIES = 1_024;
8 +const IN_FLIGHT_LOOKUP_MAX_ENTRIES = 1_024;
9 +const PRODUCTION_REQUEST_TIMEOUT_MS = 15_000;
10 const TOOLKIT_SLUG_PATTERN = /^[a-z0-9][a-z0-9_-]{0,63}$/;
11
12 const negativeCache = new Map<string, number>();
13 +const inFlightLookups = new Map<string, Promise<Toolkit | null>>();
14
15 function filteredArray<T>(schema: z.ZodType<T>) {
16 return z
@@ -69,6 +73,8 @@ const rawToolkitSchema = z
73 .object({
74 slug: z.string().catch(''),
75 name: optionalString,
76 + type: z.enum(['native', 'custom']).optional().catch(undefined),
77 + enabled: z.boolean().optional().catch(undefined),
78 composio_managed_auth_schemes: stringArray,
79 auth_config_details: filteredArray(rawAuthConfigDetailSchema),
80 meta: z
@@ -92,6 +98,8 @@ const rawToolkitSchema = z
98 .catch({
99 slug: '',
100 name: undefined,
101 + type: undefined,
102 + enabled: undefined,
103 composio_managed_auth_schemes: [],
104 auth_config_details: [],
105 meta: {
@@ -217,8 +225,9 @@ export async function fetchToolkitFromProduction(slug: string): Promise<Toolkit
225 return null;
226 }
227
220 - const normalizedSlug = slug.toLowerCase();
228 + const normalizedSlug = normalizeToolkitSlug(slug);
229 if (!TOOLKIT_SLUG_PATTERN.test(normalizedSlug)) return null;
230 + if (!isPublicToolkitSlug(normalizedSlug)) return null;
231
232 const negativeUntil = negativeCache.get(normalizedSlug);
233 if (negativeUntil !== undefined) {
@@ -226,6 +235,27 @@ export async function fetchToolkitFromProduction(slug: string): Promise<Toolkit
235 negativeCache.delete(normalizedSlug);
236 }
237
238 + const inFlightLookup = inFlightLookups.get(normalizedSlug);
239 + if (inFlightLookup) return inFlightLookup;
240 +
241 + const request = fetchAndMapToolkit(normalizedSlug, apiKey);
242 + let trackedRequest: Promise<Toolkit | null>;
243 + trackedRequest = request.finally(() => {
244 + if (inFlightLookups.get(normalizedSlug) === trackedRequest) {
245 + inFlightLookups.delete(normalizedSlug);
246 + }
247 + });
248 +
249 + if (inFlightLookups.size >= IN_FLIGHT_LOOKUP_MAX_ENTRIES) {
250 + const oldestSlug = inFlightLookups.keys().next().value;
251 + if (oldestSlug !== undefined) inFlightLookups.delete(oldestSlug);
252 + }
253 + inFlightLookups.set(normalizedSlug, trackedRequest);
254 +
255 + return trackedRequest;
256 +}
257 +
258 +async function fetchAndMapToolkit(normalizedSlug: string, apiKey: string): Promise<Toolkit | null> {
259 try {
260 const response = await fetch(`${PRODUCTION_API_V3_URL}/toolkits/${normalizedSlug}`, {
261 headers: {
@@ -233,6 +263,7 @@ export async function fetchToolkitFromProduction(slug: string): Promise<Toolkit
263 'x-api-key': apiKey,
264 },
265 next: { revalidate: 3600 },
266 + signal: AbortSignal.timeout(PRODUCTION_REQUEST_TIMEOUT_MS),
267 });
268
269 if (!response.ok) {
@@ -244,6 +275,13 @@ export async function fetchToolkitFromProduction(slug: string): Promise<Toolkit
275 }
276
277 const payload = rawToolkitSchema.parse(await response.json());
278 + if (payload.type !== 'native' || payload.enabled !== true) {
279 + recordNegative(normalizedSlug);
280 + console.warn(
281 + `[Toolkits] Production toolkit lookup rejected non-public toolkit "${normalizedSlug}"`
282 + );
283 + return null;
284 + }
285 return toolkitFromRaw(payload, normalizedSlug);
286 } catch (error) {
287 recordNegative(normalizedSlug);
docs/lib/toolkit-resolution.ts
+3
@@ -1,8 +1,11 @@
1 import { fetchToolkitFromProduction } from '@/lib/toolkit-api';
2 import { getToolkitBySlug } from '@/lib/toolkit-data';
3 +import { isPublicToolkitSlug } from '@/lib/public-toolkit-policy';
4 import type { Toolkit } from '@/types/toolkit';
5
6 export async function resolveToolkit(slug: string): Promise<Toolkit | null> {
7 + if (!isPublicToolkitSlug(slug)) return null;
8 +
9 const snapshotToolkit = await getToolkitBySlug(slug);
10 if (snapshotToolkit) return snapshotToolkit;
11
docs/scripts/generate-toolkits.ts
+2 -3
@@ -13,14 +13,13 @@ import { join } from 'path';
13 import { fetchWithRetry } from './fetch-with-retry';
14 import { requireProductionApiV3Url, stripStagingHosts } from './production-api.mjs';
15 import { applyToolkitVersions, fetchProductionToolkitVersions } from './toolkit-versions';
16 +import { isPublicToolkitSlug } from '../lib/public-toolkit-policy';
17 import { z } from 'zod';
18
19 const API_BASE = requireProductionApiV3Url(process.env.COMPOSIO_API_BASE);
20 const API_KEY = process.env.COMPOSIO_API_KEY;
21
22 const OUTPUT_DIR = join(process.cwd(), 'public/data');
22 -const EXCLUDED_PUBLIC_TOOLKIT_SLUGS = new Set(['test_app']);
23 -
23 interface Tool {
24 slug: string;
25 name: string;
@@ -375,7 +374,7 @@ export function transformToolkit(raw: unknown): Toolkit {
374 }
375
376 export function shouldPublishToolkit(raw: unknown): boolean {
378 - return !EXCLUDED_PUBLIC_TOOLKIT_SLUGS.has(transformToolkit(raw).slug);
377 + return isPublicToolkitSlug(transformToolkit(raw).slug);
378 }
379
380 async function main() {
docs/tests/static/toolkit-api.test.ts
+91
@@ -12,6 +12,8 @@ const originalFetch = globalThis.fetch;
12 const productionPayload = {
13 slug: 'GitHub',
14 name: 'GitHub',
15 + type: 'native',
16 + enabled: true,
17 composio_managed_auth_schemes: ['oauth2', 'API_KEY'],
18 auth_config_details: [
19 {
@@ -118,6 +120,7 @@ describe('fetchToolkitFromProduction', () => {
120 expect(fetcher.mock.calls[0]?.[1]).toEqual({
121 headers: { 'Content-Type': 'application/json', 'x-api-key': 'test-key' },
122 next: { revalidate: 3600 },
123 + signal: expect.any(AbortSignal),
124 });
125 });
126
@@ -151,6 +154,8 @@ describe('fetchToolkitFromProduction', () => {
154 Response.json({
155 slug: snapshot.slug,
156 name: snapshot.name,
157 + type: 'native',
158 + enabled: true,
159 composio_managed_auth_schemes: snapshot.composioManagedAuthSchemes,
160 auth_config_details: snapshot.authConfigDetails,
161 meta: {
@@ -182,6 +187,8 @@ describe('fetchToolkitFromProduction', () => {
187 Response.json({
188 slug: 'LENIENT',
189 name: 'Lenient',
190 + type: 'native',
191 + enabled: true,
192 auth_config_details: [
193 null,
194 'junk',
@@ -233,6 +240,45 @@ describe('fetchToolkitFromProduction', () => {
240 }
241 );
242
243 + test('rejects excluded public toolkit slugs before fetch', async () => {
244 + useApiKey();
245 + const fetcher = mock(async () => Response.json(productionPayload));
246 + globalThis.fetch = fetcher;
247 +
248 + expect(await fetchToolkitFromProduction('TEST_APP')).toBeNull();
249 + expect(fetcher).not.toHaveBeenCalled();
250 + });
251 +
252 + test.each([
253 + ['custom toolkit', { ...productionPayload, slug: 'custom-only', type: 'custom' }],
254 + ['disabled toolkit', { ...productionPayload, slug: 'disabled-only', enabled: false }],
255 + [
256 + 'toolkit without public flags',
257 + { slug: 'missing-public-flags', name: 'Missing public flags' },
258 + ],
259 + ])('rejects a %s returned by the public endpoint', async (_label, payload) => {
260 + useApiKey();
261 + const fetcher = mock(async () => Response.json(payload));
262 + globalThis.fetch = fetcher;
263 + spyOn(console, 'warn').mockImplementation(() => {});
264 +
265 + expect(await fetchToolkitFromProduction(payload.slug)).toBeNull();
266 + expect(fetcher).toHaveBeenCalledTimes(1);
267 + });
268 +
269 + test('bounds the production request to 15 seconds', async () => {
270 + useApiKey();
271 + const controller = new AbortController();
272 + const timeout = spyOn(AbortSignal, 'timeout').mockReturnValue(controller.signal);
273 + const fetcher = mock(async () => Response.json(productionPayload));
274 + globalThis.fetch = fetcher;
275 +
276 + await fetchToolkitFromProduction('github');
277 +
278 + expect(timeout).toHaveBeenCalledWith(15_000);
279 + expect(fetcher.mock.calls[0]?.[1]?.signal).toBe(controller.signal);
280 + });
281 +
282 test('requires an API key and warns without fetching', async () => {
283 delete process.env.COMPOSIO_API_KEY;
284 delete process.env.COMPOSIO_TOOLKIT_LIVE_FALLBACK;
@@ -284,6 +330,51 @@ describe('fetchToolkitFromProduction', () => {
330 expect(warning).toHaveBeenCalledTimes(2);
331 });
332
333 + test('coalesces concurrent identical live misses into one request', async () => {
334 + useApiKey();
335 + let release: (() => void) | undefined;
336 + const fetcher = mock(async () => {
337 + await new Promise<void>(resolve => {
338 + release = resolve;
339 + });
340 + return new Response(null, { status: 404 });
341 + });
342 + globalThis.fetch = fetcher;
343 + spyOn(console, 'warn').mockImplementation(() => {});
344 +
345 + const lookups = Array.from({ length: 20 }, () =>
346 + fetchToolkitFromProduction('concurrent-miss')
347 + );
348 + expect(fetcher).toHaveBeenCalledTimes(1);
349 +
350 + release?.();
351 + expect(await Promise.all(lookups)).toEqual(Array.from({ length: 20 }, () => null));
352 + expect(fetcher).toHaveBeenCalledTimes(1);
353 + });
354 +
355 + test('bounds concurrent lookup bookkeeping', async () => {
356 + useApiKey();
357 + let release: (() => void) | undefined;
358 + const gate = new Promise<void>(resolve => {
359 + release = resolve;
360 + });
361 + const fetcher = mock(async () => {
362 + await gate;
363 + return new Response(null, { status: 404 });
364 + });
365 + globalThis.fetch = fetcher;
366 + spyOn(console, 'warn').mockImplementation(() => {});
367 +
368 + const lookups = Array.from({ length: 1_025 }, (_, index) =>
369 + fetchToolkitFromProduction(`bounded-flight-${index}`)
370 + );
371 + const evictedLookup = fetchToolkitFromProduction('bounded-flight-0');
372 + expect(fetcher).toHaveBeenCalledTimes(1_026);
373 +
374 + release?.();
375 + await Promise.all([...lookups, evictedLookup]);
376 + });
377 +
378 test('bounds negative-cache entries for unique missing slugs', async () => {
379 useApiKey();
380 const fetcher = mock(async () => new Response(null, { status: 404 }));
docs/tests/static/toolkit-fallback.test.ts
+6
@@ -66,4 +66,10 @@ describe('resolveToolkit', () => {
66
67 expect(await resolveToolkit('__definitely-not-a-toolkit__')).toBeNull();
68 });
69 +
70 + test('rejects test_app without reading the snapshot or production', async () => {
71 + expect(await resolveToolkit('TEST_APP')).toBeNull();
72 + expect(getToolkitBySlug).not.toHaveBeenCalled();
73 + expect(fetchToolkitFromProduction).not.toHaveBeenCalled();
74 + });
75 });