@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
ts 408 lines 14.5 KB
Raw
1 /**
2 * Freshness verification for the public KB.
3 *
4 * The KB rots in ways a review calendar cannot see: a cited tool slug is renamed
5 * out of the catalog, a toolkit drops an auth scheme, a provider link 404s. Most
6 * of those claims are checkable against artifacts this repo already refreshes
7 * every five hours (`public/data/toolkits.json`, `public/openapi*.json`).
8 *
9 * This module turns published guides into a list of mechanical assertions and
10 * checks them. Prose that cannot be checked this way is exactly the prose that
11 * still needs a human — which is the point: shrink the human queue to the part
12 * that actually needs judgement.
13 */
14
15 import type { KbGuide, KbManifest } from './types';
16
17 export type KbVerifySeverity = 'error' | 'warning';
18
19 export type KbVerifyKind =
20 | 'unknown-tool-slug'
21 | 'unknown-toolkit-link'
22 | 'unknown-toolkit-auth-scheme'
23 | 'review-window-expired'
24 | 'review-window-cliff'
25 | 'verification-stale'
26 | 'unreachable-link'
27 | 'unedited-support-prose';
28
29 /**
30 * The canonical corpus is written for support agents answering a ticket, not for
31 * a reader who found the page through search. Publishing requires rewriting into
32 * direct reader guidance — the first of the required editorial transformations in
33 * the 2026-07-22 corpus audit.
34 *
35 * That rewrite is easy to skip during a bulk import, and the result reads as
36 * internal notes leaking onto a public site. These patterns catch the mechanical
37 * tells so the omission surfaces at verification rather than in front of a user.
38 */
39 const SUPPORT_PROSE_PATTERNS: Array<{ label: string; pattern: RegExp }> = [
40 {
41 label: 'addresses a support agent about "the customer" rather than the reader',
42 pattern:
43 /\b(advise|ask|tell|remind) the customer\b|\bthe customer (should|needs|must|can)\b|\bif a customer\b|\bdo not tell customers\b|customer-safe wording/i,
44 },
45 {
46 label: 'contains internal triage or escalation instructions',
47 pattern:
48 /\broute the case to (a )?human\b|\broute to (a )?human\b|\broute to support\b|\broute (it|this) as\b|\bescalate\b|\bshould be escalated\b|\bfile (a|the|it on the) (tool|feature|toolkit) request( board)?\b|\bcollect the (following|use case|exact)\b|\bask (them|for the exact)\b/i,
49 },
50 {
51 label: 'states defect or incident status',
52 pattern:
53 /\bincident\b|\bregression\b|\bknown (bug|issue)\b|\bcurrently (broken|failing)\b|\bawaiting a fix\b|\btemporarily (unavailable|disabled)\b|\bsuspended\b/i,
54 },
55 ];
56
57 export interface KbVerifyFinding {
58 severity: KbVerifySeverity;
59 kind: KbVerifyKind;
60 /** Guide slug, or `''` for manifest-wide findings. */
61 guideSlug: string;
62 detail: string;
63 evidence: string;
64 }
65
66 export interface ProductCatalog {
67 toolkitSlugs: Set<string>;
68 /** Tool and trigger slugs, uppercased, across every toolkit. */
69 toolSlugs: Set<string>;
70 authSchemesByToolkit: Map<string, Set<string>>;
71 }
72
73 export interface CatalogToolkit {
74 slug: string;
75 authSchemes?: string[] | null;
76 composioManagedAuthSchemes?: string[] | null;
77 tools?: Array<{ slug?: string } | string> | null;
78 triggers?: Array<{ slug?: string } | string> | null;
79 }
80
81 export interface KbVerifyOptions {
82 /** Warn when a published guide has not been verified in this many days. */
83 staleVerificationDays: number;
84 /** Warn when this many or more published guides share one `reviewAfter` date. */
85 reviewCliffThreshold: number;
86 /** Resolved external link statuses, keyed by URL. Absent when `--check-links` is off. */
87 linkStatuses?: Map<string, number | 'unreachable'>;
88 }
89
90 export const DEFAULT_VERIFY_OPTIONS: KbVerifyOptions = {
91 staleVerificationDays: 120,
92 reviewCliffThreshold: 5,
93 };
94
95 /**
96 * Tokens shaped like `TOOLKIT_SOME_ACTION`. Provider error codes (`NONEXISTENT_VERSION`)
97 * share this shape, so `isComposioToolSlug` filters by known toolkit prefix.
98 */
99 const TOOL_SLUG_PATTERN = /\b[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+\b/g;
100 const TOOLKIT_LINK_PATTERN = /\]\(\/toolkits\/([a-z0-9_-]+)\)/g;
101 /**
102 * Only markdown link targets are checked. Bare URLs in prose or code samples are
103 * almost always API hosts and endpoints (`https://api.ahrefs.com/v3`,
104 * `https://backend.composio.dev/api/v3.1/toolkits/`) where 401/404 is the correct
105 * response and would produce a permanently red report.
106 */
107 const EXTERNAL_LINK_PATTERN = /\]\((https?:\/\/[^\s)]+)\)/g;
108
109 /**
110 * Substitution placeholders in code samples — `<COMPOSIO_API_KEY>`, `$TOKEN`,
111 * `${TOKEN}`, `%TOKEN%`. These share the shape of a tool slug and would otherwise
112 * be reported as dead tools forever, since no catalog will ever contain them.
113 */
114 const PLACEHOLDER_PATTERNS = [
115 /<([A-Z][A-Z0-9_]+)>/g,
116 /\$\{([A-Z][A-Z0-9_]+)\}/g,
117 /\$([A-Z][A-Z0-9_]+)\b/g,
118 /%([A-Z][A-Z0-9_]+)%/g,
119 ];
120
121 function countOccurrences(text: string, token: string): number {
122 return text.split(token).length - 1;
123 }
124
125 /**
126 * A token is a placeholder rather than a citation only when *every* appearance is
127 * wrapped. A doc that names `COMPOSIO_SEARCH_TOOLS` in prose and again inside a
128 * `<...>` sample is still making a checkable claim.
129 */
130 function placeholderOnlyTokens(text: string): Set<string> {
131 const wrappedCounts = new Map<string, number>();
132 for (const pattern of PLACEHOLDER_PATTERNS) {
133 for (const match of text.matchAll(pattern)) {
134 const token = match[1];
135 if (!token) continue;
136 wrappedCounts.set(token, (wrappedCounts.get(token) ?? 0) + 1);
137 }
138 }
139
140 const placeholders = new Set<string>();
141 for (const [token, wrapped] of wrappedCounts) {
142 if (countOccurrences(text, token) <= wrapped) placeholders.add(token);
143 }
144 return placeholders;
145 }
146
147 function slugsFrom(entries: Array<{ slug?: string } | string> | null | undefined): string[] {
148 if (!entries) return [];
149 return entries
150 .map((entry) => (typeof entry === 'string' ? entry : entry.slug))
151 .filter((slug): slug is string => Boolean(slug))
152 .map((slug) => slug.toUpperCase());
153 }
154
155 export function buildProductCatalog(toolkits: CatalogToolkit[]): ProductCatalog {
156 const toolkitSlugs = new Set<string>();
157 const toolSlugs = new Set<string>();
158 const authSchemesByToolkit = new Map<string, Set<string>>();
159
160 for (const toolkit of toolkits) {
161 if (!toolkit.slug) continue;
162 toolkitSlugs.add(toolkit.slug.toLowerCase());
163 for (const slug of [...slugsFrom(toolkit.tools), ...slugsFrom(toolkit.triggers)]) {
164 toolSlugs.add(slug);
165 }
166 authSchemesByToolkit.set(
167 toolkit.slug.toLowerCase(),
168 new Set((toolkit.authSchemes ?? []).map((scheme) => scheme.toUpperCase()))
169 );
170 }
171
172 return { toolkitSlugs, toolSlugs, authSchemesByToolkit };
173 }
174
175 /**
176 * A token counts as a Composio tool slug only when some known toolkit slug is its
177 * prefix. Without this, provider error codes and constants quoted in prose would
178 * be reported as dead tools on every run and the report would be ignored.
179 */
180 export function isComposioToolSlug(token: string, catalog: ProductCatalog): boolean {
181 for (const toolkitSlug of catalog.toolkitSlugs) {
182 if (token.startsWith(`${toolkitSlug.toUpperCase()}_`)) return true;
183 }
184 return false;
185 }
186
187 export function extractToolSlugs(text: string, catalog: ProductCatalog): string[] {
188 const matches = text.match(TOOL_SLUG_PATTERN) ?? [];
189 const placeholders = placeholderOnlyTokens(text);
190 return Array.from(
191 new Set(
192 matches.filter((token) => !placeholders.has(token) && isComposioToolSlug(token, catalog))
193 )
194 );
195 }
196
197 export function extractToolkitLinks(text: string): string[] {
198 const found = new Set<string>();
199 for (const match of text.matchAll(TOOLKIT_LINK_PATTERN)) {
200 if (match[1]) found.add(match[1].toLowerCase());
201 }
202 return Array.from(found);
203 }
204
205 export function extractExternalLinks(text: string): string[] {
206 const found = new Set<string>();
207 for (const match of text.matchAll(EXTERNAL_LINK_PATTERN)) {
208 const url = match[1];
209 if (url) found.add(url);
210 }
211 return Array.from(found);
212 }
213
214 function daysBetween(from: Date, to: Date): number {
215 return Math.round((to.valueOf() - from.valueOf()) / 86_400_000);
216 }
217
218 /**
219 * Slugs a guide is allowed to cite even though they are absent from the catalog —
220 * for guides whose whole point is "this identifier no longer exists". Declared in
221 * the manifest so the exemption is reviewable rather than silent.
222 */
223 function allowedMissingSlugs(guide: KbGuide): Set<string> {
224 return new Set((guide.verifyIgnoreToolSlugs ?? []).map((slug) => slug.toUpperCase()));
225 }
226
227 export interface KbVerifyInput {
228 manifest: KbManifest;
229 guides: KbGuide[];
230 catalog: ProductCatalog;
231 now: Date;
232 options?: Partial<KbVerifyOptions>;
233 }
234
235 export interface KbVerifyReport {
236 checkedGuides: number;
237 checkedToolSlugs: number;
238 checkedToolkitLinks: number;
239 externalLinks: string[];
240 findings: KbVerifyFinding[];
241 }
242
243 export function verifyKb(input: KbVerifyInput): KbVerifyReport {
244 const options = { ...DEFAULT_VERIFY_OPTIONS, ...input.options };
245 const published = input.guides.filter((guide) => guide.state === 'published');
246 const findings: KbVerifyFinding[] = [];
247 const externalLinks = new Set<string>();
248 let checkedToolSlugs = 0;
249 let checkedToolkitLinks = 0;
250
251 for (const guide of published) {
252 const text = guide.body;
253 const exempt = allowedMissingSlugs(guide);
254
255 for (const slug of extractToolSlugs(text, input.catalog)) {
256 if (exempt.has(slug)) continue;
257 checkedToolSlugs += 1;
258 if (!input.catalog.toolSlugs.has(slug)) {
259 findings.push({
260 severity: 'error',
261 kind: 'unknown-tool-slug',
262 guideSlug: guide.slug,
263 detail: `${slug} is cited but absent from the production toolkit catalog`,
264 evidence: 'public/data/toolkits.json',
265 });
266 }
267 }
268
269 for (const { label, pattern } of SUPPORT_PROSE_PATTERNS) {
270 const match = text.match(pattern);
271 if (!match) continue;
272 findings.push({
273 severity: 'warning',
274 kind: 'unedited-support-prose',
275 guideSlug: guide.slug,
276 detail: `${label}; rewrite for a reader who arrived from search`,
277 evidence: `matched: ${match[0]}`,
278 });
279 }
280
281 for (const toolkitSlug of extractToolkitLinks(text)) {
282 checkedToolkitLinks += 1;
283 if (!input.catalog.toolkitSlugs.has(toolkitSlug)) {
284 findings.push({
285 severity: 'error',
286 kind: 'unknown-toolkit-link',
287 guideSlug: guide.slug,
288 detail: `links to /toolkits/${toolkitSlug}, which is not in the production catalog`,
289 evidence: 'public/data/toolkits.json',
290 });
291 }
292 }
293
294 for (const url of extractExternalLinks(text)) {
295 externalLinks.add(url);
296 const status = options.linkStatuses?.get(url);
297 if (status === undefined) continue;
298 // 5xx and 429 mean the provider is rate-limiting or bot-blocking the probe,
299 // not that the page is gone. Reporting those would make the check flaky, and
300 // a flaky freshness report is one nobody reads.
301 if (typeof status === 'number' && (status >= 500 || status === 429)) continue;
302 if (status === 'unreachable' || status >= 400) {
303 findings.push({
304 severity: status === 404 || status === 410 ? 'error' : 'warning',
305 kind: 'unreachable-link',
306 guideSlug: guide.slug,
307 detail: `${url} returned ${status}`,
308 evidence: url,
309 });
310 }
311 }
312
313 const reviewAfter = guide.reviewAfter ? new Date(guide.reviewAfter) : null;
314 // Compare instants, as the catalog build does: rounding to whole days would
315 // expire a guide from noon UTC on the eve of its deadline.
316 if (reviewAfter && reviewAfter.valueOf() <= input.now.valueOf()) {
317 const elapsedDays = Math.floor((input.now.valueOf() - reviewAfter.valueOf()) / 86_400_000);
318 findings.push({
319 severity: 'error',
320 kind: 'review-window-expired',
321 guideSlug: guide.slug,
322 detail: `review window expired ${elapsedDays} day(s) ago`,
323 evidence: `reviewAfter=${guide.reviewAfter}`,
324 });
325 }
326
327 const lastVerified = guide.lastVerifiedAt ? new Date(guide.lastVerifiedAt) : null;
328 if (lastVerified && !Number.isNaN(lastVerified.valueOf())) {
329 const age = daysBetween(lastVerified, input.now);
330 if (age > options.staleVerificationDays) {
331 findings.push({
332 severity: 'warning',
333 kind: 'verification-stale',
334 guideSlug: guide.slug,
335 detail: `last verified ${age} days ago (threshold ${options.staleVerificationDays})`,
336 evidence: `lastVerifiedAt=${guide.lastVerifiedAt}`,
337 });
338 }
339 }
340 }
341
342 // A cohort of guides sharing one reviewAfter date all expire together. When that
343 // day arrives the cheap move is bulk-bumping the dates rather than re-verifying,
344 // which quietly converts the review gate into a rubber stamp.
345 const byReviewDate = new Map<string, string[]>();
346 for (const guide of published) {
347 if (!guide.reviewAfter) continue;
348 const bucket = byReviewDate.get(guide.reviewAfter) ?? [];
349 bucket.push(guide.slug);
350 byReviewDate.set(guide.reviewAfter, bucket);
351 }
352 for (const [date, slugs] of byReviewDate) {
353 if (slugs.length >= options.reviewCliffThreshold) {
354 findings.push({
355 severity: 'warning',
356 kind: 'review-window-cliff',
357 guideSlug: '',
358 detail: `${slugs.length} published guides all expire on ${date}; stagger these dates`,
359 evidence: slugs.slice(0, 8).join(', ') + (slugs.length > 8 ? ', …' : ''),
360 });
361 }
362 }
363
364 return {
365 checkedGuides: published.length,
366 checkedToolSlugs,
367 checkedToolkitLinks,
368 externalLinks: Array.from(externalLinks).sort(),
369 findings,
370 };
371 }
372
373 export function renderVerifyReport(report: KbVerifyReport): string {
374 const errors = report.findings.filter((finding) => finding.severity === 'error');
375 const warnings = report.findings.filter((finding) => finding.severity === 'warning');
376 const lines: string[] = [];
377
378 lines.push('# KB freshness verification');
379 lines.push('');
380 lines.push(
381 `Checked ${report.checkedGuides} published guides — ` +
382 `${report.checkedToolSlugs} tool-slug claims, ${report.checkedToolkitLinks} toolkit links, ` +
383 `${report.externalLinks.length} external links.`
384 );
385 lines.push('');
386
387 if (!report.findings.length) {
388 lines.push('All mechanical claims verified against the current production catalog.');
389 return lines.join('\n');
390 }
391
392 for (const [label, group] of [
393 ['Errors', errors],
394 ['Warnings', warnings],
395 ] as const) {
396 if (!group.length) continue;
397 lines.push(`## ${label} (${group.length})`);
398 lines.push('');
399 for (const finding of group) {
400 const subject = finding.guideSlug || '(manifest)';
401 lines.push(`- \`${subject}\` [${finding.kind}] ${finding.detail}`);
402 lines.push(` - evidence: ${finding.evidence}`);
403 }
404 lines.push('');
405 }
406
407 return lines.join('\n');
408 }