Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 03:37 UTC
| 1 | /** |
| 2 | * Freshness verification for the public KB. |
| 3 | * |
| 4 | * The KB rots in ways a review calendar cannot see: a cited tool slug is renamed |
| 5 | * out of the catalog, a toolkit drops an auth scheme, a provider link 404s. Most |
| 6 | * of those claims are checkable against artifacts this repo already refreshes |
| 7 | * every five hours (`public/data/toolkits.json`, `public/openapi*.json`). |
| 8 | * |
| 9 | * This module turns published guides into a list of mechanical assertions and |
| 10 | * checks them. Prose that cannot be checked this way is exactly the prose that |
| 11 | * still needs a human — which is the point: shrink the human queue to the part |
| 12 | * that actually needs judgement. |
| 13 | */ |
| 14 | |
| 15 | import type { KbGuide, KbManifest } from './types'; |
| 16 | |
| 17 | export type KbVerifySeverity = 'error' | 'warning'; |
| 18 | |
| 19 | export type KbVerifyKind = |
| 20 | | 'unknown-tool-slug' |
| 21 | | 'unknown-toolkit-link' |
| 22 | | 'unknown-toolkit-auth-scheme' |
| 23 | | 'review-window-expired' |
| 24 | | 'review-window-cliff' |
| 25 | | 'verification-stale' |
| 26 | | 'unreachable-link' |
| 27 | | 'unedited-support-prose'; |
| 28 | |
| 29 | /** |
| 30 | * The canonical corpus is written for support agents answering a ticket, not for |
| 31 | * a reader who found the page through search. Publishing requires rewriting into |
| 32 | * direct reader guidance — the first of the required editorial transformations in |
| 33 | * the 2026-07-22 corpus audit. |
| 34 | * |
| 35 | * That rewrite is easy to skip during a bulk import, and the result reads as |
| 36 | * internal notes leaking onto a public site. These patterns catch the mechanical |
| 37 | * tells so the omission surfaces at verification rather than in front of a user. |
| 38 | */ |
| 39 | const SUPPORT_PROSE_PATTERNS: Array<{ label: string; pattern: RegExp }> = [ |
| 40 | { |
| 41 | label: 'addresses a support agent about "the customer" rather than the reader', |
| 42 | pattern: |
| 43 | /\b(advise|ask|tell|remind) the customer\b|\bthe customer (should|needs|must|can)\b|\bif a customer\b|\bdo not tell customers\b|customer-safe wording/i, |
| 44 | }, |
| 45 | { |
| 46 | label: 'contains internal triage or escalation instructions', |
| 47 | pattern: |
| 48 | /\broute the case to (a )?human\b|\broute to (a )?human\b|\broute to support\b|\broute (it|this) as\b|\bescalate\b|\bshould be escalated\b|\bfile (a|the|it on the) (tool|feature|toolkit) request( board)?\b|\bcollect the (following|use case|exact)\b|\bask (them|for the exact)\b/i, |
| 49 | }, |
| 50 | { |
| 51 | label: 'states defect or incident status', |
| 52 | pattern: |
| 53 | /\bincident\b|\bregression\b|\bknown (bug|issue)\b|\bcurrently (broken|failing)\b|\bawaiting a fix\b|\btemporarily (unavailable|disabled)\b|\bsuspended\b/i, |
| 54 | }, |
| 55 | ]; |
| 56 | |
| 57 | export interface KbVerifyFinding { |
| 58 | severity: KbVerifySeverity; |
| 59 | kind: KbVerifyKind; |
| 60 | /** Guide slug, or `''` for manifest-wide findings. */ |
| 61 | guideSlug: string; |
| 62 | detail: string; |
| 63 | evidence: string; |
| 64 | } |
| 65 | |
| 66 | export interface ProductCatalog { |
| 67 | toolkitSlugs: Set<string>; |
| 68 | /** Tool and trigger slugs, uppercased, across every toolkit. */ |
| 69 | toolSlugs: Set<string>; |
| 70 | authSchemesByToolkit: Map<string, Set<string>>; |
| 71 | } |
| 72 | |
| 73 | export interface CatalogToolkit { |
| 74 | slug: string; |
| 75 | authSchemes?: string[] | null; |
| 76 | composioManagedAuthSchemes?: string[] | null; |
| 77 | tools?: Array<{ slug?: string } | string> | null; |
| 78 | triggers?: Array<{ slug?: string } | string> | null; |
| 79 | } |
| 80 | |
| 81 | export interface KbVerifyOptions { |
| 82 | /** Warn when a published guide has not been verified in this many days. */ |
| 83 | staleVerificationDays: number; |
| 84 | /** Warn when this many or more published guides share one `reviewAfter` date. */ |
| 85 | reviewCliffThreshold: number; |
| 86 | /** Resolved external link statuses, keyed by URL. Absent when `--check-links` is off. */ |
| 87 | linkStatuses?: Map<string, number | 'unreachable'>; |
| 88 | } |
| 89 | |
| 90 | export const DEFAULT_VERIFY_OPTIONS: KbVerifyOptions = { |
| 91 | staleVerificationDays: 120, |
| 92 | reviewCliffThreshold: 5, |
| 93 | }; |
| 94 | |
| 95 | /** |
| 96 | * Tokens shaped like `TOOLKIT_SOME_ACTION`. Provider error codes (`NONEXISTENT_VERSION`) |
| 97 | * share this shape, so `isComposioToolSlug` filters by known toolkit prefix. |
| 98 | */ |
| 99 | const TOOL_SLUG_PATTERN = /\b[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+\b/g; |
| 100 | const TOOLKIT_LINK_PATTERN = /\]\(\/toolkits\/([a-z0-9_-]+)\)/g; |
| 101 | /** |
| 102 | * Only markdown link targets are checked. Bare URLs in prose or code samples are |
| 103 | * almost always API hosts and endpoints (`https://api.ahrefs.com/v3`, |
| 104 | * `https://backend.composio.dev/api/v3.1/toolkits/`) where 401/404 is the correct |
| 105 | * response and would produce a permanently red report. |
| 106 | */ |
| 107 | const EXTERNAL_LINK_PATTERN = /\]\((https?:\/\/[^\s)]+)\)/g; |
| 108 | |
| 109 | /** |
| 110 | * Substitution placeholders in code samples — `<COMPOSIO_API_KEY>`, `$TOKEN`, |
| 111 | * `${TOKEN}`, `%TOKEN%`. These share the shape of a tool slug and would otherwise |
| 112 | * be reported as dead tools forever, since no catalog will ever contain them. |
| 113 | */ |
| 114 | const PLACEHOLDER_PATTERNS = [ |
| 115 | /<([A-Z][A-Z0-9_]+)>/g, |
| 116 | /\$\{([A-Z][A-Z0-9_]+)\}/g, |
| 117 | /\$([A-Z][A-Z0-9_]+)\b/g, |
| 118 | /%([A-Z][A-Z0-9_]+)%/g, |
| 119 | ]; |
| 120 | |
| 121 | function countOccurrences(text: string, token: string): number { |
| 122 | return text.split(token).length - 1; |
| 123 | } |
| 124 | |
| 125 | /** |
| 126 | * A token is a placeholder rather than a citation only when *every* appearance is |
| 127 | * wrapped. A doc that names `COMPOSIO_SEARCH_TOOLS` in prose and again inside a |
| 128 | * `<...>` sample is still making a checkable claim. |
| 129 | */ |
| 130 | function placeholderOnlyTokens(text: string): Set<string> { |
| 131 | const wrappedCounts = new Map<string, number>(); |
| 132 | for (const pattern of PLACEHOLDER_PATTERNS) { |
| 133 | for (const match of text.matchAll(pattern)) { |
| 134 | const token = match[1]; |
| 135 | if (!token) continue; |
| 136 | wrappedCounts.set(token, (wrappedCounts.get(token) ?? 0) + 1); |
| 137 | } |
| 138 | } |
| 139 | |
| 140 | const placeholders = new Set<string>(); |
| 141 | for (const [token, wrapped] of wrappedCounts) { |
| 142 | if (countOccurrences(text, token) <= wrapped) placeholders.add(token); |
| 143 | } |
| 144 | return placeholders; |
| 145 | } |
| 146 | |
| 147 | function slugsFrom(entries: Array<{ slug?: string } | string> | null | undefined): string[] { |
| 148 | if (!entries) return []; |
| 149 | return entries |
| 150 | .map((entry) => (typeof entry === 'string' ? entry : entry.slug)) |
| 151 | .filter((slug): slug is string => Boolean(slug)) |
| 152 | .map((slug) => slug.toUpperCase()); |
| 153 | } |
| 154 | |
| 155 | export function buildProductCatalog(toolkits: CatalogToolkit[]): ProductCatalog { |
| 156 | const toolkitSlugs = new Set<string>(); |
| 157 | const toolSlugs = new Set<string>(); |
| 158 | const authSchemesByToolkit = new Map<string, Set<string>>(); |
| 159 | |
| 160 | for (const toolkit of toolkits) { |
| 161 | if (!toolkit.slug) continue; |
| 162 | toolkitSlugs.add(toolkit.slug.toLowerCase()); |
| 163 | for (const slug of [...slugsFrom(toolkit.tools), ...slugsFrom(toolkit.triggers)]) { |
| 164 | toolSlugs.add(slug); |
| 165 | } |
| 166 | authSchemesByToolkit.set( |
| 167 | toolkit.slug.toLowerCase(), |
| 168 | new Set((toolkit.authSchemes ?? []).map((scheme) => scheme.toUpperCase())) |
| 169 | ); |
| 170 | } |
| 171 | |
| 172 | return { toolkitSlugs, toolSlugs, authSchemesByToolkit }; |
| 173 | } |
| 174 | |
| 175 | /** |
| 176 | * A token counts as a Composio tool slug only when some known toolkit slug is its |
| 177 | * prefix. Without this, provider error codes and constants quoted in prose would |
| 178 | * be reported as dead tools on every run and the report would be ignored. |
| 179 | */ |
| 180 | export function isComposioToolSlug(token: string, catalog: ProductCatalog): boolean { |
| 181 | for (const toolkitSlug of catalog.toolkitSlugs) { |
| 182 | if (token.startsWith(`${toolkitSlug.toUpperCase()}_`)) return true; |
| 183 | } |
| 184 | return false; |
| 185 | } |
| 186 | |
| 187 | export function extractToolSlugs(text: string, catalog: ProductCatalog): string[] { |
| 188 | const matches = text.match(TOOL_SLUG_PATTERN) ?? []; |
| 189 | const placeholders = placeholderOnlyTokens(text); |
| 190 | return Array.from( |
| 191 | new Set( |
| 192 | matches.filter((token) => !placeholders.has(token) && isComposioToolSlug(token, catalog)) |
| 193 | ) |
| 194 | ); |
| 195 | } |
| 196 | |
| 197 | export function extractToolkitLinks(text: string): string[] { |
| 198 | const found = new Set<string>(); |
| 199 | for (const match of text.matchAll(TOOLKIT_LINK_PATTERN)) { |
| 200 | if (match[1]) found.add(match[1].toLowerCase()); |
| 201 | } |
| 202 | return Array.from(found); |
| 203 | } |
| 204 | |
| 205 | export function extractExternalLinks(text: string): string[] { |
| 206 | const found = new Set<string>(); |
| 207 | for (const match of text.matchAll(EXTERNAL_LINK_PATTERN)) { |
| 208 | const url = match[1]; |
| 209 | if (url) found.add(url); |
| 210 | } |
| 211 | return Array.from(found); |
| 212 | } |
| 213 | |
| 214 | function daysBetween(from: Date, to: Date): number { |
| 215 | return Math.round((to.valueOf() - from.valueOf()) / 86_400_000); |
| 216 | } |
| 217 | |
| 218 | /** |
| 219 | * Slugs a guide is allowed to cite even though they are absent from the catalog — |
| 220 | * for guides whose whole point is "this identifier no longer exists". Declared in |
| 221 | * the manifest so the exemption is reviewable rather than silent. |
| 222 | */ |
| 223 | function allowedMissingSlugs(guide: KbGuide): Set<string> { |
| 224 | return new Set((guide.verifyIgnoreToolSlugs ?? []).map((slug) => slug.toUpperCase())); |
| 225 | } |
| 226 | |
| 227 | export interface KbVerifyInput { |
| 228 | manifest: KbManifest; |
| 229 | guides: KbGuide[]; |
| 230 | catalog: ProductCatalog; |
| 231 | now: Date; |
| 232 | options?: Partial<KbVerifyOptions>; |
| 233 | } |
| 234 | |
| 235 | export interface KbVerifyReport { |
| 236 | checkedGuides: number; |
| 237 | checkedToolSlugs: number; |
| 238 | checkedToolkitLinks: number; |
| 239 | externalLinks: string[]; |
| 240 | findings: KbVerifyFinding[]; |
| 241 | } |
| 242 | |
| 243 | export function verifyKb(input: KbVerifyInput): KbVerifyReport { |
| 244 | const options = { ...DEFAULT_VERIFY_OPTIONS, ...input.options }; |
| 245 | const published = input.guides.filter((guide) => guide.state === 'published'); |
| 246 | const findings: KbVerifyFinding[] = []; |
| 247 | const externalLinks = new Set<string>(); |
| 248 | let checkedToolSlugs = 0; |
| 249 | let checkedToolkitLinks = 0; |
| 250 | |
| 251 | for (const guide of published) { |
| 252 | const text = guide.body; |
| 253 | const exempt = allowedMissingSlugs(guide); |
| 254 | |
| 255 | for (const slug of extractToolSlugs(text, input.catalog)) { |
| 256 | if (exempt.has(slug)) continue; |
| 257 | checkedToolSlugs += 1; |
| 258 | if (!input.catalog.toolSlugs.has(slug)) { |
| 259 | findings.push({ |
| 260 | severity: 'error', |
| 261 | kind: 'unknown-tool-slug', |
| 262 | guideSlug: guide.slug, |
| 263 | detail: `${slug} is cited but absent from the production toolkit catalog`, |
| 264 | evidence: 'public/data/toolkits.json', |
| 265 | }); |
| 266 | } |
| 267 | } |
| 268 | |
| 269 | for (const { label, pattern } of SUPPORT_PROSE_PATTERNS) { |
| 270 | const match = text.match(pattern); |
| 271 | if (!match) continue; |
| 272 | findings.push({ |
| 273 | severity: 'warning', |
| 274 | kind: 'unedited-support-prose', |
| 275 | guideSlug: guide.slug, |
| 276 | detail: `${label}; rewrite for a reader who arrived from search`, |
| 277 | evidence: `matched: ${match[0]}`, |
| 278 | }); |
| 279 | } |
| 280 | |
| 281 | for (const toolkitSlug of extractToolkitLinks(text)) { |
| 282 | checkedToolkitLinks += 1; |
| 283 | if (!input.catalog.toolkitSlugs.has(toolkitSlug)) { |
| 284 | findings.push({ |
| 285 | severity: 'error', |
| 286 | kind: 'unknown-toolkit-link', |
| 287 | guideSlug: guide.slug, |
| 288 | detail: `links to /toolkits/${toolkitSlug}, which is not in the production catalog`, |
| 289 | evidence: 'public/data/toolkits.json', |
| 290 | }); |
| 291 | } |
| 292 | } |
| 293 | |
| 294 | for (const url of extractExternalLinks(text)) { |
| 295 | externalLinks.add(url); |
| 296 | const status = options.linkStatuses?.get(url); |
| 297 | if (status === undefined) continue; |
| 298 | // 5xx and 429 mean the provider is rate-limiting or bot-blocking the probe, |
| 299 | // not that the page is gone. Reporting those would make the check flaky, and |
| 300 | // a flaky freshness report is one nobody reads. |
| 301 | if (typeof status === 'number' && (status >= 500 || status === 429)) continue; |
| 302 | if (status === 'unreachable' || status >= 400) { |
| 303 | findings.push({ |
| 304 | severity: status === 404 || status === 410 ? 'error' : 'warning', |
| 305 | kind: 'unreachable-link', |
| 306 | guideSlug: guide.slug, |
| 307 | detail: `${url} returned ${status}`, |
| 308 | evidence: url, |
| 309 | }); |
| 310 | } |
| 311 | } |
| 312 | |
| 313 | const reviewAfter = guide.reviewAfter ? new Date(guide.reviewAfter) : null; |
| 314 | // Compare instants, as the catalog build does: rounding to whole days would |
| 315 | // expire a guide from noon UTC on the eve of its deadline. |
| 316 | if (reviewAfter && reviewAfter.valueOf() <= input.now.valueOf()) { |
| 317 | const elapsedDays = Math.floor((input.now.valueOf() - reviewAfter.valueOf()) / 86_400_000); |
| 318 | findings.push({ |
| 319 | severity: 'error', |
| 320 | kind: 'review-window-expired', |
| 321 | guideSlug: guide.slug, |
| 322 | detail: `review window expired ${elapsedDays} day(s) ago`, |
| 323 | evidence: `reviewAfter=${guide.reviewAfter}`, |
| 324 | }); |
| 325 | } |
| 326 | |
| 327 | const lastVerified = guide.lastVerifiedAt ? new Date(guide.lastVerifiedAt) : null; |
| 328 | if (lastVerified && !Number.isNaN(lastVerified.valueOf())) { |
| 329 | const age = daysBetween(lastVerified, input.now); |
| 330 | if (age > options.staleVerificationDays) { |
| 331 | findings.push({ |
| 332 | severity: 'warning', |
| 333 | kind: 'verification-stale', |
| 334 | guideSlug: guide.slug, |
| 335 | detail: `last verified ${age} days ago (threshold ${options.staleVerificationDays})`, |
| 336 | evidence: `lastVerifiedAt=${guide.lastVerifiedAt}`, |
| 337 | }); |
| 338 | } |
| 339 | } |
| 340 | } |
| 341 | |
| 342 | // A cohort of guides sharing one reviewAfter date all expire together. When that |
| 343 | // day arrives the cheap move is bulk-bumping the dates rather than re-verifying, |
| 344 | // which quietly converts the review gate into a rubber stamp. |
| 345 | const byReviewDate = new Map<string, string[]>(); |
| 346 | for (const guide of published) { |
| 347 | if (!guide.reviewAfter) continue; |
| 348 | const bucket = byReviewDate.get(guide.reviewAfter) ?? []; |
| 349 | bucket.push(guide.slug); |
| 350 | byReviewDate.set(guide.reviewAfter, bucket); |
| 351 | } |
| 352 | for (const [date, slugs] of byReviewDate) { |
| 353 | if (slugs.length >= options.reviewCliffThreshold) { |
| 354 | findings.push({ |
| 355 | severity: 'warning', |
| 356 | kind: 'review-window-cliff', |
| 357 | guideSlug: '', |
| 358 | detail: `${slugs.length} published guides all expire on ${date}; stagger these dates`, |
| 359 | evidence: slugs.slice(0, 8).join(', ') + (slugs.length > 8 ? ', …' : ''), |
| 360 | }); |
| 361 | } |
| 362 | } |
| 363 | |
| 364 | return { |
| 365 | checkedGuides: published.length, |
| 366 | checkedToolSlugs, |
| 367 | checkedToolkitLinks, |
| 368 | externalLinks: Array.from(externalLinks).sort(), |
| 369 | findings, |
| 370 | }; |
| 371 | } |
| 372 | |
| 373 | export function renderVerifyReport(report: KbVerifyReport): string { |
| 374 | const errors = report.findings.filter((finding) => finding.severity === 'error'); |
| 375 | const warnings = report.findings.filter((finding) => finding.severity === 'warning'); |
| 376 | const lines: string[] = []; |
| 377 | |
| 378 | lines.push('# KB freshness verification'); |
| 379 | lines.push(''); |
| 380 | lines.push( |
| 381 | `Checked ${report.checkedGuides} published guides — ` + |
| 382 | `${report.checkedToolSlugs} tool-slug claims, ${report.checkedToolkitLinks} toolkit links, ` + |
| 383 | `${report.externalLinks.length} external links.` |
| 384 | ); |
| 385 | lines.push(''); |
| 386 | |
| 387 | if (!report.findings.length) { |
| 388 | lines.push('All mechanical claims verified against the current production catalog.'); |
| 389 | return lines.join('\n'); |
| 390 | } |
| 391 | |
| 392 | for (const [label, group] of [ |
| 393 | ['Errors', errors], |
| 394 | ['Warnings', warnings], |
| 395 | ] as const) { |
| 396 | if (!group.length) continue; |
| 397 | lines.push(`## ${label} (${group.length})`); |
| 398 | lines.push(''); |
| 399 | for (const finding of group) { |
| 400 | const subject = finding.guideSlug || '(manifest)'; |
| 401 | lines.push(`- \`${subject}\` [${finding.kind}] ${finding.detail}`); |
| 402 | lines.push(` - evidence: ${finding.evidence}`); |
| 403 | } |
| 404 | lines.push(''); |
| 405 | } |
| 406 | |
| 407 | return lines.join('\n'); |
| 408 | } |