Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 03:37 UTC
| 1 | import { extractGuideSections, parsePublicKbDocument } from './source-document'; |
| 2 | import type { KbCatalog, KbGuide, KbManifest, KbSourceDocument } from './types'; |
| 3 | |
| 4 | const PRIVATE_MARKERS = [ |
| 5 | { label: 'Plain thread reference', pattern: /\bT-\d{2,}\b/ }, |
| 6 | { label: 'Plain URL', pattern: /app\.plain\.com/i }, |
| 7 | { label: 'internal Slack URL', pattern: /slack\.com\/archives\//i }, |
| 8 | { label: 'internal Linear URL', pattern: /linear\.app\/composio/i }, |
| 9 | { label: 'signed download URL', pattern: /X-Amz-(?:Signature|Credential)/i }, |
| 10 | { label: 'machine-local path', pattern: /(?:\/Users\/|\/home\/|[A-Za-z]:\\Users\\)/ }, |
| 11 | { label: 'candidate-only knowledge', pattern: /\bcandidate-only\b/i }, |
| 12 | { |
| 13 | label: 'internal-only heading', |
| 14 | pattern: /^#{2,6}\s+(?:Internal|Support checks|Debug checklist|Related Plain refs)\b/im, |
| 15 | }, |
| 16 | ] as const; |
| 17 | |
| 18 | function validDate(value: string | null): Date | null { |
| 19 | if (!value) return null; |
| 20 | const date = new Date(value); |
| 21 | return Number.isNaN(date.valueOf()) ? null : date; |
| 22 | } |
| 23 | |
| 24 | function assertNoPrivateMarkers(content: string, path: string): void { |
| 25 | for (const marker of PRIVATE_MARKERS) { |
| 26 | if (marker.pattern.test(content)) { |
| 27 | throw new Error(`${path} contains ${marker.label}`); |
| 28 | } |
| 29 | } |
| 30 | } |
| 31 | |
| 32 | function articleBodyFor( |
| 33 | slug: string, |
| 34 | articlePath: unknown, |
| 35 | readArticle: ((articlePath: string) => string) | undefined |
| 36 | ): string | null { |
| 37 | if (articlePath === undefined) return null; |
| 38 | |
| 39 | if (typeof articlePath !== 'string') { |
| 40 | throw new Error(`${slug} articlePath must equal ${slug}.md`); |
| 41 | } |
| 42 | if (articlePath.includes('/') || articlePath.includes('\\')) { |
| 43 | throw new Error(`${slug} articlePath must be a flat filename`); |
| 44 | } |
| 45 | if (articlePath !== `${slug}.md`) { |
| 46 | throw new Error(`${slug} articlePath must equal ${slug}.md`); |
| 47 | } |
| 48 | if (!readArticle) { |
| 49 | throw new Error(`${slug} requires an article reader`); |
| 50 | } |
| 51 | |
| 52 | const body = readArticle(articlePath); |
| 53 | if (!body.trim()) throw new Error(`${articlePath} must not be empty`); |
| 54 | if (/^(?:\uFEFF)?---(?:\r?\n|$)/.test(body.trimStart())) { |
| 55 | throw new Error(`${articlePath} must not contain YAML frontmatter`); |
| 56 | } |
| 57 | assertNoPrivateMarkers(body, articlePath); |
| 58 | return body; |
| 59 | } |
| 60 | |
| 61 | export function buildKbCatalog( |
| 62 | manifest: KbManifest, |
| 63 | readSource: (sourcePath: string) => string, |
| 64 | now = new Date(), |
| 65 | readArticle?: (articlePath: string) => string |
| 66 | ): KbCatalog { |
| 67 | if (manifest.schemaVersion !== 2) throw new Error('Unsupported KB manifest schema'); |
| 68 | |
| 69 | for (const topic of manifest.topics) { |
| 70 | assertNoPrivateMarkers(topic.title, `topic ${topic.slug} title`); |
| 71 | assertNoPrivateMarkers(topic.description, `topic ${topic.slug} description`); |
| 72 | } |
| 73 | |
| 74 | const topicSlugs = new Set(manifest.topics.map(topic => topic.slug)); |
| 75 | if (topicSlugs.size !== manifest.topics.length) throw new Error('Duplicate KB topic slug'); |
| 76 | |
| 77 | const claimed = new Set<string>(); |
| 78 | for (const definition of manifest.guides) { |
| 79 | for (const value of [definition.slug, ...definition.aliases]) { |
| 80 | const normalized = value.toLowerCase(); |
| 81 | if (claimed.has(normalized)) throw new Error(`Duplicate KB slug or alias: ${value}`); |
| 82 | claimed.add(normalized); |
| 83 | } |
| 84 | for (const topic of definition.topics) { |
| 85 | if (!topicSlugs.has(topic)) { |
| 86 | throw new Error(`${definition.slug} has unknown topic: ${topic}`); |
| 87 | } |
| 88 | } |
| 89 | } |
| 90 | |
| 91 | const definitionSlugs = new Set(manifest.guides.map(guide => guide.slug)); |
| 92 | const documents = new Map<string, KbSourceDocument>(); |
| 93 | const documentFor = (sourcePath: string): KbSourceDocument => { |
| 94 | const cached = documents.get(sourcePath); |
| 95 | if (cached) return cached; |
| 96 | |
| 97 | const document = parsePublicKbDocument(readSource(sourcePath)); |
| 98 | if (document.metadata.visibility !== 'public') { |
| 99 | throw new Error(`${sourcePath} is not visibility: public`); |
| 100 | } |
| 101 | assertNoPrivateMarkers(document.metadata.title, `${sourcePath} title`); |
| 102 | assertNoPrivateMarkers(document.metadata.description, `${sourcePath} description`); |
| 103 | for (const tag of document.metadata.tags) { |
| 104 | assertNoPrivateMarkers(tag, `${sourcePath} tag`); |
| 105 | } |
| 106 | assertNoPrivateMarkers(document.body, sourcePath); |
| 107 | documents.set(sourcePath, document); |
| 108 | return document; |
| 109 | }; |
| 110 | const guides: KbGuide[] = manifest.guides.map(definition => { |
| 111 | assertNoPrivateMarkers(definition.title, `${definition.slug} title`); |
| 112 | assertNoPrivateMarkers(definition.description, `${definition.slug} description`); |
| 113 | for (const tag of definition.tags) { |
| 114 | assertNoPrivateMarkers(tag, `${definition.slug} tag`); |
| 115 | } |
| 116 | |
| 117 | for (const related of definition.relatedGuides) { |
| 118 | if (!definitionSlugs.has(related)) { |
| 119 | throw new Error(`${definition.slug} has unknown related guide: ${related}`); |
| 120 | } |
| 121 | } |
| 122 | |
| 123 | if (definition.sources.length === 0) { |
| 124 | throw new Error(`${definition.slug} requires at least one source`); |
| 125 | } |
| 126 | const sourceMetadata = definition.sources.map( |
| 127 | source => documentFor(source.sourcePath).metadata |
| 128 | ); |
| 129 | |
| 130 | if (definition.state === 'published') { |
| 131 | if (!validDate(definition.lastVerifiedAt)) { |
| 132 | throw new Error(`${definition.slug}: published content requires lastVerifiedAt`); |
| 133 | } |
| 134 | const reviewAfter = validDate(definition.reviewAfter); |
| 135 | if (!reviewAfter) { |
| 136 | throw new Error(`${definition.slug}: published content requires reviewAfter`); |
| 137 | } |
| 138 | if (definition.freshness === 'time-sensitive' && reviewAfter.valueOf() <= now.valueOf()) { |
| 139 | throw new Error(`${definition.slug}: review window expired`); |
| 140 | } |
| 141 | } |
| 142 | |
| 143 | const sourceBody = extractGuideSections(documentFor, definition.sources); |
| 144 | const articleBody = articleBodyFor(definition.slug, definition.articlePath, readArticle); |
| 145 | |
| 146 | return { |
| 147 | ...definition, |
| 148 | body: articleBody ?? sourceBody, |
| 149 | sourceMetadata, |
| 150 | }; |
| 151 | }); |
| 152 | |
| 153 | return { manifest, topics: manifest.topics, guides }; |
| 154 | } |