Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | import type { AlgoliaDocsRecord } from '@/lib/search-index'; |
| 2 | import { |
| 3 | KNOWLEDGE_SOURCE_LABELS, |
| 4 | type KnowledgeSourceType, |
| 5 | type ProductAreaSlug, |
| 6 | } from './types'; |
| 7 | |
| 8 | export type KnowledgeFilter = |
| 9 | | 'all' |
| 10 | | 'docs' |
| 11 | | 'kb' |
| 12 | | 'oauth' |
| 13 | | 'toolkit' |
| 14 | | 'toolkits' |
| 15 | | 'reference'; |
| 16 | |
| 17 | export interface KnowledgeFilterDefinition { |
| 18 | value: KnowledgeFilter; |
| 19 | label: string; |
| 20 | sourceTypes: KnowledgeSourceType[]; |
| 21 | } |
| 22 | |
| 23 | export interface KnowledgeSearchResult { |
| 24 | objectID: string; |
| 25 | title: string; |
| 26 | excerpt: string; |
| 27 | canonicalUrl: string; |
| 28 | sourceType: KnowledgeSourceType; |
| 29 | sourceLabel: string; |
| 30 | breadcrumbs: string[]; |
| 31 | productAreas: ProductAreaSlug[]; |
| 32 | toolkitSlugs: string[]; |
| 33 | lastVerifiedAt: string | null; |
| 34 | section?: string | null; |
| 35 | } |
| 36 | |
| 37 | export type KnowledgeRetrievalMode = 'hybrid' | 'semantic' | 'keyword'; |
| 38 | export type KnowledgeDegradationReason = |
| 39 | | 'embedding-unavailable' |
| 40 | | 'semantic-artifact-invalid' |
| 41 | | 'semantic-request-failed' |
| 42 | | 'semantic-timeout' |
| 43 | | 'semantic-rate-limited' |
| 44 | | 'semantic-capacity-limited' |
| 45 | | 'keyword-request-failed' |
| 46 | | 'preview-overlay-failed'; |
| 47 | |
| 48 | export interface KnowledgeSearchResponse { |
| 49 | query: string; |
| 50 | filter: KnowledgeFilter; |
| 51 | results: KnowledgeSearchResult[]; |
| 52 | total: number; |
| 53 | mode?: KnowledgeRetrievalMode; |
| 54 | strongMatch?: boolean; |
| 55 | } |
| 56 | |
| 57 | export const KNOWLEDGE_FILTERS: readonly KnowledgeFilterDefinition[] = [ |
| 58 | { value: 'all', label: 'All', sourceTypes: [] }, |
| 59 | { value: 'docs', label: 'Docs', sourceTypes: ['docs'] }, |
| 60 | { value: 'kb', label: 'Knowledge Base', sourceTypes: ['kb'] }, |
| 61 | { value: 'oauth', label: 'OAuth', sourceTypes: ['oauth-guide'] }, |
| 62 | { value: 'toolkits', label: 'Toolkits', sourceTypes: ['toolkit'] }, |
| 63 | { value: 'reference', label: 'Reference', sourceTypes: ['reference', 'legacy'] }, |
| 64 | ] as const; |
| 65 | |
| 66 | const FILTER_BY_VALUE = new Map<KnowledgeFilter, KnowledgeFilterDefinition>([ |
| 67 | ...KNOWLEDGE_FILTERS.map((filter) => [filter.value, filter] as const), |
| 68 | ['toolkit', { value: 'toolkit', label: 'Toolkits', sourceTypes: ['toolkit'] }], |
| 69 | ]); |
| 70 | |
| 71 | export function isKnowledgeFilter(value: string): value is KnowledgeFilter { |
| 72 | return FILTER_BY_VALUE.has(value as KnowledgeFilter); |
| 73 | } |
| 74 | |
| 75 | export function algoliaFacetFilters(filter: KnowledgeFilter): string[][] { |
| 76 | const sourceTypes = FILTER_BY_VALUE.get(filter)?.sourceTypes ?? []; |
| 77 | return sourceTypes.length > 0 |
| 78 | ? [sourceTypes.map((sourceType) => `source_type:${sourceType}`)] |
| 79 | : []; |
| 80 | } |
| 81 | |
| 82 | export function knowledgeFilterIncludesSource( |
| 83 | filter: KnowledgeFilter, |
| 84 | sourceType: KnowledgeSourceType, |
| 85 | ): boolean { |
| 86 | const sourceTypes = FILTER_BY_VALUE.get(filter)?.sourceTypes ?? []; |
| 87 | return sourceTypes.length === 0 || sourceTypes.includes(sourceType); |
| 88 | } |
| 89 | |
| 90 | export function normalizeKnowledgeText(value: string): string { |
| 91 | return value |
| 92 | .toLowerCase() |
| 93 | .replace(/[_-]+/g, ' ') |
| 94 | .replace(/[^a-z0-9]+/g, ' ') |
| 95 | .replace(/\s+/g, ' ') |
| 96 | .trim(); |
| 97 | } |
| 98 | |
| 99 | function textValues(record: AlgoliaDocsRecord): { |
| 100 | title: string; |
| 101 | identity: string[]; |
| 102 | body: string; |
| 103 | all: string; |
| 104 | } { |
| 105 | const title = normalizeKnowledgeText(record.title); |
| 106 | const identity = [ |
| 107 | ...(record.keywords ?? []), |
| 108 | record.slug ?? '', |
| 109 | ...(record.tool_names ?? []), |
| 110 | ...(record.tool_slugs ?? []), |
| 111 | ...record.toolkit_slugs, |
| 112 | ].map(normalizeKnowledgeText).filter(Boolean); |
| 113 | const body = normalizeKnowledgeText([ |
| 114 | record.description, |
| 115 | record.section, |
| 116 | record.content, |
| 117 | ...(record.headings ?? []), |
| 118 | ].filter(Boolean).join(' ')); |
| 119 | return { title, identity, body, all: [title, ...identity, body].join(' ') }; |
| 120 | } |
| 121 | |
| 122 | const LOOSE_MATCH_STOP_WORDS = new Set([ |
| 123 | 'a', 'an', 'and', 'are', 'as', 'at', 'be', 'but', 'by', 'can', 'did', 'do', 'does', |
| 124 | 'for', 'from', 'had', 'has', 'have', 'how', 'i', 'if', 'in', 'into', 'is', 'it', |
| 125 | 'my', 'of', 'on', 'or', 'that', 'the', 'their', 'then', 'they', 'this', 'to', 'was', |
| 126 | 'what', 'when', 'where', 'which', 'who', 'why', 'with', 'you', 'your', |
| 127 | ]); |
| 128 | |
| 129 | function containsToken(value: string, token: string): boolean { |
| 130 | return ` ${value} `.includes(` ${token} `); |
| 131 | } |
| 132 | |
| 133 | function textualTier(record: AlgoliaDocsRecord, normalizedQuery: string): number | null { |
| 134 | const values = textValues(record); |
| 135 | const tokens = normalizedQuery |
| 136 | .split(' ') |
| 137 | .filter(token => token && !LOOSE_MATCH_STOP_WORDS.has(token)); |
| 138 | if (values.title === normalizedQuery) return 6; |
| 139 | if (values.identity.some((value) => value === normalizedQuery)) return 5; |
| 140 | if (values.title.includes(normalizedQuery) || values.identity.some((value) => value.includes(normalizedQuery))) { |
| 141 | return 4; |
| 142 | } |
| 143 | if (values.body.includes(normalizedQuery)) return 3; |
| 144 | if (tokens.length === 0) return null; |
| 145 | if (tokens.every((token) => containsToken(values.title, token) || |
| 146 | values.identity.some((value) => containsToken(value, token)))) { |
| 147 | return 2; |
| 148 | } |
| 149 | const matchedTokens = tokens.filter(token => containsToken(values.all, token)).length; |
| 150 | if (matchedTokens === tokens.length) return 1; |
| 151 | const minimumLooseMatches = Math.max(2, Math.ceil(tokens.length / 2)); |
| 152 | if (matchedTokens >= minimumLooseMatches) return matchedTokens / tokens.length; |
| 153 | return null; |
| 154 | } |
| 155 | |
| 156 | export function filterLegacyReferenceRecords<T extends AlgoliaDocsRecord>( |
| 157 | records: T[], |
| 158 | query: string, |
| 159 | filter: KnowledgeFilter, |
| 160 | ): T[] { |
| 161 | if (filter !== 'all' && filter !== 'reference') return records; |
| 162 | |
| 163 | const normalizedQuery = normalizeKnowledgeText(query); |
| 164 | const hasCurrentExactMatch = records.some((record) => { |
| 165 | if (record.source_type !== 'reference') return false; |
| 166 | return (textualTier(record, normalizedQuery) ?? -1) >= 5; |
| 167 | }); |
| 168 | |
| 169 | return records.filter((record) => { |
| 170 | if (record.source_type !== 'legacy') return true; |
| 171 | return !hasCurrentExactMatch && (textualTier(record, normalizedQuery) ?? -1) >= 5; |
| 172 | }); |
| 173 | } |
| 174 | |
| 175 | function decodeHtmlEntities(value: string): string { |
| 176 | const namedEntities: Record<string, string> = { |
| 177 | amp: '&', |
| 178 | apos: "'", |
| 179 | gt: '>', |
| 180 | lt: '<', |
| 181 | nbsp: ' ', |
| 182 | quot: '"', |
| 183 | }; |
| 184 | |
| 185 | return value.replace(/&(#x[\da-f]+|#\d+|amp|apos|gt|lt|nbsp|quot);/gi, (match, entity: string) => { |
| 186 | if (entity.startsWith('#')) { |
| 187 | const hexadecimal = entity[1]?.toLowerCase() === 'x'; |
| 188 | const codePoint = Number.parseInt(entity.slice(hexadecimal ? 2 : 1), hexadecimal ? 16 : 10); |
| 189 | if (Number.isInteger(codePoint) && codePoint >= 0 && codePoint <= 0x10ffff) { |
| 190 | return String.fromCodePoint(codePoint); |
| 191 | } |
| 192 | return match; |
| 193 | } |
| 194 | |
| 195 | return namedEntities[entity.toLowerCase()] ?? match; |
| 196 | }); |
| 197 | } |
| 198 | |
| 199 | const HTML_TAG_NAMES = new Set([ |
| 200 | 'a', 'abbr', 'applet', 'article', 'aside', 'audio', 'b', 'base', 'bgsound', |
| 201 | 'blockquote', 'body', 'br', 'button', 'canvas', 'caption', 'code', 'col', |
| 202 | 'colgroup', 'datalist', |
| 203 | 'dd', 'details', 'dialog', 'div', 'dl', 'dt', 'em', 'embed', 'fieldset', |
| 204 | 'figcaption', 'figure', 'footer', 'form', 'frame', 'frameset', 'h1', 'h2', |
| 205 | 'h3', 'h4', 'h5', 'h6', 'head', 'header', 'hr', 'html', 'i', 'iframe', |
| 206 | 'image', 'img', 'input', 'keygen', 'label', 'li', 'link', 'listing', 'main', |
| 207 | 'mark', 'marquee', 'math', 'meta', 'nav', 'noembed', 'noframes', 'noscript', |
| 208 | 'object', 'ol', 'option', 'p', 'picture', 'plaintext', 'pre', 'script', |
| 209 | 'section', 'select', 'small', 'source', 'span', 'strong', 'style', 'sub', |
| 210 | 'summary', 'sup', 'svg', 'table', 'tbody', 'td', 'template', 'textarea', |
| 211 | 'tfoot', 'th', 'thead', 'time', 'title', 'tr', 'track', 'ul', 'video', 'xmp', |
| 212 | ]); |
| 213 | |
| 214 | function stripHtmlTags(value: string): string { |
| 215 | let plainText = ''; |
| 216 | let cursor = 0; |
| 217 | |
| 218 | while (cursor < value.length) { |
| 219 | if (value[cursor] !== '<') { |
| 220 | plainText += value[cursor]; |
| 221 | cursor += 1; |
| 222 | continue; |
| 223 | } |
| 224 | |
| 225 | const remainder = value.slice(cursor); |
| 226 | const candidate = remainder.match(/^<\/?([A-Za-z][A-Za-z0-9-]*)\b/); |
| 227 | const isDeclaration = /^<[!?]/.test(remainder); |
| 228 | const isKnownTag = HTML_TAG_NAMES.has(candidate?.[1]?.toLowerCase() ?? ''); |
| 229 | const isClosingTag = /^<\/[A-Za-z][A-Za-z0-9-]*\s*>/.test(remainder); |
| 230 | const hasAttributes = /^<[A-Za-z][A-Za-z0-9-]*\s+[^>]*=[^>]*\/?>/.test(remainder); |
| 231 | if (!isDeclaration && !isKnownTag && !isClosingTag && !hasAttributes) { |
| 232 | plainText += '<'; |
| 233 | cursor += 1; |
| 234 | continue; |
| 235 | } |
| 236 | |
| 237 | const closing = value.indexOf('>', cursor + 1); |
| 238 | if (closing < 0) break; |
| 239 | plainText += ' '; |
| 240 | cursor = closing + 1; |
| 241 | } |
| 242 | |
| 243 | return plainText; |
| 244 | } |
| 245 | |
| 246 | export function plainKnowledgeExcerpt(value: string): string { |
| 247 | let plain = value; |
| 248 | for (let pass = 0; pass < 3; pass += 1) { |
| 249 | const next = stripHtmlTags(decodeHtmlEntities(plain)); |
| 250 | if (next === plain) break; |
| 251 | plain = next; |
| 252 | } |
| 253 | |
| 254 | return plain |
| 255 | .replace(/^\s{0,3}#{1,6}\s+/gm, '') |
| 256 | .replace(/!\[([^\]]*)\]\([^)]*\)/g, '$1') |
| 257 | .replace(/\[([^\]]+)\]\([^)]*\)/g, '$1') |
| 258 | .replace(/(\*\*|__)(.*?)\1/g, '$2') |
| 259 | .replace(/[*~`]+/g, '') |
| 260 | .replace(/\s+/g, ' ') |
| 261 | .trim(); |
| 262 | } |
| 263 | |
| 264 | function excerpt(record: AlgoliaDocsRecord): string { |
| 265 | const value = plainKnowledgeExcerpt(record.description || record.content || record.title); |
| 266 | return value.length > 260 ? `${value.slice(0, 257).trimEnd()}…` : value; |
| 267 | } |
| 268 | |
| 269 | export function knowledgeSearchResultFromRecord( |
| 270 | record: AlgoliaDocsRecord, |
| 271 | matchingExcerpt?: string, |
| 272 | ): KnowledgeSearchResult { |
| 273 | const sourceLabel = KNOWLEDGE_SOURCE_LABELS[record.source_type]; |
| 274 | const breadcrumbs = (record.breadcrumbs ?? []).filter( |
| 275 | (breadcrumb, index) => index > 0 || breadcrumb.toLowerCase() !== sourceLabel.toLowerCase(), |
| 276 | ); |
| 277 | return { |
| 278 | objectID: record.objectID, |
| 279 | title: record.title, |
| 280 | excerpt: matchingExcerpt?.trim() |
| 281 | ? plainKnowledgeExcerpt(matchingExcerpt) |
| 282 | : excerpt(record), |
| 283 | canonicalUrl: record.canonical_url || record.url, |
| 284 | sourceType: record.source_type, |
| 285 | sourceLabel, |
| 286 | breadcrumbs, |
| 287 | productAreas: record.product_areas, |
| 288 | toolkitSlugs: record.toolkit_slugs, |
| 289 | lastVerifiedAt: record.last_verified_at, |
| 290 | section: record.section ?? null, |
| 291 | }; |
| 292 | } |
| 293 | |
| 294 | export function searchKnowledgeRecords( |
| 295 | records: AlgoliaDocsRecord[], |
| 296 | request: { query: string; filter: KnowledgeFilter; limit: number }, |
| 297 | ): KnowledgeSearchResponse { |
| 298 | const query = request.query.trim(); |
| 299 | const normalizedQuery = normalizeKnowledgeText(query); |
| 300 | if (!normalizedQuery) return { query: '', filter: request.filter, results: [], total: 0 }; |
| 301 | |
| 302 | const filter = FILTER_BY_VALUE.get(request.filter) ?? FILTER_BY_VALUE.get('all')!; |
| 303 | const candidates = filterLegacyReferenceRecords( |
| 304 | records.filter((record) => |
| 305 | filter.sourceTypes.length === 0 || filter.sourceTypes.includes(record.source_type)), |
| 306 | query, |
| 307 | request.filter, |
| 308 | ); |
| 309 | const scored = candidates.flatMap((record) => { |
| 310 | const tier = textualTier(record, normalizedQuery); |
| 311 | if (tier === null) return []; |
| 312 | return [{ record, tier }]; |
| 313 | }); |
| 314 | |
| 315 | const bestByCanonicalUrl = new Map<string, (typeof scored)[number]>(); |
| 316 | for (const candidate of scored) { |
| 317 | const key = candidate.record.canonical_url || candidate.record.url; |
| 318 | const current = bestByCanonicalUrl.get(key); |
| 319 | if ( |
| 320 | !current || |
| 321 | candidate.tier > current.tier || |
| 322 | (candidate.tier === current.tier && candidate.record.section_rank > current.record.section_rank) |
| 323 | ) { |
| 324 | bestByCanonicalUrl.set(key, candidate); |
| 325 | } |
| 326 | } |
| 327 | |
| 328 | const ranked = [...bestByCanonicalUrl.values()].sort((left, right) => |
| 329 | right.tier - left.tier || |
| 330 | right.record.page_rank - left.record.page_rank || |
| 331 | right.record.toolkit_popularity - left.record.toolkit_popularity || |
| 332 | right.record.section_rank - left.record.section_rank || |
| 333 | left.record.title.localeCompare(right.record.title), |
| 334 | ); |
| 335 | |
| 336 | return { |
| 337 | query, |
| 338 | filter: request.filter, |
| 339 | results: ranked.slice(0, Math.max(0, request.limit)).map(({ record }) => |
| 340 | knowledgeSearchResultFromRecord(record)), |
| 341 | total: ranked.length, |
| 342 | }; |
| 343 | } |