Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 03:37 UTC
| 1 | import { readFileSync } from 'node:fs'; |
| 2 | import { join } from 'node:path'; |
| 3 | import { liteClient } from 'algoliasearch/lite'; |
| 4 | import { z } from 'zod'; |
| 5 | import toolkitsData from '@/public/data/toolkits-list.json'; |
| 6 | import { |
| 7 | ALGOLIA_DEFAULT_APP_ID, |
| 8 | ALGOLIA_DEFAULT_INDEX_NAME, |
| 9 | getAlgoliaSearchDocuments, |
| 10 | type AlgoliaDocsRecord, |
| 11 | } from '@/lib/search-index'; |
| 12 | import { isHiddenApiTagUrl } from '@/lib/filter-api-version'; |
| 13 | import { embedTexts, embeddingContentHash } from './embeddings'; |
| 14 | import { |
| 15 | fusePublicKbCandidates, |
| 16 | isStrongLexicalCandidate, |
| 17 | publicKnowledgeCandidateFromSearchRecord, |
| 18 | publicKnowledgeCandidateFromSemantic, |
| 19 | type PublicKnowledgeCandidateRecord, |
| 20 | } from './hybrid-search'; |
| 21 | import { |
| 22 | docsContentHashFromRecords, |
| 23 | rankSemanticCandidates, |
| 24 | semanticRecordFromSearchRecord, |
| 25 | validateSemanticArtifact, |
| 26 | type KnowledgeSemanticArtifact, |
| 27 | } from './semantic-artifact'; |
| 28 | import { |
| 29 | acquireDefaultSemanticSearch, |
| 30 | defaultSemanticTimeoutMs, |
| 31 | type SemanticSearchAdmission, |
| 32 | } from './semantic-protection'; |
| 33 | import { |
| 34 | queueKnowledgeSearchAnalytics, |
| 35 | type KnowledgeSearchAnalyticsEvent, |
| 36 | } from './query-analytics'; |
| 37 | import { classifyKnowledgeRecord } from './metadata'; |
| 38 | import { |
| 39 | algoliaFacetFilters, |
| 40 | knowledgeFilterIncludesSource, |
| 41 | plainKnowledgeExcerpt, |
| 42 | searchKnowledgeRecords, |
| 43 | type KnowledgeDegradationReason, |
| 44 | type KnowledgeFilter, |
| 45 | type KnowledgeRetrievalMode, |
| 46 | type KnowledgeSearchResponse, |
| 47 | type KnowledgeSearchResult, |
| 48 | } from './search'; |
| 49 | import type { KnowledgeSourceType } from './types'; |
| 50 | |
| 51 | const KNOWLEDGE_CACHE_CONTROL = 'public, max-age=30, stale-while-revalidate=300'; |
| 52 | const KNOWLEDGE_UNAVAILABLE_MESSAGE = 'Knowledge search is temporarily unavailable'; |
| 53 | const SEMANTIC_MINIMUM_SIMILARITY = 0.3; |
| 54 | const PUBLISHED_TOOLKIT_CANONICAL_URLS = new Set( |
| 55 | toolkitsData.map(toolkit => `/toolkits/${toolkit.slug}`), |
| 56 | ); |
| 57 | |
| 58 | const knowledgeSourceTypeSchema = z.enum([ |
| 59 | 'docs', |
| 60 | 'kb', |
| 61 | 'oauth-guide', |
| 62 | 'toolkit', |
| 63 | 'example', |
| 64 | 'reference', |
| 65 | 'changelog', |
| 66 | 'legacy', |
| 67 | ]); |
| 68 | const productAreaSchema = z.enum([ |
| 69 | 'authentication-and-connected-accounts', |
| 70 | 'tools-actions-and-execution', |
| 71 | 'triggers-and-workflows', |
| 72 | 'sdk-api-and-mcp', |
| 73 | 'account-billing-and-security', |
| 74 | ]); |
| 75 | const knowledgeIntentSchema = z.enum([ |
| 76 | 'setup', |
| 77 | 'how-to', |
| 78 | 'troubleshooting', |
| 79 | 'limits-policy', |
| 80 | 'known-issue', |
| 81 | 'reference', |
| 82 | ]); |
| 83 | const optionalStringSchema = z.string().optional().catch(undefined); |
| 84 | const optionalStringArraySchema = z.array(z.string()).optional().catch(undefined); |
| 85 | const optionalNumberSchema = z.number().finite().optional().catch(undefined); |
| 86 | const algoliaSearchHitSchema = z.object({ |
| 87 | objectID: z.string().trim().min(1), |
| 88 | title: z.string().trim().min(1), |
| 89 | description: optionalStringSchema, |
| 90 | breadcrumbs: optionalStringArraySchema, |
| 91 | url: optionalStringSchema, |
| 92 | page_id: optionalStringSchema, |
| 93 | section: z.string().nullable().optional().catch(undefined), |
| 94 | section_id: optionalStringSchema, |
| 95 | content: optionalStringSchema, |
| 96 | keywords: optionalStringArraySchema, |
| 97 | slug: optionalStringSchema, |
| 98 | headings: optionalStringArraySchema, |
| 99 | tool_names: optionalStringArraySchema, |
| 100 | tool_slugs: optionalStringArraySchema, |
| 101 | type: optionalStringSchema, |
| 102 | lang: optionalStringSchema, |
| 103 | tags: optionalStringArraySchema, |
| 104 | page_rank: optionalNumberSchema, |
| 105 | toolkit_popularity: optionalNumberSchema, |
| 106 | section_rank: optionalNumberSchema, |
| 107 | position: optionalNumberSchema, |
| 108 | depth: optionalNumberSchema, |
| 109 | source_type: knowledgeSourceTypeSchema.optional().catch(undefined), |
| 110 | canonical_url: optionalStringSchema, |
| 111 | product_areas: z.array(productAreaSchema).optional().catch(undefined), |
| 112 | toolkit_slugs: optionalStringArraySchema, |
| 113 | intents: z.array(knowledgeIntentSchema).optional().catch(undefined), |
| 114 | last_verified_at: z.string().nullable().optional().catch(undefined), |
| 115 | }); |
| 116 | |
| 117 | const legacyTypeSources = new Map<string, KnowledgeSourceType>([ |
| 118 | ['docs', 'docs'], |
| 119 | ['kb', 'kb'], |
| 120 | ['oauth-guide', 'oauth-guide'], |
| 121 | ['toolkits', 'toolkit'], |
| 122 | ['examples', 'example'], |
| 123 | ['reference', 'reference'], |
| 124 | ['api-reference', 'reference'], |
| 125 | ['v3-reference', 'legacy'], |
| 126 | ['changelog', 'changelog'], |
| 127 | ]); |
| 128 | |
| 129 | const defaultTypeBySource: Record<KnowledgeSourceType, string> = { |
| 130 | docs: 'docs', |
| 131 | kb: 'kb', |
| 132 | 'oauth-guide': 'oauth-guide', |
| 133 | toolkit: 'toolkits', |
| 134 | example: 'examples', |
| 135 | reference: 'reference', |
| 136 | changelog: 'changelog', |
| 137 | legacy: 'v3-reference', |
| 138 | }; |
| 139 | |
| 140 | const defaultPageRankBySource: Record<KnowledgeSourceType, number> = { |
| 141 | docs: 2_000, |
| 142 | kb: 1_900, |
| 143 | 'oauth-guide': 1_700, |
| 144 | toolkit: 1_500, |
| 145 | example: 1_300, |
| 146 | reference: 700, |
| 147 | changelog: 350, |
| 148 | legacy: 25, |
| 149 | }; |
| 150 | |
| 151 | function normalizedCanonicalUrl(values: Array<string | undefined>): string | null { |
| 152 | for (const value of values) { |
| 153 | const trimmed = value?.trim(); |
| 154 | if (!trimmed) continue; |
| 155 | |
| 156 | if (trimmed.startsWith('/')) { |
| 157 | const pathname = trimmed.split(/[?#]/, 1)[0]?.replace(/\/+$/, '') || '/'; |
| 158 | return pathname; |
| 159 | } |
| 160 | |
| 161 | try { |
| 162 | const parsed = new URL(trimmed); |
| 163 | if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') continue; |
| 164 | const pathname = parsed.pathname.replace(/\/+$/, '') || '/'; |
| 165 | if (parsed.hostname === 'docs.composio.dev') return pathname; |
| 166 | return `${parsed.origin}${pathname}`; |
| 167 | } catch { |
| 168 | continue; |
| 169 | } |
| 170 | } |
| 171 | return null; |
| 172 | } |
| 173 | |
| 174 | function pathFromCanonicalUrl(canonicalUrl: string): { pathname: string; origin: string | null } { |
| 175 | if (canonicalUrl.startsWith('/')) return { pathname: canonicalUrl, origin: null }; |
| 176 | const parsed = new URL(canonicalUrl); |
| 177 | return { pathname: parsed.pathname, origin: parsed.origin }; |
| 178 | } |
| 179 | |
| 180 | function sourceTypeFromPath(canonicalUrl: string): KnowledgeSourceType | null { |
| 181 | const { pathname, origin } = pathFromCanonicalUrl(canonicalUrl); |
| 182 | if (origin === 'https://composio.dev' && pathname.startsWith('/auth/')) return 'oauth-guide'; |
| 183 | if (origin !== null) return null; |
| 184 | if (pathname === '/docs/changelog' || pathname.startsWith('/docs/changelog/')) return 'changelog'; |
| 185 | if (pathname === '/docs' || pathname.startsWith('/docs/')) return 'docs'; |
| 186 | if (pathname === '/kb' || pathname.startsWith('/kb/')) return 'kb'; |
| 187 | if (pathname === '/toolkits' || pathname.startsWith('/toolkits/')) return 'toolkit'; |
| 188 | if (pathname === '/examples' || pathname.startsWith('/examples/')) return 'example'; |
| 189 | if (pathname === '/reference/v3' || pathname.startsWith('/reference/v3/')) return 'legacy'; |
| 190 | if (pathname === '/reference' || pathname.startsWith('/reference/')) return 'reference'; |
| 191 | return null; |
| 192 | } |
| 193 | |
| 194 | function toolkitSlugsFromPath(canonicalUrl: string, sourceType: KnowledgeSourceType): string[] { |
| 195 | if (sourceType !== 'toolkit') return []; |
| 196 | const slug = pathFromCanonicalUrl(canonicalUrl).pathname.match(/^\/toolkits\/([^/]+)/)?.[1]; |
| 197 | return slug ? [slug] : []; |
| 198 | } |
| 199 | |
| 200 | function slugFromCanonicalUrl(canonicalUrl: string): string { |
| 201 | return pathFromCanonicalUrl(canonicalUrl).pathname |
| 202 | .split(/[\/_-]+/) |
| 203 | .filter(Boolean) |
| 204 | .join(' '); |
| 205 | } |
| 206 | |
| 207 | /** Normalizes current and pre-migration Algolia records at the external boundary. */ |
| 208 | export function normalizeAlgoliaSearchHits(hits: unknown[]): AlgoliaDocsRecord[] { |
| 209 | const records = hits.flatMap((input): AlgoliaDocsRecord[] => { |
| 210 | const parsed = algoliaSearchHitSchema.safeParse(input); |
| 211 | if (!parsed.success) return []; |
| 212 | const hit = parsed.data; |
| 213 | const canonicalUrl = normalizedCanonicalUrl([ |
| 214 | hit.canonical_url, |
| 215 | hit.url, |
| 216 | hit.page_id, |
| 217 | ]); |
| 218 | if (!canonicalUrl || isHiddenApiTagUrl(pathFromCanonicalUrl(canonicalUrl).pathname)) return []; |
| 219 | |
| 220 | const sourceType = hit.source_type ?? |
| 221 | legacyTypeSources.get(hit.type?.trim().toLowerCase() ?? '') ?? |
| 222 | sourceTypeFromPath(canonicalUrl); |
| 223 | if (!sourceType) return []; |
| 224 | if (sourceType === 'toolkit' && !PUBLISHED_TOOLKIT_CANONICAL_URLS.has(canonicalUrl)) return []; |
| 225 | const metadata = classifyKnowledgeRecord({ |
| 226 | sourceType, |
| 227 | canonicalUrl, |
| 228 | productAreas: hit.product_areas, |
| 229 | toolkitSlugs: hit.toolkit_slugs ?? toolkitSlugsFromPath(canonicalUrl, sourceType), |
| 230 | intents: hit.intents, |
| 231 | lastVerifiedAt: hit.last_verified_at, |
| 232 | }); |
| 233 | |
| 234 | return [{ |
| 235 | objectID: hit.objectID, |
| 236 | title: hit.title, |
| 237 | description: hit.description, |
| 238 | breadcrumbs: hit.breadcrumbs ?? [], |
| 239 | url: canonicalUrl, |
| 240 | page_id: canonicalUrl, |
| 241 | section: hit.section ?? undefined, |
| 242 | section_id: hit.section_id, |
| 243 | content: hit.content?.trim() || hit.description?.trim() || hit.title, |
| 244 | keywords: hit.keywords ?? [], |
| 245 | slug: hit.slug?.trim() || slugFromCanonicalUrl(canonicalUrl), |
| 246 | headings: hit.headings ?? [], |
| 247 | tool_names: hit.tool_names ?? [], |
| 248 | tool_slugs: hit.tool_slugs ?? [], |
| 249 | type: hit.type?.trim() || defaultTypeBySource[sourceType], |
| 250 | lang: hit.lang?.trim() || 'en', |
| 251 | tags: hit.tags ?? [], |
| 252 | page_rank: hit.page_rank ?? defaultPageRankBySource[sourceType], |
| 253 | toolkit_popularity: hit.toolkit_popularity ?? 0, |
| 254 | section_rank: hit.section_rank ?? 0, |
| 255 | position: hit.position ?? 0, |
| 256 | depth: hit.depth ?? 0, |
| 257 | ...metadata, |
| 258 | }]; |
| 259 | }); |
| 260 | |
| 261 | if (hits.length > 0 && records.length === 0) throw new Error('keyword-request-failed'); |
| 262 | return records; |
| 263 | } |
| 264 | |
| 265 | export interface KeywordCandidateSearch { |
| 266 | candidates: PublicKnowledgeCandidateRecord[]; |
| 267 | degradedReason?: KnowledgeDegradationReason; |
| 268 | } |
| 269 | |
| 270 | export interface LocalCandidateSearch { |
| 271 | candidates: PublicKnowledgeCandidateRecord[]; |
| 272 | canonicalUrls: string[]; |
| 273 | } |
| 274 | |
| 275 | export interface KnowledgeSearchInput { |
| 276 | query: string; |
| 277 | filter: KnowledgeFilter; |
| 278 | headers?: Headers; |
| 279 | } |
| 280 | |
| 281 | export interface KnowledgeSearchTimings { |
| 282 | totalDurationMs: number; |
| 283 | keywordDurationMs: number; |
| 284 | semanticDurationMs: number | null; |
| 285 | } |
| 286 | |
| 287 | export interface KnowledgeSearchUnavailableResponse { |
| 288 | error: typeof KNOWLEDGE_UNAVAILABLE_MESSAGE; |
| 289 | } |
| 290 | |
| 291 | export interface KnowledgeSearchExecution { |
| 292 | response: KnowledgeSearchResponse | KnowledgeSearchUnavailableResponse; |
| 293 | status: 200 | 503; |
| 294 | cacheControl: string; |
| 295 | timings: KnowledgeSearchTimings; |
| 296 | previewOverlayApplied: boolean; |
| 297 | } |
| 298 | |
| 299 | export interface KnowledgeSearchDependencies { |
| 300 | hybridEnabled: () => boolean; |
| 301 | searchKeywordCandidates: ( |
| 302 | query: string, |
| 303 | filter: KnowledgeFilter, |
| 304 | ) => Promise<KeywordCandidateSearch>; |
| 305 | searchSemanticCandidates: ( |
| 306 | query: string, |
| 307 | options?: { signal: AbortSignal }, |
| 308 | ) => Promise<PublicKnowledgeCandidateRecord[]>; |
| 309 | searchLocalCandidates?: ( |
| 310 | query: string, |
| 311 | filter: KnowledgeFilter, |
| 312 | ) => Promise<PublicKnowledgeCandidateRecord[] | LocalCandidateSearch>; |
| 313 | searchPreviewCandidates?: ( |
| 314 | query: string, |
| 315 | filter: KnowledgeFilter, |
| 316 | ) => Promise<PublicKnowledgeCandidateRecord[] | LocalCandidateSearch>; |
| 317 | previewOverlayEnabled?: () => boolean; |
| 318 | acquireSemanticSearch?: (request: Request) => SemanticSearchAdmission; |
| 319 | semanticTimeoutMs?: () => number; |
| 320 | captureSearch?: (event: KnowledgeSearchAnalyticsEvent) => void; |
| 321 | } |
| 322 | |
| 323 | function filterCandidates( |
| 324 | candidates: PublicKnowledgeCandidateRecord[], |
| 325 | query: string, |
| 326 | filter: KnowledgeFilter, |
| 327 | ): PublicKnowledgeCandidateRecord[] { |
| 328 | const filtered = candidates.filter(candidate => |
| 329 | knowledgeFilterIncludesSource(filter, candidate.sourceType ?? 'kb')); |
| 330 | if (filter !== 'all' && filter !== 'reference') return filtered; |
| 331 | |
| 332 | const currentReferenceHasExactMatch = filtered.some(candidate => |
| 333 | candidate.sourceType === 'reference' && isStrongLexicalCandidate(candidate, query)); |
| 334 | return filtered.filter(candidate => candidate.sourceType !== 'legacy' || ( |
| 335 | !currentReferenceHasExactMatch && isStrongLexicalCandidate(candidate, query) |
| 336 | )); |
| 337 | } |
| 338 | |
| 339 | function overlayCandidates( |
| 340 | shared: PublicKnowledgeCandidateRecord[], |
| 341 | local: PublicKnowledgeCandidateRecord[], |
| 342 | localCanonicalUrls: readonly string[], |
| 343 | ): PublicKnowledgeCandidateRecord[] { |
| 344 | const localByUrl = new Map(local.map(candidate => [candidate.canonicalUrl, candidate])); |
| 345 | const localUrls = new Set(localCanonicalUrls); |
| 346 | const sharedUrls = new Set(shared.map(candidate => candidate.canonicalUrl)); |
| 347 | return [ |
| 348 | ...shared.flatMap(candidate => { |
| 349 | if (!localUrls.has(candidate.canonicalUrl)) return [candidate]; |
| 350 | const replacement = localByUrl.get(candidate.canonicalUrl); |
| 351 | return replacement ? [replacement] : []; |
| 352 | }), |
| 353 | ...local.filter(candidate => !sharedUrls.has(candidate.canonicalUrl)), |
| 354 | ]; |
| 355 | } |
| 356 | |
| 357 | function localCandidateSearch( |
| 358 | result: PublicKnowledgeCandidateRecord[] | LocalCandidateSearch, |
| 359 | ): LocalCandidateSearch { |
| 360 | return Array.isArray(result) |
| 361 | ? { candidates: result, canonicalUrls: result.map(candidate => candidate.canonicalUrl) } |
| 362 | : result; |
| 363 | } |
| 364 | |
| 365 | function excerpt(value: string): string { |
| 366 | const plain = plainKnowledgeExcerpt(value); |
| 367 | return plain.length > 260 ? `${plain.slice(0, 257).trimEnd()}…` : plain; |
| 368 | } |
| 369 | |
| 370 | function resultFromCandidate(record: PublicKnowledgeCandidateRecord): KnowledgeSearchResult { |
| 371 | const sourceType = record.sourceType ?? 'kb'; |
| 372 | const sourceLabel = record.sourceLabel ?? 'Knowledge Base'; |
| 373 | return { |
| 374 | objectID: record.objectID, |
| 375 | title: record.title, |
| 376 | section: record.section, |
| 377 | excerpt: excerpt(record.content || record.description || record.title), |
| 378 | canonicalUrl: record.canonicalUrl, |
| 379 | sourceType, |
| 380 | sourceLabel, |
| 381 | breadcrumbs: record.breadcrumbs.filter( |
| 382 | (breadcrumb, index) => index > 0 || breadcrumb.toLowerCase() !== sourceLabel.toLowerCase(), |
| 383 | ), |
| 384 | productAreas: record.productAreas, |
| 385 | toolkitSlugs: record.toolkitSlugs, |
| 386 | lastVerifiedAt: record.lastVerifiedAt, |
| 387 | }; |
| 388 | } |
| 389 | |
| 390 | function keywordResults( |
| 391 | candidates: PublicKnowledgeCandidateRecord[], |
| 392 | ): KnowledgeSearchResult[] { |
| 393 | const bestByPage = new Map<string, PublicKnowledgeCandidateRecord>(); |
| 394 | for (const candidate of candidates) { |
| 395 | const page = candidate.canonicalUrl.split('#', 1)[0] ?? candidate.canonicalUrl; |
| 396 | if (!bestByPage.has(page)) bestByPage.set(page, candidate); |
| 397 | } |
| 398 | return [...bestByPage.values()].slice(0, 20).map(resultFromCandidate); |
| 399 | } |
| 400 | |
| 401 | function strongKeywordResults( |
| 402 | candidates: PublicKnowledgeCandidateRecord[], |
| 403 | query: string, |
| 404 | ): KnowledgeSearchResult[] { |
| 405 | const strongCandidates: PublicKnowledgeCandidateRecord[] = []; |
| 406 | const otherCandidates: PublicKnowledgeCandidateRecord[] = []; |
| 407 | for (const candidate of candidates) { |
| 408 | (isStrongLexicalCandidate(candidate, query) ? strongCandidates : otherCandidates).push(candidate); |
| 409 | } |
| 410 | return keywordResults([...strongCandidates, ...otherCandidates]); |
| 411 | } |
| 412 | |
| 413 | async function defaultKeywordSearch( |
| 414 | query: string, |
| 415 | filter: KnowledgeFilter, |
| 416 | ): Promise<KeywordCandidateSearch> { |
| 417 | const appId = process.env.NEXT_PUBLIC_ALGOLIA_APP_ID ?? ALGOLIA_DEFAULT_APP_ID; |
| 418 | const searchApiKey = process.env.NEXT_PUBLIC_ALGOLIA_SEARCH_API_KEY; |
| 419 | const indexName = process.env.NEXT_PUBLIC_ALGOLIA_INDEX_NAME ?? ALGOLIA_DEFAULT_INDEX_NAME; |
| 420 | if (!appId || !searchApiKey || !indexName) throw new Error('keyword-request-failed'); |
| 421 | |
| 422 | try { |
| 423 | const client = liteClient(appId, searchApiKey); |
| 424 | const response = await client.searchForHits<AlgoliaDocsRecord>({ |
| 425 | requests: [{ |
| 426 | type: 'default', |
| 427 | indexName, |
| 428 | query, |
| 429 | distinct: false, |
| 430 | hitsPerPage: 50, |
| 431 | facetFilters: algoliaFacetFilters(filter), |
| 432 | attributesToHighlight: [], |
| 433 | attributesToSnippet: [], |
| 434 | }], |
| 435 | }); |
| 436 | const records = normalizeAlgoliaSearchHits(response.results[0]?.hits ?? []); |
| 437 | return { candidates: records.map(publicKnowledgeCandidateFromSearchRecord) }; |
| 438 | } catch { |
| 439 | throw new Error('keyword-request-failed'); |
| 440 | } |
| 441 | } |
| 442 | |
| 443 | interface LocalPublicIndex { |
| 444 | records: AlgoliaDocsRecord[]; |
| 445 | candidateByObjectID: Map<string, PublicKnowledgeCandidateRecord>; |
| 446 | canonicalUrls: string[]; |
| 447 | } |
| 448 | |
| 449 | let localPublicIndex: Promise<LocalPublicIndex> | null = null; |
| 450 | |
| 451 | function loadLocalPublicIndex(): Promise<LocalPublicIndex> { |
| 452 | localPublicIndex ??= getAlgoliaSearchDocuments().then(records => { |
| 453 | const candidates = records.map(publicKnowledgeCandidateFromSearchRecord); |
| 454 | return { |
| 455 | records, |
| 456 | candidateByObjectID: new Map(candidates.map(candidate => [candidate.objectID, candidate])), |
| 457 | canonicalUrls: [...new Set(candidates.map(candidate => candidate.canonicalUrl))], |
| 458 | }; |
| 459 | }); |
| 460 | return localPublicIndex; |
| 461 | } |
| 462 | |
| 463 | async function defaultLocalSearch( |
| 464 | query: string, |
| 465 | filter: KnowledgeFilter, |
| 466 | ): Promise<LocalCandidateSearch> { |
| 467 | const index = await loadLocalPublicIndex(); |
| 468 | const candidates = searchKnowledgeRecords( |
| 469 | index.records, |
| 470 | { query, filter, limit: 50 }, |
| 471 | ).results.flatMap(result => { |
| 472 | const candidate = index.candidateByObjectID.get(result.objectID); |
| 473 | return candidate ? [candidate] : []; |
| 474 | }); |
| 475 | return { candidates, canonicalUrls: index.canonicalUrls }; |
| 476 | } |
| 477 | |
| 478 | async function defaultPreviewSearch( |
| 479 | query: string, |
| 480 | filter: KnowledgeFilter, |
| 481 | ): Promise<LocalCandidateSearch> { |
| 482 | const index = await loadLocalPublicIndex(); |
| 483 | const records = index.records.filter(record => |
| 484 | record.source_type === 'docs' || record.source_type === 'kb'); |
| 485 | const candidates = searchKnowledgeRecords( |
| 486 | records, |
| 487 | { query, filter, limit: 50 }, |
| 488 | ).results.flatMap(result => { |
| 489 | const candidate = index.candidateByObjectID.get(result.objectID); |
| 490 | return candidate ? [candidate] : []; |
| 491 | }); |
| 492 | return { |
| 493 | candidates, |
| 494 | canonicalUrls: [...new Set(records.map(record => record.canonical_url))], |
| 495 | }; |
| 496 | } |
| 497 | |
| 498 | let semanticArtifact: KnowledgeSemanticArtifact | null = null; |
| 499 | |
| 500 | async function loadSemanticArtifact(): Promise<KnowledgeSemanticArtifact> { |
| 501 | if (semanticArtifact) return semanticArtifact; |
| 502 | const artifact = JSON.parse( |
| 503 | readFileSync(join(process.cwd(), 'kb', 'semantic-index.json'), 'utf8'), |
| 504 | ) as KnowledgeSemanticArtifact; |
| 505 | const records = (await getAlgoliaSearchDocuments()) |
| 506 | .filter(record => record.source_type === 'docs' || record.source_type === 'kb'); |
| 507 | const semanticRecords = records.map(semanticRecordFromSearchRecord); |
| 508 | const contentHashes = new Map( |
| 509 | records.map(record => [record.objectID, embeddingContentHash(record)]), |
| 510 | ); |
| 511 | semanticArtifact = validateSemanticArtifact(artifact, { |
| 512 | docsContentHash: docsContentHashFromRecords(semanticRecords), |
| 513 | contentHashes, |
| 514 | }); |
| 515 | return semanticArtifact; |
| 516 | } |
| 517 | |
| 518 | async function defaultSemanticSearch( |
| 519 | query: string, |
| 520 | options?: { signal: AbortSignal }, |
| 521 | ): Promise<PublicKnowledgeCandidateRecord[]> { |
| 522 | const apiKey = process.env.OPENAI_API_KEY; |
| 523 | if (!apiKey) throw new Error('embedding-unavailable'); |
| 524 | let artifact: KnowledgeSemanticArtifact; |
| 525 | try { |
| 526 | artifact = await loadSemanticArtifact(); |
| 527 | } catch { |
| 528 | throw new Error('semantic-artifact-invalid'); |
| 529 | } |
| 530 | try { |
| 531 | const [queryVector] = await embedTexts([query], { apiKey, signal: options?.signal }); |
| 532 | if (!queryVector) throw new Error('missing query vector'); |
| 533 | return rankSemanticCandidates(artifact, queryVector, 50, { |
| 534 | minimumSimilarity: SEMANTIC_MINIMUM_SIMILARITY, |
| 535 | }).map(candidate => publicKnowledgeCandidateFromSemantic(candidate.record)); |
| 536 | } catch (error) { |
| 537 | if (options?.signal.aborted) throw error; |
| 538 | throw new Error('semantic-request-failed'); |
| 539 | } |
| 540 | } |
| 541 | |
| 542 | const defaultDependencies: KnowledgeSearchDependencies = { |
| 543 | hybridEnabled: () => process.env.KB_HYBRID_SEARCH_ENABLED === 'true', |
| 544 | searchKeywordCandidates: defaultKeywordSearch, |
| 545 | searchSemanticCandidates: defaultSemanticSearch, |
| 546 | searchLocalCandidates: defaultLocalSearch, |
| 547 | searchPreviewCandidates: defaultPreviewSearch, |
| 548 | previewOverlayEnabled: () => process.env.NODE_ENV === 'development' || process.env.VERCEL_ENV === 'preview', |
| 549 | acquireSemanticSearch: acquireDefaultSemanticSearch, |
| 550 | semanticTimeoutMs: defaultSemanticTimeoutMs, |
| 551 | captureSearch: queueKnowledgeSearchAnalytics, |
| 552 | }; |
| 553 | |
| 554 | function degradationReason( |
| 555 | error: unknown, |
| 556 | fallback: KnowledgeDegradationReason, |
| 557 | ): KnowledgeDegradationReason { |
| 558 | const message = error instanceof Error ? error.message : ''; |
| 559 | const allowed: KnowledgeDegradationReason[] = [ |
| 560 | 'embedding-unavailable', |
| 561 | 'semantic-artifact-invalid', |
| 562 | 'semantic-request-failed', |
| 563 | 'semantic-timeout', |
| 564 | 'semantic-rate-limited', |
| 565 | 'semantic-capacity-limited', |
| 566 | 'keyword-request-failed', |
| 567 | 'preview-overlay-failed', |
| 568 | ]; |
| 569 | return allowed.includes(message as KnowledgeDegradationReason) |
| 570 | ? message as KnowledgeDegradationReason |
| 571 | : fallback; |
| 572 | } |
| 573 | |
| 574 | async function protectedSemanticSearch( |
| 575 | query: string, |
| 576 | headers: Headers | undefined, |
| 577 | dependencies: KnowledgeSearchDependencies, |
| 578 | ): Promise<PublicKnowledgeCandidateRecord[]> { |
| 579 | const request = new Request('http://localhost/api/knowledge-search', { headers }); |
| 580 | const admission = dependencies.acquireSemanticSearch?.(request) |
| 581 | ?? { allowed: true as const, release: () => {} }; |
| 582 | if (!admission.allowed) throw new Error(admission.reason); |
| 583 | |
| 584 | const controller = new AbortController(); |
| 585 | const timeoutMs = dependencies.semanticTimeoutMs?.() ?? defaultSemanticTimeoutMs(); |
| 586 | let timedOut = false; |
| 587 | let timer: ReturnType<typeof setTimeout> | undefined; |
| 588 | const timeout = new Promise<never>((_resolve, reject) => { |
| 589 | timer = setTimeout(() => { |
| 590 | timedOut = true; |
| 591 | controller.abort(); |
| 592 | reject(new Error('semantic-timeout')); |
| 593 | }, timeoutMs); |
| 594 | }); |
| 595 | |
| 596 | try { |
| 597 | return await Promise.race([ |
| 598 | dependencies.searchSemanticCandidates(query, { signal: controller.signal }), |
| 599 | timeout, |
| 600 | ]); |
| 601 | } catch (error) { |
| 602 | if (timedOut) throw new Error('semantic-timeout'); |
| 603 | throw error; |
| 604 | } finally { |
| 605 | if (timer) clearTimeout(timer); |
| 606 | admission.release(); |
| 607 | } |
| 608 | } |
| 609 | |
| 610 | function semanticEligible(filter: KnowledgeFilter): boolean { |
| 611 | return filter === 'all' || filter === 'docs' || filter === 'kb'; |
| 612 | } |
| 613 | |
| 614 | function localEditorialEligible(filter: KnowledgeFilter): boolean { |
| 615 | return filter === 'all' || filter === 'docs' || filter === 'kb'; |
| 616 | } |
| 617 | |
| 618 | export async function searchPublicKnowledge( |
| 619 | input: KnowledgeSearchInput, |
| 620 | dependencies: KnowledgeSearchDependencies = defaultDependencies, |
| 621 | ): Promise<KnowledgeSearchExecution> { |
| 622 | const query = input.query.trim().slice(0, 200); |
| 623 | const startedAt = performance.now(); |
| 624 | let keywordDurationMs = 0; |
| 625 | let semanticDurationMs: number | null = null; |
| 626 | let previewOverlayApplied = false; |
| 627 | let degradationCategory: KnowledgeSearchAnalyticsEvent['degradationCategory'] = null; |
| 628 | |
| 629 | const finish = ( |
| 630 | response: KnowledgeSearchExecution['response'], |
| 631 | status: KnowledgeSearchExecution['status'], |
| 632 | ): KnowledgeSearchExecution => { |
| 633 | const totalDurationMs = performance.now() - startedAt; |
| 634 | const results = 'results' in response ? response.results : []; |
| 635 | const retrievalMode = 'mode' in response |
| 636 | ? response.mode ?? 'keyword' |
| 637 | : 'unavailable'; |
| 638 | const strongMatch = 'strongMatch' in response ? response.strongMatch ?? null : null; |
| 639 | if (degradationCategory) { |
| 640 | console.warn('[kb-search]', JSON.stringify({ |
| 641 | event: 'kb_search_degraded', |
| 642 | reason: degradationCategory, |
| 643 | retrievalMode, |
| 644 | statusCode: status, |
| 645 | })); |
| 646 | } |
| 647 | try { |
| 648 | dependencies.captureSearch?.({ |
| 649 | query, |
| 650 | filter: input.filter, |
| 651 | retrievalMode, |
| 652 | resultCount: results.length, |
| 653 | resultSourceTypes: [...new Set(results.map(result => result.sourceType))], |
| 654 | degradationCategory, |
| 655 | strongMatch, |
| 656 | statusCode: status, |
| 657 | durationMs: totalDurationMs, |
| 658 | keywordDurationMs, |
| 659 | semanticDurationMs, |
| 660 | previewOverlayApplied, |
| 661 | }); |
| 662 | } catch { |
| 663 | console.warn('[kb-search]', JSON.stringify({ event: 'kb_search_analytics_schedule_failed' })); |
| 664 | } |
| 665 | return { |
| 666 | response, |
| 667 | status, |
| 668 | cacheControl: status === 200 ? KNOWLEDGE_CACHE_CONTROL : 'no-store', |
| 669 | timings: { totalDurationMs, keywordDurationMs, semanticDurationMs }, |
| 670 | previewOverlayApplied, |
| 671 | }; |
| 672 | }; |
| 673 | |
| 674 | if (!query) { |
| 675 | keywordDurationMs = performance.now() - startedAt; |
| 676 | return { |
| 677 | response: { query: '', filter: input.filter, results: [], total: 0 }, |
| 678 | status: 200, |
| 679 | cacheControl: KNOWLEDGE_CACHE_CONTROL, |
| 680 | timings: { |
| 681 | totalDurationMs: keywordDurationMs, |
| 682 | keywordDurationMs, |
| 683 | semanticDurationMs: null, |
| 684 | }, |
| 685 | previewOverlayApplied: false, |
| 686 | }; |
| 687 | } |
| 688 | |
| 689 | const keywordStartedAt = performance.now(); |
| 690 | let keywordCandidates: PublicKnowledgeCandidateRecord[] | null = null; |
| 691 | try { |
| 692 | const keyword = await dependencies.searchKeywordCandidates(query, input.filter); |
| 693 | keywordCandidates = filterCandidates(keyword.candidates, query, input.filter); |
| 694 | degradationCategory = keyword.degradedReason ?? null; |
| 695 | } catch (error) { |
| 696 | degradationCategory = degradationReason(error, 'keyword-request-failed'); |
| 697 | } |
| 698 | |
| 699 | if (keywordCandidates && localEditorialEligible(input.filter) && |
| 700 | dependencies.previewOverlayEnabled?.() === true) { |
| 701 | try { |
| 702 | if (!dependencies.searchPreviewCandidates) throw new Error('local-index-unavailable'); |
| 703 | const localSearch = localCandidateSearch( |
| 704 | await dependencies.searchPreviewCandidates(query, input.filter), |
| 705 | ); |
| 706 | const previewCandidates = filterCandidates( |
| 707 | localSearch.candidates, |
| 708 | query, |
| 709 | input.filter, |
| 710 | ); |
| 711 | keywordCandidates = overlayCandidates( |
| 712 | keywordCandidates, |
| 713 | previewCandidates, |
| 714 | localSearch.canonicalUrls, |
| 715 | ); |
| 716 | previewOverlayApplied = true; |
| 717 | } catch { |
| 718 | degradationCategory = 'preview-overlay-failed'; |
| 719 | } |
| 720 | } |
| 721 | |
| 722 | if (!keywordCandidates) { |
| 723 | try { |
| 724 | if (!dependencies.searchLocalCandidates) throw new Error('local-index-unavailable'); |
| 725 | const localSearch = localCandidateSearch( |
| 726 | await dependencies.searchLocalCandidates(query, input.filter), |
| 727 | ); |
| 728 | keywordCandidates = filterCandidates(localSearch.candidates, query, input.filter); |
| 729 | degradationCategory = 'keyword-request-failed'; |
| 730 | } catch { |
| 731 | keywordCandidates = null; |
| 732 | } |
| 733 | } |
| 734 | keywordDurationMs = performance.now() - keywordStartedAt; |
| 735 | |
| 736 | const keywordWindow = keywordCandidates?.slice(0, 20) ?? null; |
| 737 | const hasStrongKeywordCandidate = keywordWindow?.some(candidate => |
| 738 | isStrongLexicalCandidate(candidate, query)) === true; |
| 739 | if (keywordWindow && hasStrongKeywordCandidate) { |
| 740 | const results = strongKeywordResults(keywordCandidates ?? keywordWindow, query); |
| 741 | return finish({ |
| 742 | query, |
| 743 | filter: input.filter, |
| 744 | results, |
| 745 | total: results.length, |
| 746 | mode: 'keyword', |
| 747 | strongMatch: true, |
| 748 | }, 200); |
| 749 | } |
| 750 | |
| 751 | if (!dependencies.hybridEnabled() || !semanticEligible(input.filter)) { |
| 752 | if (!keywordWindow) { |
| 753 | degradationCategory = 'all-retrievers-failed'; |
| 754 | return finish({ error: KNOWLEDGE_UNAVAILABLE_MESSAGE }, 503); |
| 755 | } |
| 756 | const results = keywordResults(keywordCandidates ?? keywordWindow); |
| 757 | return finish({ |
| 758 | query, |
| 759 | filter: input.filter, |
| 760 | results, |
| 761 | total: results.length, |
| 762 | mode: 'keyword', |
| 763 | }, 200); |
| 764 | } |
| 765 | |
| 766 | let semanticCandidates: PublicKnowledgeCandidateRecord[] | null = null; |
| 767 | const semanticStartedAt = performance.now(); |
| 768 | try { |
| 769 | semanticCandidates = filterCandidates( |
| 770 | await protectedSemanticSearch(query, input.headers, dependencies), |
| 771 | query, |
| 772 | input.filter, |
| 773 | ).filter(candidate => candidate.sourceType === 'docs' || candidate.sourceType === 'kb'); |
| 774 | } catch (error) { |
| 775 | const semanticDegradation = degradationReason(error, 'semantic-request-failed'); |
| 776 | const isGuardrail = semanticDegradation === 'semantic-timeout' || |
| 777 | semanticDegradation === 'semantic-rate-limited' || |
| 778 | semanticDegradation === 'semantic-capacity-limited'; |
| 779 | if (keywordWindow && (!degradationCategory || isGuardrail)) { |
| 780 | degradationCategory = semanticDegradation; |
| 781 | } |
| 782 | } finally { |
| 783 | semanticDurationMs = performance.now() - semanticStartedAt; |
| 784 | } |
| 785 | |
| 786 | if (!keywordWindow && !semanticCandidates) { |
| 787 | degradationCategory = 'all-retrievers-failed'; |
| 788 | return finish({ error: KNOWLEDGE_UNAVAILABLE_MESSAGE }, 503); |
| 789 | } |
| 790 | |
| 791 | let mode: KnowledgeRetrievalMode; |
| 792 | if (keywordWindow && semanticCandidates) mode = 'hybrid'; |
| 793 | else if (semanticCandidates) mode = 'semantic'; |
| 794 | else mode = 'keyword'; |
| 795 | const fused = fusePublicKbCandidates({ |
| 796 | query, |
| 797 | keyword: keywordWindow ?? [], |
| 798 | semantic: semanticCandidates ?? [], |
| 799 | limit: 20, |
| 800 | }); |
| 801 | const strongMatch = semanticCandidates |
| 802 | ? semanticCandidates.length > 0 || fused.some(candidate => |
| 803 | candidate.keywordRank !== null && candidate.exactTier > 0) |
| 804 | : undefined; |
| 805 | const results = (strongMatch === false ? [] : fused).map(candidate => |
| 806 | resultFromCandidate(candidate.record)); |
| 807 | return finish({ |
| 808 | query, |
| 809 | filter: input.filter, |
| 810 | results, |
| 811 | total: results.length, |
| 812 | mode, |
| 813 | ...(strongMatch === undefined ? {} : { strongMatch }), |
| 814 | }, 200); |
| 815 | } |