@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
ts 815 lines 27.8 KB
Raw
1 import { readFileSync } from 'node:fs';
2 import { join } from 'node:path';
3 import { liteClient } from 'algoliasearch/lite';
4 import { z } from 'zod';
5 import toolkitsData from '@/public/data/toolkits-list.json';
6 import {
7 ALGOLIA_DEFAULT_APP_ID,
8 ALGOLIA_DEFAULT_INDEX_NAME,
9 getAlgoliaSearchDocuments,
10 type AlgoliaDocsRecord,
11 } from '@/lib/search-index';
12 import { isHiddenApiTagUrl } from '@/lib/filter-api-version';
13 import { embedTexts, embeddingContentHash } from './embeddings';
14 import {
15 fusePublicKbCandidates,
16 isStrongLexicalCandidate,
17 publicKnowledgeCandidateFromSearchRecord,
18 publicKnowledgeCandidateFromSemantic,
19 type PublicKnowledgeCandidateRecord,
20 } from './hybrid-search';
21 import {
22 docsContentHashFromRecords,
23 rankSemanticCandidates,
24 semanticRecordFromSearchRecord,
25 validateSemanticArtifact,
26 type KnowledgeSemanticArtifact,
27 } from './semantic-artifact';
28 import {
29 acquireDefaultSemanticSearch,
30 defaultSemanticTimeoutMs,
31 type SemanticSearchAdmission,
32 } from './semantic-protection';
33 import {
34 queueKnowledgeSearchAnalytics,
35 type KnowledgeSearchAnalyticsEvent,
36 } from './query-analytics';
37 import { classifyKnowledgeRecord } from './metadata';
38 import {
39 algoliaFacetFilters,
40 knowledgeFilterIncludesSource,
41 plainKnowledgeExcerpt,
42 searchKnowledgeRecords,
43 type KnowledgeDegradationReason,
44 type KnowledgeFilter,
45 type KnowledgeRetrievalMode,
46 type KnowledgeSearchResponse,
47 type KnowledgeSearchResult,
48 } from './search';
49 import type { KnowledgeSourceType } from './types';
50
51 const KNOWLEDGE_CACHE_CONTROL = 'public, max-age=30, stale-while-revalidate=300';
52 const KNOWLEDGE_UNAVAILABLE_MESSAGE = 'Knowledge search is temporarily unavailable';
53 const SEMANTIC_MINIMUM_SIMILARITY = 0.3;
54 const PUBLISHED_TOOLKIT_CANONICAL_URLS = new Set(
55 toolkitsData.map(toolkit => `/toolkits/${toolkit.slug}`),
56 );
57
58 const knowledgeSourceTypeSchema = z.enum([
59 'docs',
60 'kb',
61 'oauth-guide',
62 'toolkit',
63 'example',
64 'reference',
65 'changelog',
66 'legacy',
67 ]);
68 const productAreaSchema = z.enum([
69 'authentication-and-connected-accounts',
70 'tools-actions-and-execution',
71 'triggers-and-workflows',
72 'sdk-api-and-mcp',
73 'account-billing-and-security',
74 ]);
75 const knowledgeIntentSchema = z.enum([
76 'setup',
77 'how-to',
78 'troubleshooting',
79 'limits-policy',
80 'known-issue',
81 'reference',
82 ]);
83 const optionalStringSchema = z.string().optional().catch(undefined);
84 const optionalStringArraySchema = z.array(z.string()).optional().catch(undefined);
85 const optionalNumberSchema = z.number().finite().optional().catch(undefined);
86 const algoliaSearchHitSchema = z.object({
87 objectID: z.string().trim().min(1),
88 title: z.string().trim().min(1),
89 description: optionalStringSchema,
90 breadcrumbs: optionalStringArraySchema,
91 url: optionalStringSchema,
92 page_id: optionalStringSchema,
93 section: z.string().nullable().optional().catch(undefined),
94 section_id: optionalStringSchema,
95 content: optionalStringSchema,
96 keywords: optionalStringArraySchema,
97 slug: optionalStringSchema,
98 headings: optionalStringArraySchema,
99 tool_names: optionalStringArraySchema,
100 tool_slugs: optionalStringArraySchema,
101 type: optionalStringSchema,
102 lang: optionalStringSchema,
103 tags: optionalStringArraySchema,
104 page_rank: optionalNumberSchema,
105 toolkit_popularity: optionalNumberSchema,
106 section_rank: optionalNumberSchema,
107 position: optionalNumberSchema,
108 depth: optionalNumberSchema,
109 source_type: knowledgeSourceTypeSchema.optional().catch(undefined),
110 canonical_url: optionalStringSchema,
111 product_areas: z.array(productAreaSchema).optional().catch(undefined),
112 toolkit_slugs: optionalStringArraySchema,
113 intents: z.array(knowledgeIntentSchema).optional().catch(undefined),
114 last_verified_at: z.string().nullable().optional().catch(undefined),
115 });
116
117 const legacyTypeSources = new Map<string, KnowledgeSourceType>([
118 ['docs', 'docs'],
119 ['kb', 'kb'],
120 ['oauth-guide', 'oauth-guide'],
121 ['toolkits', 'toolkit'],
122 ['examples', 'example'],
123 ['reference', 'reference'],
124 ['api-reference', 'reference'],
125 ['v3-reference', 'legacy'],
126 ['changelog', 'changelog'],
127 ]);
128
129 const defaultTypeBySource: Record<KnowledgeSourceType, string> = {
130 docs: 'docs',
131 kb: 'kb',
132 'oauth-guide': 'oauth-guide',
133 toolkit: 'toolkits',
134 example: 'examples',
135 reference: 'reference',
136 changelog: 'changelog',
137 legacy: 'v3-reference',
138 };
139
140 const defaultPageRankBySource: Record<KnowledgeSourceType, number> = {
141 docs: 2_000,
142 kb: 1_900,
143 'oauth-guide': 1_700,
144 toolkit: 1_500,
145 example: 1_300,
146 reference: 700,
147 changelog: 350,
148 legacy: 25,
149 };
150
151 function normalizedCanonicalUrl(values: Array<string | undefined>): string | null {
152 for (const value of values) {
153 const trimmed = value?.trim();
154 if (!trimmed) continue;
155
156 if (trimmed.startsWith('/')) {
157 const pathname = trimmed.split(/[?#]/, 1)[0]?.replace(/\/+$/, '') || '/';
158 return pathname;
159 }
160
161 try {
162 const parsed = new URL(trimmed);
163 if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') continue;
164 const pathname = parsed.pathname.replace(/\/+$/, '') || '/';
165 if (parsed.hostname === 'docs.composio.dev') return pathname;
166 return `${parsed.origin}${pathname}`;
167 } catch {
168 continue;
169 }
170 }
171 return null;
172 }
173
174 function pathFromCanonicalUrl(canonicalUrl: string): { pathname: string; origin: string | null } {
175 if (canonicalUrl.startsWith('/')) return { pathname: canonicalUrl, origin: null };
176 const parsed = new URL(canonicalUrl);
177 return { pathname: parsed.pathname, origin: parsed.origin };
178 }
179
180 function sourceTypeFromPath(canonicalUrl: string): KnowledgeSourceType | null {
181 const { pathname, origin } = pathFromCanonicalUrl(canonicalUrl);
182 if (origin === 'https://composio.dev' && pathname.startsWith('/auth/')) return 'oauth-guide';
183 if (origin !== null) return null;
184 if (pathname === '/docs/changelog' || pathname.startsWith('/docs/changelog/')) return 'changelog';
185 if (pathname === '/docs' || pathname.startsWith('/docs/')) return 'docs';
186 if (pathname === '/kb' || pathname.startsWith('/kb/')) return 'kb';
187 if (pathname === '/toolkits' || pathname.startsWith('/toolkits/')) return 'toolkit';
188 if (pathname === '/examples' || pathname.startsWith('/examples/')) return 'example';
189 if (pathname === '/reference/v3' || pathname.startsWith('/reference/v3/')) return 'legacy';
190 if (pathname === '/reference' || pathname.startsWith('/reference/')) return 'reference';
191 return null;
192 }
193
194 function toolkitSlugsFromPath(canonicalUrl: string, sourceType: KnowledgeSourceType): string[] {
195 if (sourceType !== 'toolkit') return [];
196 const slug = pathFromCanonicalUrl(canonicalUrl).pathname.match(/^\/toolkits\/([^/]+)/)?.[1];
197 return slug ? [slug] : [];
198 }
199
200 function slugFromCanonicalUrl(canonicalUrl: string): string {
201 return pathFromCanonicalUrl(canonicalUrl).pathname
202 .split(/[\/_-]+/)
203 .filter(Boolean)
204 .join(' ');
205 }
206
207 /** Normalizes current and pre-migration Algolia records at the external boundary. */
208 export function normalizeAlgoliaSearchHits(hits: unknown[]): AlgoliaDocsRecord[] {
209 const records = hits.flatMap((input): AlgoliaDocsRecord[] => {
210 const parsed = algoliaSearchHitSchema.safeParse(input);
211 if (!parsed.success) return [];
212 const hit = parsed.data;
213 const canonicalUrl = normalizedCanonicalUrl([
214 hit.canonical_url,
215 hit.url,
216 hit.page_id,
217 ]);
218 if (!canonicalUrl || isHiddenApiTagUrl(pathFromCanonicalUrl(canonicalUrl).pathname)) return [];
219
220 const sourceType = hit.source_type ??
221 legacyTypeSources.get(hit.type?.trim().toLowerCase() ?? '') ??
222 sourceTypeFromPath(canonicalUrl);
223 if (!sourceType) return [];
224 if (sourceType === 'toolkit' && !PUBLISHED_TOOLKIT_CANONICAL_URLS.has(canonicalUrl)) return [];
225 const metadata = classifyKnowledgeRecord({
226 sourceType,
227 canonicalUrl,
228 productAreas: hit.product_areas,
229 toolkitSlugs: hit.toolkit_slugs ?? toolkitSlugsFromPath(canonicalUrl, sourceType),
230 intents: hit.intents,
231 lastVerifiedAt: hit.last_verified_at,
232 });
233
234 return [{
235 objectID: hit.objectID,
236 title: hit.title,
237 description: hit.description,
238 breadcrumbs: hit.breadcrumbs ?? [],
239 url: canonicalUrl,
240 page_id: canonicalUrl,
241 section: hit.section ?? undefined,
242 section_id: hit.section_id,
243 content: hit.content?.trim() || hit.description?.trim() || hit.title,
244 keywords: hit.keywords ?? [],
245 slug: hit.slug?.trim() || slugFromCanonicalUrl(canonicalUrl),
246 headings: hit.headings ?? [],
247 tool_names: hit.tool_names ?? [],
248 tool_slugs: hit.tool_slugs ?? [],
249 type: hit.type?.trim() || defaultTypeBySource[sourceType],
250 lang: hit.lang?.trim() || 'en',
251 tags: hit.tags ?? [],
252 page_rank: hit.page_rank ?? defaultPageRankBySource[sourceType],
253 toolkit_popularity: hit.toolkit_popularity ?? 0,
254 section_rank: hit.section_rank ?? 0,
255 position: hit.position ?? 0,
256 depth: hit.depth ?? 0,
257 ...metadata,
258 }];
259 });
260
261 if (hits.length > 0 && records.length === 0) throw new Error('keyword-request-failed');
262 return records;
263 }
264
265 export interface KeywordCandidateSearch {
266 candidates: PublicKnowledgeCandidateRecord[];
267 degradedReason?: KnowledgeDegradationReason;
268 }
269
270 export interface LocalCandidateSearch {
271 candidates: PublicKnowledgeCandidateRecord[];
272 canonicalUrls: string[];
273 }
274
275 export interface KnowledgeSearchInput {
276 query: string;
277 filter: KnowledgeFilter;
278 headers?: Headers;
279 }
280
281 export interface KnowledgeSearchTimings {
282 totalDurationMs: number;
283 keywordDurationMs: number;
284 semanticDurationMs: number | null;
285 }
286
287 export interface KnowledgeSearchUnavailableResponse {
288 error: typeof KNOWLEDGE_UNAVAILABLE_MESSAGE;
289 }
290
291 export interface KnowledgeSearchExecution {
292 response: KnowledgeSearchResponse | KnowledgeSearchUnavailableResponse;
293 status: 200 | 503;
294 cacheControl: string;
295 timings: KnowledgeSearchTimings;
296 previewOverlayApplied: boolean;
297 }
298
299 export interface KnowledgeSearchDependencies {
300 hybridEnabled: () => boolean;
301 searchKeywordCandidates: (
302 query: string,
303 filter: KnowledgeFilter,
304 ) => Promise<KeywordCandidateSearch>;
305 searchSemanticCandidates: (
306 query: string,
307 options?: { signal: AbortSignal },
308 ) => Promise<PublicKnowledgeCandidateRecord[]>;
309 searchLocalCandidates?: (
310 query: string,
311 filter: KnowledgeFilter,
312 ) => Promise<PublicKnowledgeCandidateRecord[] | LocalCandidateSearch>;
313 searchPreviewCandidates?: (
314 query: string,
315 filter: KnowledgeFilter,
316 ) => Promise<PublicKnowledgeCandidateRecord[] | LocalCandidateSearch>;
317 previewOverlayEnabled?: () => boolean;
318 acquireSemanticSearch?: (request: Request) => SemanticSearchAdmission;
319 semanticTimeoutMs?: () => number;
320 captureSearch?: (event: KnowledgeSearchAnalyticsEvent) => void;
321 }
322
323 function filterCandidates(
324 candidates: PublicKnowledgeCandidateRecord[],
325 query: string,
326 filter: KnowledgeFilter,
327 ): PublicKnowledgeCandidateRecord[] {
328 const filtered = candidates.filter(candidate =>
329 knowledgeFilterIncludesSource(filter, candidate.sourceType ?? 'kb'));
330 if (filter !== 'all' && filter !== 'reference') return filtered;
331
332 const currentReferenceHasExactMatch = filtered.some(candidate =>
333 candidate.sourceType === 'reference' && isStrongLexicalCandidate(candidate, query));
334 return filtered.filter(candidate => candidate.sourceType !== 'legacy' || (
335 !currentReferenceHasExactMatch && isStrongLexicalCandidate(candidate, query)
336 ));
337 }
338
339 function overlayCandidates(
340 shared: PublicKnowledgeCandidateRecord[],
341 local: PublicKnowledgeCandidateRecord[],
342 localCanonicalUrls: readonly string[],
343 ): PublicKnowledgeCandidateRecord[] {
344 const localByUrl = new Map(local.map(candidate => [candidate.canonicalUrl, candidate]));
345 const localUrls = new Set(localCanonicalUrls);
346 const sharedUrls = new Set(shared.map(candidate => candidate.canonicalUrl));
347 return [
348 ...shared.flatMap(candidate => {
349 if (!localUrls.has(candidate.canonicalUrl)) return [candidate];
350 const replacement = localByUrl.get(candidate.canonicalUrl);
351 return replacement ? [replacement] : [];
352 }),
353 ...local.filter(candidate => !sharedUrls.has(candidate.canonicalUrl)),
354 ];
355 }
356
357 function localCandidateSearch(
358 result: PublicKnowledgeCandidateRecord[] | LocalCandidateSearch,
359 ): LocalCandidateSearch {
360 return Array.isArray(result)
361 ? { candidates: result, canonicalUrls: result.map(candidate => candidate.canonicalUrl) }
362 : result;
363 }
364
365 function excerpt(value: string): string {
366 const plain = plainKnowledgeExcerpt(value);
367 return plain.length > 260 ? `${plain.slice(0, 257).trimEnd()}…` : plain;
368 }
369
370 function resultFromCandidate(record: PublicKnowledgeCandidateRecord): KnowledgeSearchResult {
371 const sourceType = record.sourceType ?? 'kb';
372 const sourceLabel = record.sourceLabel ?? 'Knowledge Base';
373 return {
374 objectID: record.objectID,
375 title: record.title,
376 section: record.section,
377 excerpt: excerpt(record.content || record.description || record.title),
378 canonicalUrl: record.canonicalUrl,
379 sourceType,
380 sourceLabel,
381 breadcrumbs: record.breadcrumbs.filter(
382 (breadcrumb, index) => index > 0 || breadcrumb.toLowerCase() !== sourceLabel.toLowerCase(),
383 ),
384 productAreas: record.productAreas,
385 toolkitSlugs: record.toolkitSlugs,
386 lastVerifiedAt: record.lastVerifiedAt,
387 };
388 }
389
390 function keywordResults(
391 candidates: PublicKnowledgeCandidateRecord[],
392 ): KnowledgeSearchResult[] {
393 const bestByPage = new Map<string, PublicKnowledgeCandidateRecord>();
394 for (const candidate of candidates) {
395 const page = candidate.canonicalUrl.split('#', 1)[0] ?? candidate.canonicalUrl;
396 if (!bestByPage.has(page)) bestByPage.set(page, candidate);
397 }
398 return [...bestByPage.values()].slice(0, 20).map(resultFromCandidate);
399 }
400
401 function strongKeywordResults(
402 candidates: PublicKnowledgeCandidateRecord[],
403 query: string,
404 ): KnowledgeSearchResult[] {
405 const strongCandidates: PublicKnowledgeCandidateRecord[] = [];
406 const otherCandidates: PublicKnowledgeCandidateRecord[] = [];
407 for (const candidate of candidates) {
408 (isStrongLexicalCandidate(candidate, query) ? strongCandidates : otherCandidates).push(candidate);
409 }
410 return keywordResults([...strongCandidates, ...otherCandidates]);
411 }
412
413 async function defaultKeywordSearch(
414 query: string,
415 filter: KnowledgeFilter,
416 ): Promise<KeywordCandidateSearch> {
417 const appId = process.env.NEXT_PUBLIC_ALGOLIA_APP_ID ?? ALGOLIA_DEFAULT_APP_ID;
418 const searchApiKey = process.env.NEXT_PUBLIC_ALGOLIA_SEARCH_API_KEY;
419 const indexName = process.env.NEXT_PUBLIC_ALGOLIA_INDEX_NAME ?? ALGOLIA_DEFAULT_INDEX_NAME;
420 if (!appId || !searchApiKey || !indexName) throw new Error('keyword-request-failed');
421
422 try {
423 const client = liteClient(appId, searchApiKey);
424 const response = await client.searchForHits<AlgoliaDocsRecord>({
425 requests: [{
426 type: 'default',
427 indexName,
428 query,
429 distinct: false,
430 hitsPerPage: 50,
431 facetFilters: algoliaFacetFilters(filter),
432 attributesToHighlight: [],
433 attributesToSnippet: [],
434 }],
435 });
436 const records = normalizeAlgoliaSearchHits(response.results[0]?.hits ?? []);
437 return { candidates: records.map(publicKnowledgeCandidateFromSearchRecord) };
438 } catch {
439 throw new Error('keyword-request-failed');
440 }
441 }
442
443 interface LocalPublicIndex {
444 records: AlgoliaDocsRecord[];
445 candidateByObjectID: Map<string, PublicKnowledgeCandidateRecord>;
446 canonicalUrls: string[];
447 }
448
449 let localPublicIndex: Promise<LocalPublicIndex> | null = null;
450
451 function loadLocalPublicIndex(): Promise<LocalPublicIndex> {
452 localPublicIndex ??= getAlgoliaSearchDocuments().then(records => {
453 const candidates = records.map(publicKnowledgeCandidateFromSearchRecord);
454 return {
455 records,
456 candidateByObjectID: new Map(candidates.map(candidate => [candidate.objectID, candidate])),
457 canonicalUrls: [...new Set(candidates.map(candidate => candidate.canonicalUrl))],
458 };
459 });
460 return localPublicIndex;
461 }
462
463 async function defaultLocalSearch(
464 query: string,
465 filter: KnowledgeFilter,
466 ): Promise<LocalCandidateSearch> {
467 const index = await loadLocalPublicIndex();
468 const candidates = searchKnowledgeRecords(
469 index.records,
470 { query, filter, limit: 50 },
471 ).results.flatMap(result => {
472 const candidate = index.candidateByObjectID.get(result.objectID);
473 return candidate ? [candidate] : [];
474 });
475 return { candidates, canonicalUrls: index.canonicalUrls };
476 }
477
478 async function defaultPreviewSearch(
479 query: string,
480 filter: KnowledgeFilter,
481 ): Promise<LocalCandidateSearch> {
482 const index = await loadLocalPublicIndex();
483 const records = index.records.filter(record =>
484 record.source_type === 'docs' || record.source_type === 'kb');
485 const candidates = searchKnowledgeRecords(
486 records,
487 { query, filter, limit: 50 },
488 ).results.flatMap(result => {
489 const candidate = index.candidateByObjectID.get(result.objectID);
490 return candidate ? [candidate] : [];
491 });
492 return {
493 candidates,
494 canonicalUrls: [...new Set(records.map(record => record.canonical_url))],
495 };
496 }
497
498 let semanticArtifact: KnowledgeSemanticArtifact | null = null;
499
500 async function loadSemanticArtifact(): Promise<KnowledgeSemanticArtifact> {
501 if (semanticArtifact) return semanticArtifact;
502 const artifact = JSON.parse(
503 readFileSync(join(process.cwd(), 'kb', 'semantic-index.json'), 'utf8'),
504 ) as KnowledgeSemanticArtifact;
505 const records = (await getAlgoliaSearchDocuments())
506 .filter(record => record.source_type === 'docs' || record.source_type === 'kb');
507 const semanticRecords = records.map(semanticRecordFromSearchRecord);
508 const contentHashes = new Map(
509 records.map(record => [record.objectID, embeddingContentHash(record)]),
510 );
511 semanticArtifact = validateSemanticArtifact(artifact, {
512 docsContentHash: docsContentHashFromRecords(semanticRecords),
513 contentHashes,
514 });
515 return semanticArtifact;
516 }
517
518 async function defaultSemanticSearch(
519 query: string,
520 options?: { signal: AbortSignal },
521 ): Promise<PublicKnowledgeCandidateRecord[]> {
522 const apiKey = process.env.OPENAI_API_KEY;
523 if (!apiKey) throw new Error('embedding-unavailable');
524 let artifact: KnowledgeSemanticArtifact;
525 try {
526 artifact = await loadSemanticArtifact();
527 } catch {
528 throw new Error('semantic-artifact-invalid');
529 }
530 try {
531 const [queryVector] = await embedTexts([query], { apiKey, signal: options?.signal });
532 if (!queryVector) throw new Error('missing query vector');
533 return rankSemanticCandidates(artifact, queryVector, 50, {
534 minimumSimilarity: SEMANTIC_MINIMUM_SIMILARITY,
535 }).map(candidate => publicKnowledgeCandidateFromSemantic(candidate.record));
536 } catch (error) {
537 if (options?.signal.aborted) throw error;
538 throw new Error('semantic-request-failed');
539 }
540 }
541
542 const defaultDependencies: KnowledgeSearchDependencies = {
543 hybridEnabled: () => process.env.KB_HYBRID_SEARCH_ENABLED === 'true',
544 searchKeywordCandidates: defaultKeywordSearch,
545 searchSemanticCandidates: defaultSemanticSearch,
546 searchLocalCandidates: defaultLocalSearch,
547 searchPreviewCandidates: defaultPreviewSearch,
548 previewOverlayEnabled: () => process.env.NODE_ENV === 'development' || process.env.VERCEL_ENV === 'preview',
549 acquireSemanticSearch: acquireDefaultSemanticSearch,
550 semanticTimeoutMs: defaultSemanticTimeoutMs,
551 captureSearch: queueKnowledgeSearchAnalytics,
552 };
553
554 function degradationReason(
555 error: unknown,
556 fallback: KnowledgeDegradationReason,
557 ): KnowledgeDegradationReason {
558 const message = error instanceof Error ? error.message : '';
559 const allowed: KnowledgeDegradationReason[] = [
560 'embedding-unavailable',
561 'semantic-artifact-invalid',
562 'semantic-request-failed',
563 'semantic-timeout',
564 'semantic-rate-limited',
565 'semantic-capacity-limited',
566 'keyword-request-failed',
567 'preview-overlay-failed',
568 ];
569 return allowed.includes(message as KnowledgeDegradationReason)
570 ? message as KnowledgeDegradationReason
571 : fallback;
572 }
573
574 async function protectedSemanticSearch(
575 query: string,
576 headers: Headers | undefined,
577 dependencies: KnowledgeSearchDependencies,
578 ): Promise<PublicKnowledgeCandidateRecord[]> {
579 const request = new Request('http://localhost/api/knowledge-search', { headers });
580 const admission = dependencies.acquireSemanticSearch?.(request)
581 ?? { allowed: true as const, release: () => {} };
582 if (!admission.allowed) throw new Error(admission.reason);
583
584 const controller = new AbortController();
585 const timeoutMs = dependencies.semanticTimeoutMs?.() ?? defaultSemanticTimeoutMs();
586 let timedOut = false;
587 let timer: ReturnType<typeof setTimeout> | undefined;
588 const timeout = new Promise<never>((_resolve, reject) => {
589 timer = setTimeout(() => {
590 timedOut = true;
591 controller.abort();
592 reject(new Error('semantic-timeout'));
593 }, timeoutMs);
594 });
595
596 try {
597 return await Promise.race([
598 dependencies.searchSemanticCandidates(query, { signal: controller.signal }),
599 timeout,
600 ]);
601 } catch (error) {
602 if (timedOut) throw new Error('semantic-timeout');
603 throw error;
604 } finally {
605 if (timer) clearTimeout(timer);
606 admission.release();
607 }
608 }
609
610 function semanticEligible(filter: KnowledgeFilter): boolean {
611 return filter === 'all' || filter === 'docs' || filter === 'kb';
612 }
613
614 function localEditorialEligible(filter: KnowledgeFilter): boolean {
615 return filter === 'all' || filter === 'docs' || filter === 'kb';
616 }
617
618 export async function searchPublicKnowledge(
619 input: KnowledgeSearchInput,
620 dependencies: KnowledgeSearchDependencies = defaultDependencies,
621 ): Promise<KnowledgeSearchExecution> {
622 const query = input.query.trim().slice(0, 200);
623 const startedAt = performance.now();
624 let keywordDurationMs = 0;
625 let semanticDurationMs: number | null = null;
626 let previewOverlayApplied = false;
627 let degradationCategory: KnowledgeSearchAnalyticsEvent['degradationCategory'] = null;
628
629 const finish = (
630 response: KnowledgeSearchExecution['response'],
631 status: KnowledgeSearchExecution['status'],
632 ): KnowledgeSearchExecution => {
633 const totalDurationMs = performance.now() - startedAt;
634 const results = 'results' in response ? response.results : [];
635 const retrievalMode = 'mode' in response
636 ? response.mode ?? 'keyword'
637 : 'unavailable';
638 const strongMatch = 'strongMatch' in response ? response.strongMatch ?? null : null;
639 if (degradationCategory) {
640 console.warn('[kb-search]', JSON.stringify({
641 event: 'kb_search_degraded',
642 reason: degradationCategory,
643 retrievalMode,
644 statusCode: status,
645 }));
646 }
647 try {
648 dependencies.captureSearch?.({
649 query,
650 filter: input.filter,
651 retrievalMode,
652 resultCount: results.length,
653 resultSourceTypes: [...new Set(results.map(result => result.sourceType))],
654 degradationCategory,
655 strongMatch,
656 statusCode: status,
657 durationMs: totalDurationMs,
658 keywordDurationMs,
659 semanticDurationMs,
660 previewOverlayApplied,
661 });
662 } catch {
663 console.warn('[kb-search]', JSON.stringify({ event: 'kb_search_analytics_schedule_failed' }));
664 }
665 return {
666 response,
667 status,
668 cacheControl: status === 200 ? KNOWLEDGE_CACHE_CONTROL : 'no-store',
669 timings: { totalDurationMs, keywordDurationMs, semanticDurationMs },
670 previewOverlayApplied,
671 };
672 };
673
674 if (!query) {
675 keywordDurationMs = performance.now() - startedAt;
676 return {
677 response: { query: '', filter: input.filter, results: [], total: 0 },
678 status: 200,
679 cacheControl: KNOWLEDGE_CACHE_CONTROL,
680 timings: {
681 totalDurationMs: keywordDurationMs,
682 keywordDurationMs,
683 semanticDurationMs: null,
684 },
685 previewOverlayApplied: false,
686 };
687 }
688
689 const keywordStartedAt = performance.now();
690 let keywordCandidates: PublicKnowledgeCandidateRecord[] | null = null;
691 try {
692 const keyword = await dependencies.searchKeywordCandidates(query, input.filter);
693 keywordCandidates = filterCandidates(keyword.candidates, query, input.filter);
694 degradationCategory = keyword.degradedReason ?? null;
695 } catch (error) {
696 degradationCategory = degradationReason(error, 'keyword-request-failed');
697 }
698
699 if (keywordCandidates && localEditorialEligible(input.filter) &&
700 dependencies.previewOverlayEnabled?.() === true) {
701 try {
702 if (!dependencies.searchPreviewCandidates) throw new Error('local-index-unavailable');
703 const localSearch = localCandidateSearch(
704 await dependencies.searchPreviewCandidates(query, input.filter),
705 );
706 const previewCandidates = filterCandidates(
707 localSearch.candidates,
708 query,
709 input.filter,
710 );
711 keywordCandidates = overlayCandidates(
712 keywordCandidates,
713 previewCandidates,
714 localSearch.canonicalUrls,
715 );
716 previewOverlayApplied = true;
717 } catch {
718 degradationCategory = 'preview-overlay-failed';
719 }
720 }
721
722 if (!keywordCandidates) {
723 try {
724 if (!dependencies.searchLocalCandidates) throw new Error('local-index-unavailable');
725 const localSearch = localCandidateSearch(
726 await dependencies.searchLocalCandidates(query, input.filter),
727 );
728 keywordCandidates = filterCandidates(localSearch.candidates, query, input.filter);
729 degradationCategory = 'keyword-request-failed';
730 } catch {
731 keywordCandidates = null;
732 }
733 }
734 keywordDurationMs = performance.now() - keywordStartedAt;
735
736 const keywordWindow = keywordCandidates?.slice(0, 20) ?? null;
737 const hasStrongKeywordCandidate = keywordWindow?.some(candidate =>
738 isStrongLexicalCandidate(candidate, query)) === true;
739 if (keywordWindow && hasStrongKeywordCandidate) {
740 const results = strongKeywordResults(keywordCandidates ?? keywordWindow, query);
741 return finish({
742 query,
743 filter: input.filter,
744 results,
745 total: results.length,
746 mode: 'keyword',
747 strongMatch: true,
748 }, 200);
749 }
750
751 if (!dependencies.hybridEnabled() || !semanticEligible(input.filter)) {
752 if (!keywordWindow) {
753 degradationCategory = 'all-retrievers-failed';
754 return finish({ error: KNOWLEDGE_UNAVAILABLE_MESSAGE }, 503);
755 }
756 const results = keywordResults(keywordCandidates ?? keywordWindow);
757 return finish({
758 query,
759 filter: input.filter,
760 results,
761 total: results.length,
762 mode: 'keyword',
763 }, 200);
764 }
765
766 let semanticCandidates: PublicKnowledgeCandidateRecord[] | null = null;
767 const semanticStartedAt = performance.now();
768 try {
769 semanticCandidates = filterCandidates(
770 await protectedSemanticSearch(query, input.headers, dependencies),
771 query,
772 input.filter,
773 ).filter(candidate => candidate.sourceType === 'docs' || candidate.sourceType === 'kb');
774 } catch (error) {
775 const semanticDegradation = degradationReason(error, 'semantic-request-failed');
776 const isGuardrail = semanticDegradation === 'semantic-timeout' ||
777 semanticDegradation === 'semantic-rate-limited' ||
778 semanticDegradation === 'semantic-capacity-limited';
779 if (keywordWindow && (!degradationCategory || isGuardrail)) {
780 degradationCategory = semanticDegradation;
781 }
782 } finally {
783 semanticDurationMs = performance.now() - semanticStartedAt;
784 }
785
786 if (!keywordWindow && !semanticCandidates) {
787 degradationCategory = 'all-retrievers-failed';
788 return finish({ error: KNOWLEDGE_UNAVAILABLE_MESSAGE }, 503);
789 }
790
791 let mode: KnowledgeRetrievalMode;
792 if (keywordWindow && semanticCandidates) mode = 'hybrid';
793 else if (semanticCandidates) mode = 'semantic';
794 else mode = 'keyword';
795 const fused = fusePublicKbCandidates({
796 query,
797 keyword: keywordWindow ?? [],
798 semantic: semanticCandidates ?? [],
799 limit: 20,
800 });
801 const strongMatch = semanticCandidates
802 ? semanticCandidates.length > 0 || fused.some(candidate =>
803 candidate.keywordRank !== null && candidate.exactTier > 0)
804 : undefined;
805 const results = (strongMatch === false ? [] : fused).map(candidate =>
806 resultFromCandidate(candidate.record));
807 return finish({
808 query,
809 filter: input.filter,
810 results,
811 total: results.length,
812 mode,
813 ...(strongMatch === undefined ? {} : { strongMatch }),
814 }, 200);
815 }