Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 343 lines 11.4 KB
Raw
1 import type { AlgoliaDocsRecord } from '@/lib/search-index';
2 import {
3 KNOWLEDGE_SOURCE_LABELS,
4 type KnowledgeSourceType,
5 type ProductAreaSlug,
6 } from './types';
7
8 export type KnowledgeFilter =
9 | 'all'
10 | 'docs'
11 | 'kb'
12 | 'oauth'
13 | 'toolkit'
14 | 'toolkits'
15 | 'reference';
16
17 export interface KnowledgeFilterDefinition {
18 value: KnowledgeFilter;
19 label: string;
20 sourceTypes: KnowledgeSourceType[];
21 }
22
23 export interface KnowledgeSearchResult {
24 objectID: string;
25 title: string;
26 excerpt: string;
27 canonicalUrl: string;
28 sourceType: KnowledgeSourceType;
29 sourceLabel: string;
30 breadcrumbs: string[];
31 productAreas: ProductAreaSlug[];
32 toolkitSlugs: string[];
33 lastVerifiedAt: string | null;
34 section?: string | null;
35 }
36
37 export type KnowledgeRetrievalMode = 'hybrid' | 'semantic' | 'keyword';
38 export type KnowledgeDegradationReason =
39 | 'embedding-unavailable'
40 | 'semantic-artifact-invalid'
41 | 'semantic-request-failed'
42 | 'semantic-timeout'
43 | 'semantic-rate-limited'
44 | 'semantic-capacity-limited'
45 | 'keyword-request-failed'
46 | 'preview-overlay-failed';
47
48 export interface KnowledgeSearchResponse {
49 query: string;
50 filter: KnowledgeFilter;
51 results: KnowledgeSearchResult[];
52 total: number;
53 mode?: KnowledgeRetrievalMode;
54 strongMatch?: boolean;
55 }
56
57 export const KNOWLEDGE_FILTERS: readonly KnowledgeFilterDefinition[] = [
58 { value: 'all', label: 'All', sourceTypes: [] },
59 { value: 'docs', label: 'Docs', sourceTypes: ['docs'] },
60 { value: 'kb', label: 'Knowledge Base', sourceTypes: ['kb'] },
61 { value: 'oauth', label: 'OAuth', sourceTypes: ['oauth-guide'] },
62 { value: 'toolkits', label: 'Toolkits', sourceTypes: ['toolkit'] },
63 { value: 'reference', label: 'Reference', sourceTypes: ['reference', 'legacy'] },
64 ] as const;
65
66 const FILTER_BY_VALUE = new Map<KnowledgeFilter, KnowledgeFilterDefinition>([
67 ...KNOWLEDGE_FILTERS.map((filter) => [filter.value, filter] as const),
68 ['toolkit', { value: 'toolkit', label: 'Toolkits', sourceTypes: ['toolkit'] }],
69 ]);
70
71 export function isKnowledgeFilter(value: string): value is KnowledgeFilter {
72 return FILTER_BY_VALUE.has(value as KnowledgeFilter);
73 }
74
75 export function algoliaFacetFilters(filter: KnowledgeFilter): string[][] {
76 const sourceTypes = FILTER_BY_VALUE.get(filter)?.sourceTypes ?? [];
77 return sourceTypes.length > 0
78 ? [sourceTypes.map((sourceType) => `source_type:${sourceType}`)]
79 : [];
80 }
81
82 export function knowledgeFilterIncludesSource(
83 filter: KnowledgeFilter,
84 sourceType: KnowledgeSourceType,
85 ): boolean {
86 const sourceTypes = FILTER_BY_VALUE.get(filter)?.sourceTypes ?? [];
87 return sourceTypes.length === 0 || sourceTypes.includes(sourceType);
88 }
89
90 export function normalizeKnowledgeText(value: string): string {
91 return value
92 .toLowerCase()
93 .replace(/[_-]+/g, ' ')
94 .replace(/[^a-z0-9]+/g, ' ')
95 .replace(/\s+/g, ' ')
96 .trim();
97 }
98
99 function textValues(record: AlgoliaDocsRecord): {
100 title: string;
101 identity: string[];
102 body: string;
103 all: string;
104 } {
105 const title = normalizeKnowledgeText(record.title);
106 const identity = [
107 ...(record.keywords ?? []),
108 record.slug ?? '',
109 ...(record.tool_names ?? []),
110 ...(record.tool_slugs ?? []),
111 ...record.toolkit_slugs,
112 ].map(normalizeKnowledgeText).filter(Boolean);
113 const body = normalizeKnowledgeText([
114 record.description,
115 record.section,
116 record.content,
117 ...(record.headings ?? []),
118 ].filter(Boolean).join(' '));
119 return { title, identity, body, all: [title, ...identity, body].join(' ') };
120 }
121
122 const LOOSE_MATCH_STOP_WORDS = new Set([
123 'a', 'an', 'and', 'are', 'as', 'at', 'be', 'but', 'by', 'can', 'did', 'do', 'does',
124 'for', 'from', 'had', 'has', 'have', 'how', 'i', 'if', 'in', 'into', 'is', 'it',
125 'my', 'of', 'on', 'or', 'that', 'the', 'their', 'then', 'they', 'this', 'to', 'was',
126 'what', 'when', 'where', 'which', 'who', 'why', 'with', 'you', 'your',
127 ]);
128
129 function containsToken(value: string, token: string): boolean {
130 return ` ${value} `.includes(` ${token} `);
131 }
132
133 function textualTier(record: AlgoliaDocsRecord, normalizedQuery: string): number | null {
134 const values = textValues(record);
135 const tokens = normalizedQuery
136 .split(' ')
137 .filter(token => token && !LOOSE_MATCH_STOP_WORDS.has(token));
138 if (values.title === normalizedQuery) return 6;
139 if (values.identity.some((value) => value === normalizedQuery)) return 5;
140 if (values.title.includes(normalizedQuery) || values.identity.some((value) => value.includes(normalizedQuery))) {
141 return 4;
142 }
143 if (values.body.includes(normalizedQuery)) return 3;
144 if (tokens.length === 0) return null;
145 if (tokens.every((token) => containsToken(values.title, token) ||
146 values.identity.some((value) => containsToken(value, token)))) {
147 return 2;
148 }
149 const matchedTokens = tokens.filter(token => containsToken(values.all, token)).length;
150 if (matchedTokens === tokens.length) return 1;
151 const minimumLooseMatches = Math.max(2, Math.ceil(tokens.length / 2));
152 if (matchedTokens >= minimumLooseMatches) return matchedTokens / tokens.length;
153 return null;
154 }
155
156 export function filterLegacyReferenceRecords<T extends AlgoliaDocsRecord>(
157 records: T[],
158 query: string,
159 filter: KnowledgeFilter,
160 ): T[] {
161 if (filter !== 'all' && filter !== 'reference') return records;
162
163 const normalizedQuery = normalizeKnowledgeText(query);
164 const hasCurrentExactMatch = records.some((record) => {
165 if (record.source_type !== 'reference') return false;
166 return (textualTier(record, normalizedQuery) ?? -1) >= 5;
167 });
168
169 return records.filter((record) => {
170 if (record.source_type !== 'legacy') return true;
171 return !hasCurrentExactMatch && (textualTier(record, normalizedQuery) ?? -1) >= 5;
172 });
173 }
174
175 function decodeHtmlEntities(value: string): string {
176 const namedEntities: Record<string, string> = {
177 amp: '&',
178 apos: "'",
179 gt: '>',
180 lt: '<',
181 nbsp: ' ',
182 quot: '"',
183 };
184
185 return value.replace(/&(#x[\da-f]+|#\d+|amp|apos|gt|lt|nbsp|quot);/gi, (match, entity: string) => {
186 if (entity.startsWith('#')) {
187 const hexadecimal = entity[1]?.toLowerCase() === 'x';
188 const codePoint = Number.parseInt(entity.slice(hexadecimal ? 2 : 1), hexadecimal ? 16 : 10);
189 if (Number.isInteger(codePoint) && codePoint >= 0 && codePoint <= 0x10ffff) {
190 return String.fromCodePoint(codePoint);
191 }
192 return match;
193 }
194
195 return namedEntities[entity.toLowerCase()] ?? match;
196 });
197 }
198
199 const HTML_TAG_NAMES = new Set([
200 'a', 'abbr', 'applet', 'article', 'aside', 'audio', 'b', 'base', 'bgsound',
201 'blockquote', 'body', 'br', 'button', 'canvas', 'caption', 'code', 'col',
202 'colgroup', 'datalist',
203 'dd', 'details', 'dialog', 'div', 'dl', 'dt', 'em', 'embed', 'fieldset',
204 'figcaption', 'figure', 'footer', 'form', 'frame', 'frameset', 'h1', 'h2',
205 'h3', 'h4', 'h5', 'h6', 'head', 'header', 'hr', 'html', 'i', 'iframe',
206 'image', 'img', 'input', 'keygen', 'label', 'li', 'link', 'listing', 'main',
207 'mark', 'marquee', 'math', 'meta', 'nav', 'noembed', 'noframes', 'noscript',
208 'object', 'ol', 'option', 'p', 'picture', 'plaintext', 'pre', 'script',
209 'section', 'select', 'small', 'source', 'span', 'strong', 'style', 'sub',
210 'summary', 'sup', 'svg', 'table', 'tbody', 'td', 'template', 'textarea',
211 'tfoot', 'th', 'thead', 'time', 'title', 'tr', 'track', 'ul', 'video', 'xmp',
212 ]);
213
214 function stripHtmlTags(value: string): string {
215 let plainText = '';
216 let cursor = 0;
217
218 while (cursor < value.length) {
219 if (value[cursor] !== '<') {
220 plainText += value[cursor];
221 cursor += 1;
222 continue;
223 }
224
225 const remainder = value.slice(cursor);
226 const candidate = remainder.match(/^<\/?([A-Za-z][A-Za-z0-9-]*)\b/);
227 const isDeclaration = /^<[!?]/.test(remainder);
228 const isKnownTag = HTML_TAG_NAMES.has(candidate?.[1]?.toLowerCase() ?? '');
229 const isClosingTag = /^<\/[A-Za-z][A-Za-z0-9-]*\s*>/.test(remainder);
230 const hasAttributes = /^<[A-Za-z][A-Za-z0-9-]*\s+[^>]*=[^>]*\/?>/.test(remainder);
231 if (!isDeclaration && !isKnownTag && !isClosingTag && !hasAttributes) {
232 plainText += '<';
233 cursor += 1;
234 continue;
235 }
236
237 const closing = value.indexOf('>', cursor + 1);
238 if (closing < 0) break;
239 plainText += ' ';
240 cursor = closing + 1;
241 }
242
243 return plainText;
244 }
245
246 export function plainKnowledgeExcerpt(value: string): string {
247 let plain = value;
248 for (let pass = 0; pass < 3; pass += 1) {
249 const next = stripHtmlTags(decodeHtmlEntities(plain));
250 if (next === plain) break;
251 plain = next;
252 }
253
254 return plain
255 .replace(/^\s{0,3}#{1,6}\s+/gm, '')
256 .replace(/!\[([^\]]*)\]\([^)]*\)/g, '$1')
257 .replace(/\[([^\]]+)\]\([^)]*\)/g, '$1')
258 .replace(/(\*\*|__)(.*?)\1/g, '$2')
259 .replace(/[*~`]+/g, '')
260 .replace(/\s+/g, ' ')
261 .trim();
262 }
263
264 function excerpt(record: AlgoliaDocsRecord): string {
265 const value = plainKnowledgeExcerpt(record.description || record.content || record.title);
266 return value.length > 260 ? `${value.slice(0, 257).trimEnd()}…` : value;
267 }
268
269 export function knowledgeSearchResultFromRecord(
270 record: AlgoliaDocsRecord,
271 matchingExcerpt?: string,
272 ): KnowledgeSearchResult {
273 const sourceLabel = KNOWLEDGE_SOURCE_LABELS[record.source_type];
274 const breadcrumbs = (record.breadcrumbs ?? []).filter(
275 (breadcrumb, index) => index > 0 || breadcrumb.toLowerCase() !== sourceLabel.toLowerCase(),
276 );
277 return {
278 objectID: record.objectID,
279 title: record.title,
280 excerpt: matchingExcerpt?.trim()
281 ? plainKnowledgeExcerpt(matchingExcerpt)
282 : excerpt(record),
283 canonicalUrl: record.canonical_url || record.url,
284 sourceType: record.source_type,
285 sourceLabel,
286 breadcrumbs,
287 productAreas: record.product_areas,
288 toolkitSlugs: record.toolkit_slugs,
289 lastVerifiedAt: record.last_verified_at,
290 section: record.section ?? null,
291 };
292 }
293
294 export function searchKnowledgeRecords(
295 records: AlgoliaDocsRecord[],
296 request: { query: string; filter: KnowledgeFilter; limit: number },
297 ): KnowledgeSearchResponse {
298 const query = request.query.trim();
299 const normalizedQuery = normalizeKnowledgeText(query);
300 if (!normalizedQuery) return { query: '', filter: request.filter, results: [], total: 0 };
301
302 const filter = FILTER_BY_VALUE.get(request.filter) ?? FILTER_BY_VALUE.get('all')!;
303 const candidates = filterLegacyReferenceRecords(
304 records.filter((record) =>
305 filter.sourceTypes.length === 0 || filter.sourceTypes.includes(record.source_type)),
306 query,
307 request.filter,
308 );
309 const scored = candidates.flatMap((record) => {
310 const tier = textualTier(record, normalizedQuery);
311 if (tier === null) return [];
312 return [{ record, tier }];
313 });
314
315 const bestByCanonicalUrl = new Map<string, (typeof scored)[number]>();
316 for (const candidate of scored) {
317 const key = candidate.record.canonical_url || candidate.record.url;
318 const current = bestByCanonicalUrl.get(key);
319 if (
320 !current ||
321 candidate.tier > current.tier ||
322 (candidate.tier === current.tier && candidate.record.section_rank > current.record.section_rank)
323 ) {
324 bestByCanonicalUrl.set(key, candidate);
325 }
326 }
327
328 const ranked = [...bestByCanonicalUrl.values()].sort((left, right) =>
329 right.tier - left.tier ||
330 right.record.page_rank - left.record.page_rank ||
331 right.record.toolkit_popularity - left.record.toolkit_popularity ||
332 right.record.section_rank - left.record.section_rank ||
333 left.record.title.localeCompare(right.record.title),
334 );
335
336 return {
337 query,
338 filter: request.filter,
339 results: ranked.slice(0, Math.max(0, request.limit)).map(({ record }) =>
340 knowledgeSearchResultFromRecord(record)),
341 total: ranked.length,
342 };
343 }