Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 102 lines 3.8 KB
Raw
1 import { createHash } from 'node:crypto';
2 import type { AlgoliaDocsRecord } from '@/lib/search-index';
3
4 export const KB_EMBEDDING_PROVIDER = 'openai' as const;
5 export const KB_EMBEDDING_MODEL = 'text-embedding-3-small' as const;
6 export const KB_EMBEDDING_DIMENSIONS = 256;
7
8 function compact(values: Array<string | undefined>): string[] {
9 return values.map(value => value?.trim()).filter((value): value is string => Boolean(value));
10 }
11
12 export function embeddingText(record: AlgoliaDocsRecord): string {
13 const exactTerms = compact([
14 ...(record.keywords ?? []),
15 record.slug,
16 ...(record.tool_names ?? []),
17 ...(record.tool_slugs ?? []),
18 ]);
19 const lines = compact([
20 `Title: ${record.title}`,
21 record.section ? `Section: ${record.section}` : undefined,
22 record.description ? `Description: ${record.description}` : undefined,
23 exactTerms.length > 0 ? `Aliases and exact terms: ${exactTerms.join(' | ')}` : undefined,
24 record.toolkit_slugs.length > 0 ? `Toolkits: ${record.toolkit_slugs.join(' | ')}` : undefined,
25 `Content: ${record.content}`,
26 ]);
27 return lines.join('\n');
28 }
29
30 export function embeddingContentHash(record: AlgoliaDocsRecord): string {
31 return createHash('sha256').update(embeddingText(record), 'utf8').digest('hex');
32 }
33
34 function normalized(vector: unknown, expectedDimensions?: number): number[] {
35 if (!Array.isArray(vector) || vector.length === 0) {
36 throw new Error('Embedding response vector is empty');
37 }
38 if (expectedDimensions !== undefined && vector.length !== expectedDimensions) {
39 throw new Error(`Embedding response dimension mismatch: expected ${expectedDimensions}, got ${vector.length}`);
40 }
41 const values = vector.map(value => {
42 if (typeof value !== 'number' || !Number.isFinite(value)) {
43 throw new Error('Embedding response contains a non-finite value');
44 }
45 return value;
46 });
47 const norm = Math.sqrt(values.reduce((total, value) => total + value * value, 0));
48 if (!Number.isFinite(norm) || norm === 0) throw new Error('Embedding response vector has zero norm');
49 return values.map(value => value / norm);
50 }
51
52 interface EmbeddingResponse {
53 data?: Array<{ index?: number; embedding?: unknown }>;
54 error?: { message?: string };
55 }
56
57 export async function embedTexts(
58 texts: string[],
59 options: {
60 apiKey: string;
61 fetch?: typeof globalThis.fetch;
62 signal?: AbortSignal;
63 },
64 ): Promise<number[][]> {
65 if (!options.apiKey.trim()) throw new Error('OpenAI embedding API key is missing');
66 if (texts.length === 0) return [];
67 const fetchImplementation = options.fetch ?? globalThis.fetch;
68 const response = await fetchImplementation('https://api.openai.com/v1/embeddings', {
69 method: 'POST',
70 headers: {
71 Authorization: `Bearer ${options.apiKey}`,
72 'Content-Type': 'application/json',
73 },
74 body: JSON.stringify({
75 model: KB_EMBEDDING_MODEL,
76 dimensions: KB_EMBEDDING_DIMENSIONS,
77 encoding_format: 'float',
78 input: texts,
79 }),
80 signal: options.signal,
81 });
82 const body = await response.json() as EmbeddingResponse;
83 if (!response.ok) {
84 throw new Error(`OpenAI embedding request failed with HTTP ${response.status}`);
85 }
86 if (!Array.isArray(body.data) || body.data.length !== texts.length) {
87 throw new Error('Embedding response record count mismatch');
88 }
89
90 const ordered = new Array<number[]>(texts.length);
91 for (const item of body.data) {
92 if (!Number.isInteger(item.index) || (item.index ?? -1) < 0 || (item.index ?? -1) >= texts.length) {
93 throw new Error('Embedding response index is invalid');
94 }
95 if (ordered[item.index!] !== undefined) throw new Error('Embedding response index is duplicated');
96 ordered[item.index!] = normalized(item.embedding);
97 }
98 if (ordered.some(vector => vector === undefined)) {
99 throw new Error('Embedding response index is missing');
100 }
101 return ordered;
102 }