Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 03:37 UTC
| 1 | import { createHash } from 'node:crypto'; |
| 2 | import type { AlgoliaDocsRecord } from '@/lib/search-index'; |
| 3 | |
| 4 | export const KB_EMBEDDING_PROVIDER = 'openai' as const; |
| 5 | export const KB_EMBEDDING_MODEL = 'text-embedding-3-small' as const; |
| 6 | export const KB_EMBEDDING_DIMENSIONS = 256; |
| 7 | |
| 8 | function compact(values: Array<string | undefined>): string[] { |
| 9 | return values.map(value => value?.trim()).filter((value): value is string => Boolean(value)); |
| 10 | } |
| 11 | |
| 12 | export function embeddingText(record: AlgoliaDocsRecord): string { |
| 13 | const exactTerms = compact([ |
| 14 | ...(record.keywords ?? []), |
| 15 | record.slug, |
| 16 | ...(record.tool_names ?? []), |
| 17 | ...(record.tool_slugs ?? []), |
| 18 | ]); |
| 19 | const lines = compact([ |
| 20 | `Title: ${record.title}`, |
| 21 | record.section ? `Section: ${record.section}` : undefined, |
| 22 | record.description ? `Description: ${record.description}` : undefined, |
| 23 | exactTerms.length > 0 ? `Aliases and exact terms: ${exactTerms.join(' | ')}` : undefined, |
| 24 | record.toolkit_slugs.length > 0 ? `Toolkits: ${record.toolkit_slugs.join(' | ')}` : undefined, |
| 25 | `Content: ${record.content}`, |
| 26 | ]); |
| 27 | return lines.join('\n'); |
| 28 | } |
| 29 | |
| 30 | export function embeddingContentHash(record: AlgoliaDocsRecord): string { |
| 31 | return createHash('sha256').update(embeddingText(record), 'utf8').digest('hex'); |
| 32 | } |
| 33 | |
| 34 | function normalized(vector: unknown, expectedDimensions?: number): number[] { |
| 35 | if (!Array.isArray(vector) || vector.length === 0) { |
| 36 | throw new Error('Embedding response vector is empty'); |
| 37 | } |
| 38 | if (expectedDimensions !== undefined && vector.length !== expectedDimensions) { |
| 39 | throw new Error(`Embedding response dimension mismatch: expected ${expectedDimensions}, got ${vector.length}`); |
| 40 | } |
| 41 | const values = vector.map(value => { |
| 42 | if (typeof value !== 'number' || !Number.isFinite(value)) { |
| 43 | throw new Error('Embedding response contains a non-finite value'); |
| 44 | } |
| 45 | return value; |
| 46 | }); |
| 47 | const norm = Math.sqrt(values.reduce((total, value) => total + value * value, 0)); |
| 48 | if (!Number.isFinite(norm) || norm === 0) throw new Error('Embedding response vector has zero norm'); |
| 49 | return values.map(value => value / norm); |
| 50 | } |
| 51 | |
| 52 | interface EmbeddingResponse { |
| 53 | data?: Array<{ index?: number; embedding?: unknown }>; |
| 54 | error?: { message?: string }; |
| 55 | } |
| 56 | |
| 57 | export async function embedTexts( |
| 58 | texts: string[], |
| 59 | options: { |
| 60 | apiKey: string; |
| 61 | fetch?: typeof globalThis.fetch; |
| 62 | signal?: AbortSignal; |
| 63 | }, |
| 64 | ): Promise<number[][]> { |
| 65 | if (!options.apiKey.trim()) throw new Error('OpenAI embedding API key is missing'); |
| 66 | if (texts.length === 0) return []; |
| 67 | const fetchImplementation = options.fetch ?? globalThis.fetch; |
| 68 | const response = await fetchImplementation('https://api.openai.com/v1/embeddings', { |
| 69 | method: 'POST', |
| 70 | headers: { |
| 71 | Authorization: `Bearer ${options.apiKey}`, |
| 72 | 'Content-Type': 'application/json', |
| 73 | }, |
| 74 | body: JSON.stringify({ |
| 75 | model: KB_EMBEDDING_MODEL, |
| 76 | dimensions: KB_EMBEDDING_DIMENSIONS, |
| 77 | encoding_format: 'float', |
| 78 | input: texts, |
| 79 | }), |
| 80 | signal: options.signal, |
| 81 | }); |
| 82 | const body = await response.json() as EmbeddingResponse; |
| 83 | if (!response.ok) { |
| 84 | throw new Error(`OpenAI embedding request failed with HTTP ${response.status}`); |
| 85 | } |
| 86 | if (!Array.isArray(body.data) || body.data.length !== texts.length) { |
| 87 | throw new Error('Embedding response record count mismatch'); |
| 88 | } |
| 89 | |
| 90 | const ordered = new Array<number[]>(texts.length); |
| 91 | for (const item of body.data) { |
| 92 | if (!Number.isInteger(item.index) || (item.index ?? -1) < 0 || (item.index ?? -1) >= texts.length) { |
| 93 | throw new Error('Embedding response index is invalid'); |
| 94 | } |
| 95 | if (ordered[item.index!] !== undefined) throw new Error('Embedding response index is duplicated'); |
| 96 | ordered[item.index!] = normalized(item.embedding); |
| 97 | } |
| 98 | if (ordered.some(vector => vector === undefined)) { |
| 99 | throw new Error('Embedding response index is missing'); |
| 100 | } |
| 101 | return ordered; |
| 102 | } |