Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 03:37 UTC
| 1 | import { |
| 2 | embeddingContentHash, |
| 3 | embeddingText, |
| 4 | KB_EMBEDDING_DIMENSIONS, |
| 5 | KB_EMBEDDING_MODEL, |
| 6 | KB_EMBEDDING_PROVIDER, |
| 7 | } from './embeddings'; |
| 8 | import { createHash } from 'node:crypto'; |
| 9 | import type { AlgoliaDocsRecord } from '@/lib/search-index'; |
| 10 | import { |
| 11 | KNOWLEDGE_SOURCE_LABELS, |
| 12 | type KnowledgeSourceType, |
| 13 | type ProductAreaSlug, |
| 14 | } from './types'; |
| 15 | |
| 16 | export interface KnowledgeSemanticRecord { |
| 17 | objectID: string; |
| 18 | sourceType: Extract<KnowledgeSourceType, 'docs' | 'kb'>; |
| 19 | sourceLabel: string; |
| 20 | pageID: string; |
| 21 | title: string; |
| 22 | section: string | null; |
| 23 | description: string; |
| 24 | content: string; |
| 25 | canonicalUrl: string; |
| 26 | breadcrumbs: string[]; |
| 27 | productAreas: ProductAreaSlug[]; |
| 28 | toolkitSlugs: string[]; |
| 29 | keywords: string[]; |
| 30 | slug: string; |
| 31 | toolNames: string[]; |
| 32 | toolSlugs: string[]; |
| 33 | pageRank: number; |
| 34 | sectionRank: number; |
| 35 | lastVerifiedAt: string | null; |
| 36 | contentHash: string; |
| 37 | visibility: 'public'; |
| 38 | } |
| 39 | |
| 40 | /** @deprecated Use KnowledgeSemanticRecord. */ |
| 41 | export type KbSemanticRecord = KnowledgeSemanticRecord; |
| 42 | |
| 43 | export interface KnowledgeSemanticArtifact { |
| 44 | formatVersion: 2; |
| 45 | provider: typeof KB_EMBEDDING_PROVIDER; |
| 46 | model: typeof KB_EMBEDDING_MODEL; |
| 47 | dimensions: number; |
| 48 | source: { |
| 49 | docsContentHash: string; |
| 50 | }; |
| 51 | builtAt: string; |
| 52 | records: KnowledgeSemanticRecord[]; |
| 53 | vectorsBase64: string; |
| 54 | } |
| 55 | |
| 56 | /** @deprecated Use KnowledgeSemanticArtifact. */ |
| 57 | export type KbSemanticArtifact = KnowledgeSemanticArtifact; |
| 58 | |
| 59 | export interface RankedSemanticCandidate { |
| 60 | record: KnowledgeSemanticRecord; |
| 61 | rank: number; |
| 62 | similarity: number; |
| 63 | } |
| 64 | |
| 65 | export function semanticRecordFromSearchRecord(record: AlgoliaDocsRecord): KnowledgeSemanticRecord { |
| 66 | if (record.source_type !== 'kb' && record.source_type !== 'docs') { |
| 67 | throw new Error(`Only public docs and KB records can be embedded: ${record.objectID}`); |
| 68 | } |
| 69 | return { |
| 70 | objectID: record.objectID, |
| 71 | sourceType: record.source_type, |
| 72 | sourceLabel: KNOWLEDGE_SOURCE_LABELS[record.source_type], |
| 73 | pageID: record.page_id, |
| 74 | title: record.title, |
| 75 | section: record.section ?? null, |
| 76 | description: record.description ?? '', |
| 77 | content: record.content, |
| 78 | canonicalUrl: record.canonical_url, |
| 79 | breadcrumbs: record.breadcrumbs ?? [], |
| 80 | productAreas: record.product_areas, |
| 81 | toolkitSlugs: record.toolkit_slugs, |
| 82 | keywords: record.keywords ?? [], |
| 83 | slug: record.slug ?? '', |
| 84 | toolNames: record.tool_names ?? [], |
| 85 | toolSlugs: record.tool_slugs ?? [], |
| 86 | pageRank: record.page_rank, |
| 87 | sectionRank: record.section_rank, |
| 88 | lastVerifiedAt: record.last_verified_at, |
| 89 | contentHash: embeddingContentHash(record), |
| 90 | visibility: 'public', |
| 91 | }; |
| 92 | } |
| 93 | |
| 94 | export function docsContentHashFromRecords(records: readonly KnowledgeSemanticRecord[]): string { |
| 95 | const content = records |
| 96 | .filter(record => record.sourceType === 'docs') |
| 97 | .sort((left, right) => left.objectID.localeCompare(right.objectID)) |
| 98 | .map(record => `${record.objectID}\u0000${record.contentHash}`) |
| 99 | .join('\n'); |
| 100 | return createHash('sha256').update(content).digest('hex'); |
| 101 | } |
| 102 | |
| 103 | export function encodeVectors(vectors: number[][]): string { |
| 104 | const dimensions = vectors[0]?.length ?? 0; |
| 105 | if (dimensions === 0 || vectors.some(vector => vector.length !== dimensions)) { |
| 106 | throw new Error('Semantic vectors must have one consistent non-zero dimension'); |
| 107 | } |
| 108 | const values = new Float32Array(vectors.length * dimensions); |
| 109 | let offset = 0; |
| 110 | for (const vector of vectors) { |
| 111 | for (const value of vector) { |
| 112 | if (!Number.isFinite(value)) throw new Error('Semantic vector contains a non-finite value'); |
| 113 | values[offset++] = value; |
| 114 | } |
| 115 | } |
| 116 | return Buffer.from(values.buffer, values.byteOffset, values.byteLength).toString('base64'); |
| 117 | } |
| 118 | |
| 119 | export function decodeVectors( |
| 120 | vectorsBase64: string, |
| 121 | recordCount: number, |
| 122 | dimensions: number, |
| 123 | ): Float32Array { |
| 124 | const bytes = Buffer.from(vectorsBase64, 'base64'); |
| 125 | const expectedBytes = recordCount * dimensions * Float32Array.BYTES_PER_ELEMENT; |
| 126 | if (bytes.byteLength !== expectedBytes) { |
| 127 | throw new Error(`Semantic vector byte length mismatch: expected ${expectedBytes}, got ${bytes.byteLength}`); |
| 128 | } |
| 129 | const values = new Float32Array(recordCount * dimensions); |
| 130 | const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); |
| 131 | for (let index = 0; index < values.length; index += 1) { |
| 132 | values[index] = view.getFloat32(index * Float32Array.BYTES_PER_ELEMENT, true); |
| 133 | } |
| 134 | return values; |
| 135 | } |
| 136 | |
| 137 | /** A valid artifact that needs to be rebuilt for the current corpus. */ |
| 138 | export class SemanticArtifactStaleError extends Error {} |
| 139 | |
| 140 | export function validateSemanticArtifact( |
| 141 | artifact: KnowledgeSemanticArtifact, |
| 142 | expected: { |
| 143 | dimensions?: number; |
| 144 | docsContentHash?: string; |
| 145 | contentHashes?: ReadonlyMap<string, string>; |
| 146 | }, |
| 147 | ): KnowledgeSemanticArtifact { |
| 148 | if (artifact.formatVersion !== 2) throw new Error('Semantic artifact format mismatch'); |
| 149 | if (artifact.provider !== KB_EMBEDDING_PROVIDER) throw new Error('Semantic artifact provider mismatch'); |
| 150 | if (artifact.model !== KB_EMBEDDING_MODEL) throw new Error('Semantic artifact model mismatch'); |
| 151 | if (artifact.dimensions !== (expected.dimensions ?? KB_EMBEDDING_DIMENSIONS)) { |
| 152 | throw new Error('Semantic artifact dimension mismatch'); |
| 153 | } |
| 154 | if (artifact.records.length === 0) throw new Error('Semantic artifact has no records'); |
| 155 | |
| 156 | const seen = new Set<string>(); |
| 157 | for (const record of artifact.records) { |
| 158 | if (record.sourceType !== 'docs' && record.sourceType !== 'kb') { |
| 159 | throw new Error(`Semantic record ${record.objectID} has an invalid source type`); |
| 160 | } |
| 161 | if (record.visibility !== 'public') throw new Error(`Semantic record ${record.objectID} is not public`); |
| 162 | if (!record.objectID || seen.has(record.objectID)) { |
| 163 | throw new Error(`Semantic artifact has duplicate object ID: ${record.objectID}`); |
| 164 | } |
| 165 | seen.add(record.objectID); |
| 166 | } |
| 167 | |
| 168 | const vectors = decodeVectors(artifact.vectorsBase64, artifact.records.length, artifact.dimensions); |
| 169 | for (let row = 0; row < artifact.records.length; row += 1) { |
| 170 | let squaredNorm = 0; |
| 171 | const start = row * artifact.dimensions; |
| 172 | for (let column = 0; column < artifact.dimensions; column += 1) { |
| 173 | const value = vectors[start + column]; |
| 174 | if (!Number.isFinite(value)) throw new Error('Semantic artifact contains a non-finite vector'); |
| 175 | squaredNorm += value * value; |
| 176 | } |
| 177 | if (Math.abs(Math.sqrt(squaredNorm) - 1) > 0.002) { |
| 178 | throw new Error(`Semantic vector for ${artifact.records[row]?.objectID} is not normalized`); |
| 179 | } |
| 180 | } |
| 181 | // Check integrity first so advisory freshness checks cannot hide corrupt artifacts. |
| 182 | if (expected.docsContentHash && artifact.source.docsContentHash !== expected.docsContentHash) { |
| 183 | throw new SemanticArtifactStaleError('Semantic artifact docs content hash mismatch'); |
| 184 | } |
| 185 | if (expected.contentHashes) { |
| 186 | for (const record of artifact.records) { |
| 187 | if (expected.contentHashes.get(record.objectID) !== record.contentHash) { |
| 188 | throw new SemanticArtifactStaleError(`Semantic artifact content hash mismatch for ${record.objectID}`); |
| 189 | } |
| 190 | } |
| 191 | if (expected.contentHashes.size !== artifact.records.length) { |
| 192 | throw new SemanticArtifactStaleError('Semantic artifact content hash record count mismatch'); |
| 193 | } |
| 194 | } |
| 195 | return artifact; |
| 196 | } |
| 197 | |
| 198 | function normalizeQueryVector(queryVector: number[], dimensions: number): number[] { |
| 199 | if (queryVector.length !== dimensions) throw new Error('Semantic query vector dimension mismatch'); |
| 200 | if (queryVector.some(value => !Number.isFinite(value))) { |
| 201 | throw new Error('Semantic query vector contains a non-finite value'); |
| 202 | } |
| 203 | const norm = Math.sqrt(queryVector.reduce((total, value) => total + value * value, 0)); |
| 204 | if (norm === 0) throw new Error('Semantic query vector has zero norm'); |
| 205 | return queryVector.map(value => value / norm); |
| 206 | } |
| 207 | |
| 208 | export function rankSemanticCandidates( |
| 209 | artifact: KnowledgeSemanticArtifact, |
| 210 | queryVector: number[], |
| 211 | limit: number, |
| 212 | options?: { minimumSimilarity?: number }, |
| 213 | ): RankedSemanticCandidate[] { |
| 214 | const query = normalizeQueryVector(queryVector, artifact.dimensions); |
| 215 | const vectors = decodeVectors(artifact.vectorsBase64, artifact.records.length, artifact.dimensions); |
| 216 | const scored = artifact.records.map((record, row) => { |
| 217 | let similarity = 0; |
| 218 | const start = row * artifact.dimensions; |
| 219 | for (let column = 0; column < artifact.dimensions; column += 1) { |
| 220 | similarity += query[column]! * vectors[start + column]!; |
| 221 | } |
| 222 | return { record, similarity }; |
| 223 | }); |
| 224 | scored.sort((left, right) => |
| 225 | right.similarity - left.similarity || left.record.objectID.localeCompare(right.record.objectID), |
| 226 | ); |
| 227 | const minimumSimilarity = options?.minimumSimilarity ?? Number.NEGATIVE_INFINITY; |
| 228 | return scored |
| 229 | .filter(candidate => candidate.similarity >= minimumSimilarity) |
| 230 | .slice(0, Math.max(0, limit)).map((candidate, index) => ({ |
| 231 | ...candidate, |
| 232 | rank: index + 1, |
| 233 | })); |
| 234 | } |
| 235 | |
| 236 | export async function buildSemanticArtifact(input: { |
| 237 | records: AlgoliaDocsRecord[]; |
| 238 | docsContentHash?: string; |
| 239 | builtAt: string; |
| 240 | previousArtifact?: KnowledgeSemanticArtifact; |
| 241 | embed: (texts: string[]) => Promise<number[][]>; |
| 242 | }): Promise<KnowledgeSemanticArtifact> { |
| 243 | const records = [...input.records] |
| 244 | .sort((left, right) => left.objectID.localeCompare(right.objectID)); |
| 245 | const metadata = records.map(semanticRecordFromSearchRecord); |
| 246 | const docsContentHash = input.docsContentHash ?? docsContentHashFromRecords(metadata); |
| 247 | const previousVectors = input.previousArtifact && |
| 248 | input.previousArtifact.provider === KB_EMBEDDING_PROVIDER && |
| 249 | input.previousArtifact.model === KB_EMBEDDING_MODEL && |
| 250 | input.previousArtifact.dimensions === KB_EMBEDDING_DIMENSIONS |
| 251 | ? decodeVectors( |
| 252 | input.previousArtifact.vectorsBase64, |
| 253 | input.previousArtifact.records.length, |
| 254 | input.previousArtifact.dimensions, |
| 255 | ) |
| 256 | : null; |
| 257 | const previousRows = new Map( |
| 258 | input.previousArtifact?.records.map((record, index) => [record.objectID, { record, index }]) ?? [], |
| 259 | ); |
| 260 | |
| 261 | const vectors = new Array<number[] | undefined>(records.length); |
| 262 | const missingRows: number[] = []; |
| 263 | for (let index = 0; index < metadata.length; index += 1) { |
| 264 | const current = metadata[index]!; |
| 265 | const previous = previousRows.get(current.objectID); |
| 266 | if (previous && previousVectors && previous.record.contentHash === current.contentHash) { |
| 267 | const start = previous.index * KB_EMBEDDING_DIMENSIONS; |
| 268 | vectors[index] = Array.from( |
| 269 | previousVectors.subarray(start, start + KB_EMBEDDING_DIMENSIONS), |
| 270 | ); |
| 271 | } else { |
| 272 | missingRows.push(index); |
| 273 | } |
| 274 | } |
| 275 | |
| 276 | if (missingRows.length > 0) { |
| 277 | const embedded = await input.embed(missingRows.map(index => embeddingText(records[index]!))); |
| 278 | if (embedded.length !== missingRows.length) { |
| 279 | throw new Error('Embedding builder result count mismatch'); |
| 280 | } |
| 281 | for (let index = 0; index < missingRows.length; index += 1) { |
| 282 | const vector = embedded[index]; |
| 283 | if (!vector || vector.length !== KB_EMBEDDING_DIMENSIONS) { |
| 284 | throw new Error('Embedding builder dimension mismatch'); |
| 285 | } |
| 286 | vectors[missingRows[index]!] = vector; |
| 287 | } |
| 288 | } |
| 289 | |
| 290 | if (vectors.some(vector => vector === undefined)) { |
| 291 | throw new Error('Embedding builder left a record without a vector'); |
| 292 | } |
| 293 | const artifact: KnowledgeSemanticArtifact = { |
| 294 | formatVersion: 2, |
| 295 | provider: KB_EMBEDDING_PROVIDER, |
| 296 | model: KB_EMBEDDING_MODEL, |
| 297 | dimensions: KB_EMBEDDING_DIMENSIONS, |
| 298 | source: { docsContentHash }, |
| 299 | builtAt: input.builtAt, |
| 300 | records: metadata, |
| 301 | vectorsBase64: encodeVectors(vectors as number[][]), |
| 302 | }; |
| 303 | return validateSemanticArtifact(artifact, { |
| 304 | docsContentHash, |
| 305 | contentHashes: new Map(metadata.map(record => [record.objectID, record.contentHash])), |
| 306 | }); |
| 307 | } |