@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
ts 307 lines 11.5 KB
Raw
1 import {
2 embeddingContentHash,
3 embeddingText,
4 KB_EMBEDDING_DIMENSIONS,
5 KB_EMBEDDING_MODEL,
6 KB_EMBEDDING_PROVIDER,
7 } from './embeddings';
8 import { createHash } from 'node:crypto';
9 import type { AlgoliaDocsRecord } from '@/lib/search-index';
10 import {
11 KNOWLEDGE_SOURCE_LABELS,
12 type KnowledgeSourceType,
13 type ProductAreaSlug,
14 } from './types';
15
16 export interface KnowledgeSemanticRecord {
17 objectID: string;
18 sourceType: Extract<KnowledgeSourceType, 'docs' | 'kb'>;
19 sourceLabel: string;
20 pageID: string;
21 title: string;
22 section: string | null;
23 description: string;
24 content: string;
25 canonicalUrl: string;
26 breadcrumbs: string[];
27 productAreas: ProductAreaSlug[];
28 toolkitSlugs: string[];
29 keywords: string[];
30 slug: string;
31 toolNames: string[];
32 toolSlugs: string[];
33 pageRank: number;
34 sectionRank: number;
35 lastVerifiedAt: string | null;
36 contentHash: string;
37 visibility: 'public';
38 }
39
40 /** @deprecated Use KnowledgeSemanticRecord. */
41 export type KbSemanticRecord = KnowledgeSemanticRecord;
42
43 export interface KnowledgeSemanticArtifact {
44 formatVersion: 2;
45 provider: typeof KB_EMBEDDING_PROVIDER;
46 model: typeof KB_EMBEDDING_MODEL;
47 dimensions: number;
48 source: {
49 docsContentHash: string;
50 };
51 builtAt: string;
52 records: KnowledgeSemanticRecord[];
53 vectorsBase64: string;
54 }
55
56 /** @deprecated Use KnowledgeSemanticArtifact. */
57 export type KbSemanticArtifact = KnowledgeSemanticArtifact;
58
59 export interface RankedSemanticCandidate {
60 record: KnowledgeSemanticRecord;
61 rank: number;
62 similarity: number;
63 }
64
65 export function semanticRecordFromSearchRecord(record: AlgoliaDocsRecord): KnowledgeSemanticRecord {
66 if (record.source_type !== 'kb' && record.source_type !== 'docs') {
67 throw new Error(`Only public docs and KB records can be embedded: ${record.objectID}`);
68 }
69 return {
70 objectID: record.objectID,
71 sourceType: record.source_type,
72 sourceLabel: KNOWLEDGE_SOURCE_LABELS[record.source_type],
73 pageID: record.page_id,
74 title: record.title,
75 section: record.section ?? null,
76 description: record.description ?? '',
77 content: record.content,
78 canonicalUrl: record.canonical_url,
79 breadcrumbs: record.breadcrumbs ?? [],
80 productAreas: record.product_areas,
81 toolkitSlugs: record.toolkit_slugs,
82 keywords: record.keywords ?? [],
83 slug: record.slug ?? '',
84 toolNames: record.tool_names ?? [],
85 toolSlugs: record.tool_slugs ?? [],
86 pageRank: record.page_rank,
87 sectionRank: record.section_rank,
88 lastVerifiedAt: record.last_verified_at,
89 contentHash: embeddingContentHash(record),
90 visibility: 'public',
91 };
92 }
93
94 export function docsContentHashFromRecords(records: readonly KnowledgeSemanticRecord[]): string {
95 const content = records
96 .filter(record => record.sourceType === 'docs')
97 .sort((left, right) => left.objectID.localeCompare(right.objectID))
98 .map(record => `${record.objectID}\u0000${record.contentHash}`)
99 .join('\n');
100 return createHash('sha256').update(content).digest('hex');
101 }
102
103 export function encodeVectors(vectors: number[][]): string {
104 const dimensions = vectors[0]?.length ?? 0;
105 if (dimensions === 0 || vectors.some(vector => vector.length !== dimensions)) {
106 throw new Error('Semantic vectors must have one consistent non-zero dimension');
107 }
108 const values = new Float32Array(vectors.length * dimensions);
109 let offset = 0;
110 for (const vector of vectors) {
111 for (const value of vector) {
112 if (!Number.isFinite(value)) throw new Error('Semantic vector contains a non-finite value');
113 values[offset++] = value;
114 }
115 }
116 return Buffer.from(values.buffer, values.byteOffset, values.byteLength).toString('base64');
117 }
118
119 export function decodeVectors(
120 vectorsBase64: string,
121 recordCount: number,
122 dimensions: number,
123 ): Float32Array {
124 const bytes = Buffer.from(vectorsBase64, 'base64');
125 const expectedBytes = recordCount * dimensions * Float32Array.BYTES_PER_ELEMENT;
126 if (bytes.byteLength !== expectedBytes) {
127 throw new Error(`Semantic vector byte length mismatch: expected ${expectedBytes}, got ${bytes.byteLength}`);
128 }
129 const values = new Float32Array(recordCount * dimensions);
130 const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
131 for (let index = 0; index < values.length; index += 1) {
132 values[index] = view.getFloat32(index * Float32Array.BYTES_PER_ELEMENT, true);
133 }
134 return values;
135 }
136
137 /** A valid artifact that needs to be rebuilt for the current corpus. */
138 export class SemanticArtifactStaleError extends Error {}
139
140 export function validateSemanticArtifact(
141 artifact: KnowledgeSemanticArtifact,
142 expected: {
143 dimensions?: number;
144 docsContentHash?: string;
145 contentHashes?: ReadonlyMap<string, string>;
146 },
147 ): KnowledgeSemanticArtifact {
148 if (artifact.formatVersion !== 2) throw new Error('Semantic artifact format mismatch');
149 if (artifact.provider !== KB_EMBEDDING_PROVIDER) throw new Error('Semantic artifact provider mismatch');
150 if (artifact.model !== KB_EMBEDDING_MODEL) throw new Error('Semantic artifact model mismatch');
151 if (artifact.dimensions !== (expected.dimensions ?? KB_EMBEDDING_DIMENSIONS)) {
152 throw new Error('Semantic artifact dimension mismatch');
153 }
154 if (artifact.records.length === 0) throw new Error('Semantic artifact has no records');
155
156 const seen = new Set<string>();
157 for (const record of artifact.records) {
158 if (record.sourceType !== 'docs' && record.sourceType !== 'kb') {
159 throw new Error(`Semantic record ${record.objectID} has an invalid source type`);
160 }
161 if (record.visibility !== 'public') throw new Error(`Semantic record ${record.objectID} is not public`);
162 if (!record.objectID || seen.has(record.objectID)) {
163 throw new Error(`Semantic artifact has duplicate object ID: ${record.objectID}`);
164 }
165 seen.add(record.objectID);
166 }
167
168 const vectors = decodeVectors(artifact.vectorsBase64, artifact.records.length, artifact.dimensions);
169 for (let row = 0; row < artifact.records.length; row += 1) {
170 let squaredNorm = 0;
171 const start = row * artifact.dimensions;
172 for (let column = 0; column < artifact.dimensions; column += 1) {
173 const value = vectors[start + column];
174 if (!Number.isFinite(value)) throw new Error('Semantic artifact contains a non-finite vector');
175 squaredNorm += value * value;
176 }
177 if (Math.abs(Math.sqrt(squaredNorm) - 1) > 0.002) {
178 throw new Error(`Semantic vector for ${artifact.records[row]?.objectID} is not normalized`);
179 }
180 }
181 // Check integrity first so advisory freshness checks cannot hide corrupt artifacts.
182 if (expected.docsContentHash && artifact.source.docsContentHash !== expected.docsContentHash) {
183 throw new SemanticArtifactStaleError('Semantic artifact docs content hash mismatch');
184 }
185 if (expected.contentHashes) {
186 for (const record of artifact.records) {
187 if (expected.contentHashes.get(record.objectID) !== record.contentHash) {
188 throw new SemanticArtifactStaleError(`Semantic artifact content hash mismatch for ${record.objectID}`);
189 }
190 }
191 if (expected.contentHashes.size !== artifact.records.length) {
192 throw new SemanticArtifactStaleError('Semantic artifact content hash record count mismatch');
193 }
194 }
195 return artifact;
196 }
197
198 function normalizeQueryVector(queryVector: number[], dimensions: number): number[] {
199 if (queryVector.length !== dimensions) throw new Error('Semantic query vector dimension mismatch');
200 if (queryVector.some(value => !Number.isFinite(value))) {
201 throw new Error('Semantic query vector contains a non-finite value');
202 }
203 const norm = Math.sqrt(queryVector.reduce((total, value) => total + value * value, 0));
204 if (norm === 0) throw new Error('Semantic query vector has zero norm');
205 return queryVector.map(value => value / norm);
206 }
207
208 export function rankSemanticCandidates(
209 artifact: KnowledgeSemanticArtifact,
210 queryVector: number[],
211 limit: number,
212 options?: { minimumSimilarity?: number },
213 ): RankedSemanticCandidate[] {
214 const query = normalizeQueryVector(queryVector, artifact.dimensions);
215 const vectors = decodeVectors(artifact.vectorsBase64, artifact.records.length, artifact.dimensions);
216 const scored = artifact.records.map((record, row) => {
217 let similarity = 0;
218 const start = row * artifact.dimensions;
219 for (let column = 0; column < artifact.dimensions; column += 1) {
220 similarity += query[column]! * vectors[start + column]!;
221 }
222 return { record, similarity };
223 });
224 scored.sort((left, right) =>
225 right.similarity - left.similarity || left.record.objectID.localeCompare(right.record.objectID),
226 );
227 const minimumSimilarity = options?.minimumSimilarity ?? Number.NEGATIVE_INFINITY;
228 return scored
229 .filter(candidate => candidate.similarity >= minimumSimilarity)
230 .slice(0, Math.max(0, limit)).map((candidate, index) => ({
231 ...candidate,
232 rank: index + 1,
233 }));
234 }
235
236 export async function buildSemanticArtifact(input: {
237 records: AlgoliaDocsRecord[];
238 docsContentHash?: string;
239 builtAt: string;
240 previousArtifact?: KnowledgeSemanticArtifact;
241 embed: (texts: string[]) => Promise<number[][]>;
242 }): Promise<KnowledgeSemanticArtifact> {
243 const records = [...input.records]
244 .sort((left, right) => left.objectID.localeCompare(right.objectID));
245 const metadata = records.map(semanticRecordFromSearchRecord);
246 const docsContentHash = input.docsContentHash ?? docsContentHashFromRecords(metadata);
247 const previousVectors = input.previousArtifact &&
248 input.previousArtifact.provider === KB_EMBEDDING_PROVIDER &&
249 input.previousArtifact.model === KB_EMBEDDING_MODEL &&
250 input.previousArtifact.dimensions === KB_EMBEDDING_DIMENSIONS
251 ? decodeVectors(
252 input.previousArtifact.vectorsBase64,
253 input.previousArtifact.records.length,
254 input.previousArtifact.dimensions,
255 )
256 : null;
257 const previousRows = new Map(
258 input.previousArtifact?.records.map((record, index) => [record.objectID, { record, index }]) ?? [],
259 );
260
261 const vectors = new Array<number[] | undefined>(records.length);
262 const missingRows: number[] = [];
263 for (let index = 0; index < metadata.length; index += 1) {
264 const current = metadata[index]!;
265 const previous = previousRows.get(current.objectID);
266 if (previous && previousVectors && previous.record.contentHash === current.contentHash) {
267 const start = previous.index * KB_EMBEDDING_DIMENSIONS;
268 vectors[index] = Array.from(
269 previousVectors.subarray(start, start + KB_EMBEDDING_DIMENSIONS),
270 );
271 } else {
272 missingRows.push(index);
273 }
274 }
275
276 if (missingRows.length > 0) {
277 const embedded = await input.embed(missingRows.map(index => embeddingText(records[index]!)));
278 if (embedded.length !== missingRows.length) {
279 throw new Error('Embedding builder result count mismatch');
280 }
281 for (let index = 0; index < missingRows.length; index += 1) {
282 const vector = embedded[index];
283 if (!vector || vector.length !== KB_EMBEDDING_DIMENSIONS) {
284 throw new Error('Embedding builder dimension mismatch');
285 }
286 vectors[missingRows[index]!] = vector;
287 }
288 }
289
290 if (vectors.some(vector => vector === undefined)) {
291 throw new Error('Embedding builder left a record without a vector');
292 }
293 const artifact: KnowledgeSemanticArtifact = {
294 formatVersion: 2,
295 provider: KB_EMBEDDING_PROVIDER,
296 model: KB_EMBEDDING_MODEL,
297 dimensions: KB_EMBEDDING_DIMENSIONS,
298 source: { docsContentHash },
299 builtAt: input.builtAt,
300 records: metadata,
301 vectorsBase64: encodeVectors(vectors as number[][]),
302 };
303 return validateSemanticArtifact(artifact, {
304 docsContentHash,
305 contentHashes: new Map(metadata.map(record => [record.objectID, record.contentHash])),
306 });
307 }