@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
ts 334 lines 11.5 KB
Raw
1 import { describe, expect, test } from 'bun:test';
2 import type { AlgoliaDocsRecord } from '@/lib/search-index';
3 import {
4 embeddingContentHash,
5 embeddingText,
6 embedTexts,
7 } from '@/lib/knowledge/embeddings';
8 import {
9 buildSemanticArtifact,
10 decodeVectors,
11 encodeVectors,
12 rankSemanticCandidates,
13 semanticRecordFromSearchRecord,
14 SemanticArtifactStaleError,
15 validateSemanticArtifact,
16 type KnowledgeSemanticArtifact,
17 type KnowledgeSemanticRecord,
18 } from '@/lib/knowledge/semantic-artifact';
19
20 function searchRecord(overrides: Partial<AlgoliaDocsRecord> = {}): AlgoliaDocsRecord {
21 return {
22 objectID: '/kb/guide/github__tokens__0__abc',
23 title: 'GitHub troubleshooting',
24 description: 'Known-good GitHub setup and troubleshooting guidance.',
25 breadcrumbs: ['Knowledge Base', 'GitHub'],
26 url: '/kb/guide/github#tokens',
27 page_id: '/kb/guide/github',
28 section: 'Tokens are redacted',
29 section_id: 'tokens-are-redacted',
30 content: 'Provider tokens are redacted from connected-account responses.',
31 keywords: ['github', 'connected accounts'],
32 slug: 'kb guide github',
33 headings: ['Tokens are redacted'],
34 tool_names: ['Get authenticated user'],
35 tool_slugs: ['GITHUB_GET_THE_AUTHENTICATED_USER'],
36 type: 'kb',
37 lang: 'en',
38 page_rank: 1_900,
39 toolkit_popularity: 0,
40 section_rank: 96,
41 position: 0,
42 depth: 2,
43 source_type: 'kb',
44 canonical_url: '/kb/guide/github#tokens-are-redacted',
45 product_areas: ['connected-accounts'],
46 toolkit_slugs: ['github'],
47 intents: ['troubleshoot'],
48 last_verified_at: '2026-08-12',
49 ...overrides,
50 };
51 }
52
53 function semanticRecord(
54 objectID: string,
55 title: string,
56 contentHash: string,
57 ): KnowledgeSemanticRecord {
58 return {
59 objectID,
60 sourceType: 'kb',
61 sourceLabel: 'Knowledge Base',
62 pageID: `/kb/guide/${objectID}`,
63 title,
64 section: title,
65 description: `${title} description`,
66 content: `${title} content`,
67 canonicalUrl: `/kb/guide/${objectID}#answer`,
68 breadcrumbs: ['Knowledge Base'],
69 productAreas: [],
70 toolkitSlugs: [],
71 keywords: [],
72 slug: objectID,
73 toolNames: [],
74 toolSlugs: [],
75 pageRank: 1_900,
76 sectionRank: 96,
77 lastVerifiedAt: '2026-08-12',
78 contentHash,
79 visibility: 'public',
80 };
81 }
82
83 function artifact(): KnowledgeSemanticArtifact {
84 return {
85 formatVersion: 2,
86 provider: 'openai',
87 model: 'text-embedding-3-small',
88 dimensions: 2,
89 source: { docsContentHash: 'docs-hash' },
90 builtAt: '2026-08-17T00:00:00.000Z',
91 records: [
92 semanticRecord('exact-x', 'Exact X', 'hash-x'),
93 semanticRecord('diagonal', 'Diagonal', 'hash-d'),
94 semanticRecord('exact-y', 'Exact Y', 'hash-y'),
95 ],
96 vectorsBase64: encodeVectors([
97 [1, 0],
98 [Math.SQRT1_2, Math.SQRT1_2],
99 [0, 1],
100 ]),
101 };
102 }
103
104 describe('KB embeddings', () => {
105 test('builds stable labeled embedding text from retrieval fields', () => {
106 const record = searchRecord();
107
108 expect(embeddingText(record)).toBe(`Title: GitHub troubleshooting
109 Section: Tokens are redacted
110 Description: Known-good GitHub setup and troubleshooting guidance.
111 Aliases and exact terms: github | connected accounts | kb guide github | Get authenticated user | GITHUB_GET_THE_AUTHENTICATED_USER
112 Toolkits: github
113 Content: Provider tokens are redacted from connected-account responses.`);
114 expect(embeddingContentHash(record)).toBe(
115 '09245e80265bd75ee43ed2b5ba1db149ff10077e8c8edc61f3ba2d7c82bc624b',
116 );
117 });
118
119 test('sends the fixed production model contract and restores response index order', async () => {
120 let requestBody: unknown;
121 const vectors = await embedTexts(['first', 'second'], {
122 apiKey: 'test-key',
123 fetch: async (_input, init) => {
124 requestBody = JSON.parse(String(init?.body));
125 return Response.json({
126 data: [
127 { index: 1, embedding: [0, 2] },
128 { index: 0, embedding: [3, 4] },
129 ],
130 });
131 },
132 });
133
134 expect(requestBody).toEqual({
135 model: 'text-embedding-3-small',
136 dimensions: 256,
137 encoding_format: 'float',
138 input: ['first', 'second'],
139 });
140 expect(vectors[0]?.[0]).toBeCloseTo(0.6);
141 expect(vectors[0]?.[1]).toBeCloseTo(0.8);
142 expect(vectors[1]).toEqual([0, 1]);
143 });
144
145 test('forwards cancellation to the OpenAI embedding request', async () => {
146 const controller = new AbortController();
147 let requestSignal: AbortSignal | null | undefined;
148
149 await embedTexts(['github oauth'], {
150 apiKey: 'test-key',
151 signal: controller.signal,
152 fetch: async (_input, init) => {
153 requestSignal = init?.signal;
154 return Response.json({ data: [{ index: 0, embedding: [1, 0] }] });
155 },
156 });
157
158 expect(requestSignal).toBe(controller.signal);
159 });
160 });
161
162 describe('KB semantic artifact', () => {
163 test.each([
164 { docsContentHash: 'new-docs-hash' },
165 { contentHashes: new Map([['exact-x', 'changed-hash']]) },
166 { contentHashes: new Map([
167 ['exact-x', 'hash-x'], ['diagonal', 'hash-d'], ['exact-y', 'hash-y'], ['new', 'hash-new'],
168 ]) },
169 ])('classifies corpus drift as stale: %j', drift => {
170 expect(() => validateSemanticArtifact(artifact(), {
171 dimensions: 2,
172 ...drift,
173 })).toThrow(SemanticArtifactStaleError);
174 });
175
176 test.each([
177 { model: 'another-model' },
178 { vectorsBase64: encodeVectors([[2, 0], [0, 1], [0, 1]]) },
179 { vectorsBase64: encodeVectors([[1, 0]]) },
180 { records: [] },
181 { records: [semanticRecord('duplicate', 'A', 'a'), semanticRecord('duplicate', 'B', 'b')] },
182 { records: [{ ...semanticRecord('private', 'Private', 'hash'), visibility: 'private' }] },
183 ])('rejects corruption even when the corpus is stale: %j', corruption => {
184 let caught: unknown;
185 try {
186 validateSemanticArtifact({ ...artifact(), ...corruption } as KnowledgeSemanticArtifact, {
187 dimensions: 2,
188 docsContentHash: 'new-docs-hash',
189 });
190 } catch (error) {
191 caught = error;
192 }
193 expect(caught).toBeInstanceOf(Error);
194 expect(caught).not.toBeInstanceOf(SemanticArtifactStaleError);
195 });
196
197 test('round-trips row-major float32 vectors', () => {
198 const encoded = encodeVectors([[1, 0], [0.25, -0.5]]);
199 expect([...decodeVectors(encoded, 2, 2)]).toEqual([1, 0, 0.25, -0.5]);
200 });
201
202 test('validates model, source, public records, hashes, and normalized vectors', () => {
203 const value = artifact();
204 expect(validateSemanticArtifact(value, {
205 dimensions: 2,
206 docsContentHash: 'docs-hash',
207 contentHashes: new Map([
208 ['exact-x', 'hash-x'],
209 ['diagonal', 'hash-d'],
210 ['exact-y', 'hash-y'],
211 ]),
212 })).toBe(value);
213
214 expect(() => validateSemanticArtifact(
215 { ...value, model: 'another-model' },
216 { dimensions: 2, docsContentHash: 'docs-hash' },
217 )).toThrow('model mismatch');
218 expect(() => validateSemanticArtifact(value, {
219 dimensions: 2,
220 docsContentHash: 'docs-hash',
221 contentHashes: new Map([
222 ['exact-x', 'tampered'],
223 ['diagonal', 'hash-d'],
224 ['exact-y', 'hash-y'],
225 ]),
226 })).toThrow('content hash mismatch');
227
228 const nonUnit = { ...value, vectorsBase64: encodeVectors([[2, 0], [0, 1], [0, 1]]) };
229 expect(() => validateSemanticArtifact(
230 nonUnit,
231 { dimensions: 2, docsContentHash: 'docs-hash' },
232 )).toThrow('not normalized');
233 expect(() => validateSemanticArtifact(value, {
234 dimensions: 2,
235 docsContentHash: 'tampered-docs-hash',
236 })).toThrow('docs content hash mismatch');
237
238 const parsed = JSON.parse(JSON.stringify(value)) as KnowledgeSemanticArtifact;
239 parsed.records[0]!.sourceType = 'toolkit' as never;
240 expect(() => validateSemanticArtifact(parsed, {
241 dimensions: 2,
242 docsContentHash: 'docs-hash',
243 })).toThrow('source type');
244 });
245
246 test('ranks every public record by exact cosine similarity', () => {
247 const ranked = rankSemanticCandidates(artifact(), [1, 0], 2);
248
249 expect(ranked.map(candidate => candidate.record.objectID)).toEqual(['exact-x', 'diagonal']);
250 expect(ranked[0]?.similarity).toBeCloseTo(1);
251 expect(ranked[1]?.similarity).toBeCloseTo(Math.SQRT1_2);
252 expect(ranked.map(candidate => candidate.rank)).toEqual([1, 2]);
253 });
254
255 test('omits candidates below the caller semantic confidence floor', () => {
256 const ranked = rankSemanticCandidates(artifact(), [1, 0], 3, {
257 minimumSimilarity: 0.8,
258 });
259
260 expect(ranked.map(candidate => candidate.record.objectID)).toEqual(['exact-x']);
261 expect(ranked[0]?.rank).toBe(1);
262 });
263
264 test('maps public docs and KB records into the artifact contract', () => {
265 expect(semanticRecordFromSearchRecord(searchRecord())).toEqual({
266 objectID: '/kb/guide/github__tokens__0__abc',
267 sourceType: 'kb',
268 sourceLabel: 'Knowledge Base',
269 pageID: '/kb/guide/github',
270 title: 'GitHub troubleshooting',
271 section: 'Tokens are redacted',
272 description: 'Known-good GitHub setup and troubleshooting guidance.',
273 content: 'Provider tokens are redacted from connected-account responses.',
274 canonicalUrl: '/kb/guide/github#tokens-are-redacted',
275 breadcrumbs: ['Knowledge Base', 'GitHub'],
276 productAreas: ['connected-accounts'],
277 toolkitSlugs: ['github'],
278 keywords: ['github', 'connected accounts'],
279 slug: 'kb guide github',
280 toolNames: ['Get authenticated user'],
281 toolSlugs: ['GITHUB_GET_THE_AUTHENTICATED_USER'],
282 pageRank: 1_900,
283 sectionRank: 96,
284 lastVerifiedAt: '2026-08-12',
285 contentHash: '09245e80265bd75ee43ed2b5ba1db149ff10077e8c8edc61f3ba2d7c82bc624b',
286 visibility: 'public',
287 });
288 expect(semanticRecordFromSearchRecord(searchRecord({
289 objectID: '/docs/claude',
290 source_type: 'docs',
291 title: 'Connect Claude',
292 }))).toMatchObject({ sourceType: 'docs', sourceLabel: 'Docs' });
293 expect(() => semanticRecordFromSearchRecord(searchRecord({ source_type: 'toolkit' }))).toThrow(
294 'Only public docs and KB records can be embedded',
295 );
296 expect(() => semanticRecordFromSearchRecord(searchRecord({ source_type: 'reference' }))).toThrow(
297 'Only public docs and KB records can be embedded',
298 );
299 });
300
301 test('reuses unchanged vectors and embeds only changed records', async () => {
302 const unchanged = searchRecord({ objectID: 'a', title: 'A' });
303 const changed = searchRecord({ objectID: 'b', title: 'B', content: 'new body' });
304 const unchangedMetadata = semanticRecordFromSearchRecord(unchanged);
305 const prior: KnowledgeSemanticArtifact = {
306 formatVersion: 2,
307 provider: 'openai',
308 model: 'text-embedding-3-small',
309 dimensions: 256,
310 source: { docsContentHash: 'old-docs-hash' },
311 builtAt: '2026-08-16T00:00:00.000Z',
312 records: [unchangedMetadata],
313 vectorsBase64: encodeVectors([[1, ...Array.from({ length: 255 }, () => 0)]]),
314 };
315 const embeddedTexts: string[] = [];
316
317 const built = await buildSemanticArtifact({
318 records: [changed, unchanged],
319 docsContentHash: 'new-docs-hash',
320 builtAt: '2026-08-17T00:00:00.000Z',
321 previousArtifact: prior,
322 embed: async texts => {
323 embeddedTexts.push(...texts);
324 return texts.map(() => [0, 1, ...Array.from({ length: 254 }, () => 0)]);
325 },
326 });
327
328 expect(embeddedTexts).toEqual([embeddingText(changed)]);
329 expect(built.records.map(record => record.objectID)).toEqual(['a', 'b']);
330 const vectors = decodeVectors(built.vectorsBase64, 2, 256);
331 expect([vectors[0], vectors[1], vectors[256], vectors[257]]).toEqual([1, 0, 0, 1]);
332 expect(built.source).toEqual({ docsContentHash: 'new-docs-hash' });
333 });
334 });