Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 03:37 UTC
| 1 | import { describe, expect, test } from 'bun:test'; |
| 2 | import type { AlgoliaDocsRecord } from '@/lib/search-index'; |
| 3 | import { |
| 4 | embeddingContentHash, |
| 5 | embeddingText, |
| 6 | embedTexts, |
| 7 | } from '@/lib/knowledge/embeddings'; |
| 8 | import { |
| 9 | buildSemanticArtifact, |
| 10 | decodeVectors, |
| 11 | encodeVectors, |
| 12 | rankSemanticCandidates, |
| 13 | semanticRecordFromSearchRecord, |
| 14 | SemanticArtifactStaleError, |
| 15 | validateSemanticArtifact, |
| 16 | type KnowledgeSemanticArtifact, |
| 17 | type KnowledgeSemanticRecord, |
| 18 | } from '@/lib/knowledge/semantic-artifact'; |
| 19 | |
| 20 | function searchRecord(overrides: Partial<AlgoliaDocsRecord> = {}): AlgoliaDocsRecord { |
| 21 | return { |
| 22 | objectID: '/kb/guide/github__tokens__0__abc', |
| 23 | title: 'GitHub troubleshooting', |
| 24 | description: 'Known-good GitHub setup and troubleshooting guidance.', |
| 25 | breadcrumbs: ['Knowledge Base', 'GitHub'], |
| 26 | url: '/kb/guide/github#tokens', |
| 27 | page_id: '/kb/guide/github', |
| 28 | section: 'Tokens are redacted', |
| 29 | section_id: 'tokens-are-redacted', |
| 30 | content: 'Provider tokens are redacted from connected-account responses.', |
| 31 | keywords: ['github', 'connected accounts'], |
| 32 | slug: 'kb guide github', |
| 33 | headings: ['Tokens are redacted'], |
| 34 | tool_names: ['Get authenticated user'], |
| 35 | tool_slugs: ['GITHUB_GET_THE_AUTHENTICATED_USER'], |
| 36 | type: 'kb', |
| 37 | lang: 'en', |
| 38 | page_rank: 1_900, |
| 39 | toolkit_popularity: 0, |
| 40 | section_rank: 96, |
| 41 | position: 0, |
| 42 | depth: 2, |
| 43 | source_type: 'kb', |
| 44 | canonical_url: '/kb/guide/github#tokens-are-redacted', |
| 45 | product_areas: ['connected-accounts'], |
| 46 | toolkit_slugs: ['github'], |
| 47 | intents: ['troubleshoot'], |
| 48 | last_verified_at: '2026-08-12', |
| 49 | ...overrides, |
| 50 | }; |
| 51 | } |
| 52 | |
| 53 | function semanticRecord( |
| 54 | objectID: string, |
| 55 | title: string, |
| 56 | contentHash: string, |
| 57 | ): KnowledgeSemanticRecord { |
| 58 | return { |
| 59 | objectID, |
| 60 | sourceType: 'kb', |
| 61 | sourceLabel: 'Knowledge Base', |
| 62 | pageID: `/kb/guide/${objectID}`, |
| 63 | title, |
| 64 | section: title, |
| 65 | description: `${title} description`, |
| 66 | content: `${title} content`, |
| 67 | canonicalUrl: `/kb/guide/${objectID}#answer`, |
| 68 | breadcrumbs: ['Knowledge Base'], |
| 69 | productAreas: [], |
| 70 | toolkitSlugs: [], |
| 71 | keywords: [], |
| 72 | slug: objectID, |
| 73 | toolNames: [], |
| 74 | toolSlugs: [], |
| 75 | pageRank: 1_900, |
| 76 | sectionRank: 96, |
| 77 | lastVerifiedAt: '2026-08-12', |
| 78 | contentHash, |
| 79 | visibility: 'public', |
| 80 | }; |
| 81 | } |
| 82 | |
| 83 | function artifact(): KnowledgeSemanticArtifact { |
| 84 | return { |
| 85 | formatVersion: 2, |
| 86 | provider: 'openai', |
| 87 | model: 'text-embedding-3-small', |
| 88 | dimensions: 2, |
| 89 | source: { docsContentHash: 'docs-hash' }, |
| 90 | builtAt: '2026-08-17T00:00:00.000Z', |
| 91 | records: [ |
| 92 | semanticRecord('exact-x', 'Exact X', 'hash-x'), |
| 93 | semanticRecord('diagonal', 'Diagonal', 'hash-d'), |
| 94 | semanticRecord('exact-y', 'Exact Y', 'hash-y'), |
| 95 | ], |
| 96 | vectorsBase64: encodeVectors([ |
| 97 | [1, 0], |
| 98 | [Math.SQRT1_2, Math.SQRT1_2], |
| 99 | [0, 1], |
| 100 | ]), |
| 101 | }; |
| 102 | } |
| 103 | |
| 104 | describe('KB embeddings', () => { |
| 105 | test('builds stable labeled embedding text from retrieval fields', () => { |
| 106 | const record = searchRecord(); |
| 107 | |
| 108 | expect(embeddingText(record)).toBe(`Title: GitHub troubleshooting |
| 109 | Section: Tokens are redacted |
| 110 | Description: Known-good GitHub setup and troubleshooting guidance. |
| 111 | Aliases and exact terms: github | connected accounts | kb guide github | Get authenticated user | GITHUB_GET_THE_AUTHENTICATED_USER |
| 112 | Toolkits: github |
| 113 | Content: Provider tokens are redacted from connected-account responses.`); |
| 114 | expect(embeddingContentHash(record)).toBe( |
| 115 | '09245e80265bd75ee43ed2b5ba1db149ff10077e8c8edc61f3ba2d7c82bc624b', |
| 116 | ); |
| 117 | }); |
| 118 | |
| 119 | test('sends the fixed production model contract and restores response index order', async () => { |
| 120 | let requestBody: unknown; |
| 121 | const vectors = await embedTexts(['first', 'second'], { |
| 122 | apiKey: 'test-key', |
| 123 | fetch: async (_input, init) => { |
| 124 | requestBody = JSON.parse(String(init?.body)); |
| 125 | return Response.json({ |
| 126 | data: [ |
| 127 | { index: 1, embedding: [0, 2] }, |
| 128 | { index: 0, embedding: [3, 4] }, |
| 129 | ], |
| 130 | }); |
| 131 | }, |
| 132 | }); |
| 133 | |
| 134 | expect(requestBody).toEqual({ |
| 135 | model: 'text-embedding-3-small', |
| 136 | dimensions: 256, |
| 137 | encoding_format: 'float', |
| 138 | input: ['first', 'second'], |
| 139 | }); |
| 140 | expect(vectors[0]?.[0]).toBeCloseTo(0.6); |
| 141 | expect(vectors[0]?.[1]).toBeCloseTo(0.8); |
| 142 | expect(vectors[1]).toEqual([0, 1]); |
| 143 | }); |
| 144 | |
| 145 | test('forwards cancellation to the OpenAI embedding request', async () => { |
| 146 | const controller = new AbortController(); |
| 147 | let requestSignal: AbortSignal | null | undefined; |
| 148 | |
| 149 | await embedTexts(['github oauth'], { |
| 150 | apiKey: 'test-key', |
| 151 | signal: controller.signal, |
| 152 | fetch: async (_input, init) => { |
| 153 | requestSignal = init?.signal; |
| 154 | return Response.json({ data: [{ index: 0, embedding: [1, 0] }] }); |
| 155 | }, |
| 156 | }); |
| 157 | |
| 158 | expect(requestSignal).toBe(controller.signal); |
| 159 | }); |
| 160 | }); |
| 161 | |
| 162 | describe('KB semantic artifact', () => { |
| 163 | test.each([ |
| 164 | { docsContentHash: 'new-docs-hash' }, |
| 165 | { contentHashes: new Map([['exact-x', 'changed-hash']]) }, |
| 166 | { contentHashes: new Map([ |
| 167 | ['exact-x', 'hash-x'], ['diagonal', 'hash-d'], ['exact-y', 'hash-y'], ['new', 'hash-new'], |
| 168 | ]) }, |
| 169 | ])('classifies corpus drift as stale: %j', drift => { |
| 170 | expect(() => validateSemanticArtifact(artifact(), { |
| 171 | dimensions: 2, |
| 172 | ...drift, |
| 173 | })).toThrow(SemanticArtifactStaleError); |
| 174 | }); |
| 175 | |
| 176 | test.each([ |
| 177 | { model: 'another-model' }, |
| 178 | { vectorsBase64: encodeVectors([[2, 0], [0, 1], [0, 1]]) }, |
| 179 | { vectorsBase64: encodeVectors([[1, 0]]) }, |
| 180 | { records: [] }, |
| 181 | { records: [semanticRecord('duplicate', 'A', 'a'), semanticRecord('duplicate', 'B', 'b')] }, |
| 182 | { records: [{ ...semanticRecord('private', 'Private', 'hash'), visibility: 'private' }] }, |
| 183 | ])('rejects corruption even when the corpus is stale: %j', corruption => { |
| 184 | let caught: unknown; |
| 185 | try { |
| 186 | validateSemanticArtifact({ ...artifact(), ...corruption } as KnowledgeSemanticArtifact, { |
| 187 | dimensions: 2, |
| 188 | docsContentHash: 'new-docs-hash', |
| 189 | }); |
| 190 | } catch (error) { |
| 191 | caught = error; |
| 192 | } |
| 193 | expect(caught).toBeInstanceOf(Error); |
| 194 | expect(caught).not.toBeInstanceOf(SemanticArtifactStaleError); |
| 195 | }); |
| 196 | |
| 197 | test('round-trips row-major float32 vectors', () => { |
| 198 | const encoded = encodeVectors([[1, 0], [0.25, -0.5]]); |
| 199 | expect([...decodeVectors(encoded, 2, 2)]).toEqual([1, 0, 0.25, -0.5]); |
| 200 | }); |
| 201 | |
| 202 | test('validates model, source, public records, hashes, and normalized vectors', () => { |
| 203 | const value = artifact(); |
| 204 | expect(validateSemanticArtifact(value, { |
| 205 | dimensions: 2, |
| 206 | docsContentHash: 'docs-hash', |
| 207 | contentHashes: new Map([ |
| 208 | ['exact-x', 'hash-x'], |
| 209 | ['diagonal', 'hash-d'], |
| 210 | ['exact-y', 'hash-y'], |
| 211 | ]), |
| 212 | })).toBe(value); |
| 213 | |
| 214 | expect(() => validateSemanticArtifact( |
| 215 | { ...value, model: 'another-model' }, |
| 216 | { dimensions: 2, docsContentHash: 'docs-hash' }, |
| 217 | )).toThrow('model mismatch'); |
| 218 | expect(() => validateSemanticArtifact(value, { |
| 219 | dimensions: 2, |
| 220 | docsContentHash: 'docs-hash', |
| 221 | contentHashes: new Map([ |
| 222 | ['exact-x', 'tampered'], |
| 223 | ['diagonal', 'hash-d'], |
| 224 | ['exact-y', 'hash-y'], |
| 225 | ]), |
| 226 | })).toThrow('content hash mismatch'); |
| 227 | |
| 228 | const nonUnit = { ...value, vectorsBase64: encodeVectors([[2, 0], [0, 1], [0, 1]]) }; |
| 229 | expect(() => validateSemanticArtifact( |
| 230 | nonUnit, |
| 231 | { dimensions: 2, docsContentHash: 'docs-hash' }, |
| 232 | )).toThrow('not normalized'); |
| 233 | expect(() => validateSemanticArtifact(value, { |
| 234 | dimensions: 2, |
| 235 | docsContentHash: 'tampered-docs-hash', |
| 236 | })).toThrow('docs content hash mismatch'); |
| 237 | |
| 238 | const parsed = JSON.parse(JSON.stringify(value)) as KnowledgeSemanticArtifact; |
| 239 | parsed.records[0]!.sourceType = 'toolkit' as never; |
| 240 | expect(() => validateSemanticArtifact(parsed, { |
| 241 | dimensions: 2, |
| 242 | docsContentHash: 'docs-hash', |
| 243 | })).toThrow('source type'); |
| 244 | }); |
| 245 | |
| 246 | test('ranks every public record by exact cosine similarity', () => { |
| 247 | const ranked = rankSemanticCandidates(artifact(), [1, 0], 2); |
| 248 | |
| 249 | expect(ranked.map(candidate => candidate.record.objectID)).toEqual(['exact-x', 'diagonal']); |
| 250 | expect(ranked[0]?.similarity).toBeCloseTo(1); |
| 251 | expect(ranked[1]?.similarity).toBeCloseTo(Math.SQRT1_2); |
| 252 | expect(ranked.map(candidate => candidate.rank)).toEqual([1, 2]); |
| 253 | }); |
| 254 | |
| 255 | test('omits candidates below the caller semantic confidence floor', () => { |
| 256 | const ranked = rankSemanticCandidates(artifact(), [1, 0], 3, { |
| 257 | minimumSimilarity: 0.8, |
| 258 | }); |
| 259 | |
| 260 | expect(ranked.map(candidate => candidate.record.objectID)).toEqual(['exact-x']); |
| 261 | expect(ranked[0]?.rank).toBe(1); |
| 262 | }); |
| 263 | |
| 264 | test('maps public docs and KB records into the artifact contract', () => { |
| 265 | expect(semanticRecordFromSearchRecord(searchRecord())).toEqual({ |
| 266 | objectID: '/kb/guide/github__tokens__0__abc', |
| 267 | sourceType: 'kb', |
| 268 | sourceLabel: 'Knowledge Base', |
| 269 | pageID: '/kb/guide/github', |
| 270 | title: 'GitHub troubleshooting', |
| 271 | section: 'Tokens are redacted', |
| 272 | description: 'Known-good GitHub setup and troubleshooting guidance.', |
| 273 | content: 'Provider tokens are redacted from connected-account responses.', |
| 274 | canonicalUrl: '/kb/guide/github#tokens-are-redacted', |
| 275 | breadcrumbs: ['Knowledge Base', 'GitHub'], |
| 276 | productAreas: ['connected-accounts'], |
| 277 | toolkitSlugs: ['github'], |
| 278 | keywords: ['github', 'connected accounts'], |
| 279 | slug: 'kb guide github', |
| 280 | toolNames: ['Get authenticated user'], |
| 281 | toolSlugs: ['GITHUB_GET_THE_AUTHENTICATED_USER'], |
| 282 | pageRank: 1_900, |
| 283 | sectionRank: 96, |
| 284 | lastVerifiedAt: '2026-08-12', |
| 285 | contentHash: '09245e80265bd75ee43ed2b5ba1db149ff10077e8c8edc61f3ba2d7c82bc624b', |
| 286 | visibility: 'public', |
| 287 | }); |
| 288 | expect(semanticRecordFromSearchRecord(searchRecord({ |
| 289 | objectID: '/docs/claude', |
| 290 | source_type: 'docs', |
| 291 | title: 'Connect Claude', |
| 292 | }))).toMatchObject({ sourceType: 'docs', sourceLabel: 'Docs' }); |
| 293 | expect(() => semanticRecordFromSearchRecord(searchRecord({ source_type: 'toolkit' }))).toThrow( |
| 294 | 'Only public docs and KB records can be embedded', |
| 295 | ); |
| 296 | expect(() => semanticRecordFromSearchRecord(searchRecord({ source_type: 'reference' }))).toThrow( |
| 297 | 'Only public docs and KB records can be embedded', |
| 298 | ); |
| 299 | }); |
| 300 | |
| 301 | test('reuses unchanged vectors and embeds only changed records', async () => { |
| 302 | const unchanged = searchRecord({ objectID: 'a', title: 'A' }); |
| 303 | const changed = searchRecord({ objectID: 'b', title: 'B', content: 'new body' }); |
| 304 | const unchangedMetadata = semanticRecordFromSearchRecord(unchanged); |
| 305 | const prior: KnowledgeSemanticArtifact = { |
| 306 | formatVersion: 2, |
| 307 | provider: 'openai', |
| 308 | model: 'text-embedding-3-small', |
| 309 | dimensions: 256, |
| 310 | source: { docsContentHash: 'old-docs-hash' }, |
| 311 | builtAt: '2026-08-16T00:00:00.000Z', |
| 312 | records: [unchangedMetadata], |
| 313 | vectorsBase64: encodeVectors([[1, ...Array.from({ length: 255 }, () => 0)]]), |
| 314 | }; |
| 315 | const embeddedTexts: string[] = []; |
| 316 | |
| 317 | const built = await buildSemanticArtifact({ |
| 318 | records: [changed, unchanged], |
| 319 | docsContentHash: 'new-docs-hash', |
| 320 | builtAt: '2026-08-17T00:00:00.000Z', |
| 321 | previousArtifact: prior, |
| 322 | embed: async texts => { |
| 323 | embeddedTexts.push(...texts); |
| 324 | return texts.map(() => [0, 1, ...Array.from({ length: 254 }, () => 0)]); |
| 325 | }, |
| 326 | }); |
| 327 | |
| 328 | expect(embeddedTexts).toEqual([embeddingText(changed)]); |
| 329 | expect(built.records.map(record => record.objectID)).toEqual(['a', 'b']); |
| 330 | const vectors = decodeVectors(built.vectorsBase64, 2, 256); |
| 331 | expect([vectors[0], vectors[1], vectors[256], vectors[257]]).toEqual([1, 0, 0, 1]); |
| 332 | expect(built.source).toEqual({ docsContentHash: 'new-docs-hash' }); |
| 333 | }); |
| 334 | }); |