Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 03:37 UTC
| 1 | import { describe, expect, test } from 'bun:test'; |
| 2 | import { createKnowledgeSearchHandler } from '@/app/api/knowledge-search/route'; |
| 3 | import { getPostHogPageViewUrl } from '@/components/posthog-provider'; |
| 4 | import { |
| 5 | buildKnowledgeSearchCapture, |
| 6 | redactKnowledgeSearchQuery, |
| 7 | sendKnowledgeSearchAnalytics, |
| 8 | } from '@/lib/knowledge/query-analytics'; |
| 9 | import type { PublicKnowledgeCandidateRecord } from '@/lib/knowledge/hybrid-search'; |
| 10 | |
| 11 | interface CapturedSearchEvent { |
| 12 | query: string; |
| 13 | filter: string; |
| 14 | retrievalMode: string; |
| 15 | resultCount: number; |
| 16 | degradationCategory: string | null; |
| 17 | strongMatch: boolean | null; |
| 18 | statusCode: number; |
| 19 | durationMs: number; |
| 20 | keywordDurationMs: number; |
| 21 | semanticDurationMs: number | null; |
| 22 | resultSourceTypes: string[]; |
| 23 | previewOverlayApplied: boolean; |
| 24 | } |
| 25 | |
| 26 | function weakKeywordCandidate(): PublicKnowledgeCandidateRecord { |
| 27 | return { |
| 28 | objectID: 'weak-keyword', |
| 29 | sourceType: 'kb', |
| 30 | sourceLabel: 'Knowledge Base', |
| 31 | pageID: '/kb/weak-keyword', |
| 32 | title: 'GitHub OAuth guide', |
| 33 | section: null, |
| 34 | description: 'General GitHub OAuth guidance.', |
| 35 | content: 'Reconnect GitHub access when authorization expires.', |
| 36 | canonicalUrl: '/kb/weak-keyword', |
| 37 | breadcrumbs: ['Knowledge Base'], |
| 38 | productAreas: [], |
| 39 | toolkitSlugs: ['github'], |
| 40 | keywords: [], |
| 41 | slug: 'github-oauth-guide', |
| 42 | toolNames: [], |
| 43 | toolSlugs: [], |
| 44 | pageRank: 1_900, |
| 45 | sectionRank: 100, |
| 46 | lastVerifiedAt: '2026-08-18', |
| 47 | }; |
| 48 | } |
| 49 | |
| 50 | describe('knowledge search analytics', () => { |
| 51 | test.each([ |
| 52 | ['x-api-key header', 'x-api-key: api-value-123', 'x-api-key: [REDACTED]'], |
| 53 | ['X-API-KEY environment field', 'X-API-KEY=api-value-123', 'X-API-KEY=[REDACTED]'], |
| 54 | ['X_API_KEY environment field', 'X_API_KEY=api-value-123', 'X_API_KEY=[REDACTED]'], |
| 55 | ])('redacts a %s', (_case, query, expected) => { |
| 56 | expect(redactKnowledgeSearchQuery(query)).toBe(expected); |
| 57 | }); |
| 58 | |
| 59 | test.each([ |
| 60 | ['ordinary error code', 'Error code: NONEXISTENT_VERSION'], |
| 61 | ['ordinary HTTP state', 'HTTP state=failed'], |
| 62 | ])('preserves an %s', (_case, query) => { |
| 63 | expect(redactKnowledgeSearchQuery(query)).toBe(query); |
| 64 | }); |
| 65 | |
| 66 | test('redacts credential fields in raw and URL-shaped queries without deleting prose', () => { |
| 67 | expect(redactKnowledgeSearchQuery( |
| 68 | "refresh_token=refresh-value-123 access-token: access-value-123 client_secret='client-value-123'", |
| 69 | )).toBe( |
| 70 | 'refresh_token=[REDACTED] access-token: [REDACTED] client_secret=[REDACTED]', |
| 71 | ); |
| 72 | expect(redactKnowledgeSearchQuery( |
| 73 | 'https://example.com/callback?refresh-token=refresh-value-123&access_token=access-value-123&client-secret=client-value-123&page=1', |
| 74 | )).toBe( |
| 75 | 'https://example.com/callback?refresh-token=[REDACTED]&access_token=[REDACTED]&client-secret=[REDACTED]&page=1', |
| 76 | ); |
| 77 | expect(redactKnowledgeSearchQuery( |
| 78 | 'https://example.com/callback?authorization=Bearer-value-123&code=oauth-code-123&state=opaque-state-123&page=1', |
| 79 | )).toBe( |
| 80 | 'https://example.com/callback?authorization=[REDACTED]&code=[REDACTED]&state=[REDACTED]&page=1', |
| 81 | ); |
| 82 | expect(redactKnowledgeSearchQuery( |
| 83 | '{"code":"oauth-code-123","state":"opaque-state-123","status":"pending"}', |
| 84 | )).toBe( |
| 85 | '{"code":[REDACTED],"state":[REDACTED],"status":"pending"}', |
| 86 | ); |
| 87 | expect(redactKnowledgeSearchQuery( |
| 88 | 'authorization: Bearer short-secret', |
| 89 | )).toBe( |
| 90 | 'authorization: [REDACTED]', |
| 91 | ); |
| 92 | expect(redactKnowledgeSearchQuery( |
| 93 | 'Authorization code flow keeps state between browser redirects, and refresh token rotation is automatic.', |
| 94 | )).toBe( |
| 95 | 'Authorization code flow keeps state between browser redirects, and refresh token rotation is automatic.', |
| 96 | ); |
| 97 | }); |
| 98 | |
| 99 | test('records each completed API search with its retrieval outcome', async () => { |
| 100 | const events: CapturedSearchEvent[] = []; |
| 101 | const handler = createKnowledgeSearchHandler({ |
| 102 | hybridEnabled: () => true, |
| 103 | searchKeywordCandidates: async () => ({ candidates: [] }), |
| 104 | searchSemanticCandidates: async () => [], |
| 105 | captureSearch: event => events.push(event), |
| 106 | }); |
| 107 | |
| 108 | const response = await handler(new Request( |
| 109 | 'http://localhost/api/knowledge-search?q=why+did+github+oauth+fail&filter=kb', |
| 110 | )); |
| 111 | |
| 112 | expect(response.status).toBe(200); |
| 113 | expect(events).toHaveLength(1); |
| 114 | expect(events[0]).toMatchObject({ |
| 115 | query: 'why did github oauth fail', |
| 116 | filter: 'kb', |
| 117 | retrievalMode: 'hybrid', |
| 118 | resultCount: 0, |
| 119 | degradationCategory: null, |
| 120 | strongMatch: false, |
| 121 | statusCode: 200, |
| 122 | resultSourceTypes: [], |
| 123 | previewOverlayApplied: false, |
| 124 | }); |
| 125 | expect(events[0]?.durationMs).toBeGreaterThanOrEqual(0); |
| 126 | expect(events[0]?.keywordDurationMs).toBeGreaterThanOrEqual(0); |
| 127 | expect(events[0]?.semanticDurationMs).toBeGreaterThanOrEqual(0); |
| 128 | }); |
| 129 | |
| 130 | test('records semantic guardrail hits without failing the search request', async () => { |
| 131 | const events: CapturedSearchEvent[] = []; |
| 132 | const handler = createKnowledgeSearchHandler({ |
| 133 | hybridEnabled: () => true, |
| 134 | searchKeywordCandidates: async () => ({ candidates: [] }), |
| 135 | searchSemanticCandidates: async () => { |
| 136 | throw new Error('semantic search should not run after admission is denied'); |
| 137 | }, |
| 138 | acquireSemanticSearch: () => ({ |
| 139 | allowed: false, |
| 140 | reason: 'semantic-capacity-limited', |
| 141 | }), |
| 142 | captureSearch: event => events.push(event), |
| 143 | }); |
| 144 | |
| 145 | const response = await handler(new Request( |
| 146 | 'http://localhost/api/knowledge-search?q=github+oauth&filter=kb', |
| 147 | )); |
| 148 | |
| 149 | expect(response.status).toBe(200); |
| 150 | const responseBody = await response.json() as Record<string, unknown>; |
| 151 | expect(responseBody).toMatchObject({ mode: 'keyword' }); |
| 152 | expect(responseBody).not.toHaveProperty('degradedReason'); |
| 153 | expect(events).toHaveLength(1); |
| 154 | expect(events[0]).toMatchObject({ |
| 155 | retrievalMode: 'keyword', |
| 156 | degradationCategory: 'semantic-capacity-limited', |
| 157 | statusCode: 200, |
| 158 | }); |
| 159 | }); |
| 160 | |
| 161 | for (const guardrail of [ |
| 162 | 'semantic-timeout', |
| 163 | 'semantic-rate-limited', |
| 164 | 'semantic-capacity-limited', |
| 165 | ] as const) { |
| 166 | test(`keeps ${guardrail} visible when the preview overlay also fails`, async () => { |
| 167 | const events: CapturedSearchEvent[] = []; |
| 168 | const handler = createKnowledgeSearchHandler({ |
| 169 | hybridEnabled: () => true, |
| 170 | searchKeywordCandidates: async () => ({ candidates: [weakKeywordCandidate()] }), |
| 171 | searchSemanticCandidates: async () => guardrail === 'semantic-timeout' |
| 172 | ? new Promise<never>(() => {}) |
| 173 | : Promise.reject(new Error('semantic search should not run after admission is denied')), |
| 174 | searchLocalCandidates: async () => { throw new Error('preview-overlay-failed'); }, |
| 175 | searchPreviewCandidates: async () => { throw new Error('preview-overlay-failed'); }, |
| 176 | previewOverlayEnabled: () => true, |
| 177 | ...(guardrail === 'semantic-timeout' |
| 178 | ? { semanticTimeoutMs: () => 1 } |
| 179 | : { |
| 180 | acquireSemanticSearch: () => ({ |
| 181 | allowed: false as const, |
| 182 | reason: guardrail, |
| 183 | }), |
| 184 | }), |
| 185 | captureSearch: event => events.push(event), |
| 186 | }); |
| 187 | |
| 188 | const response = await handler(new Request( |
| 189 | 'http://localhost/api/knowledge-search?q=why+did+authorization+fail+again&filter=kb', |
| 190 | )); |
| 191 | |
| 192 | expect(response.status).toBe(200); |
| 193 | expect(events).toHaveLength(1); |
| 194 | expect(events[0]).toMatchObject({ degradationCategory: guardrail }); |
| 195 | }); |
| 196 | } |
| 197 | |
| 198 | test('builds an anonymous PostHog event with a lightly redacted query', () => { |
| 199 | const capture = buildKnowledgeSearchCapture({ |
| 200 | query: 'GITHUB_CREATE_AN_ISSUE failed token=sk-secretvalue123456', |
| 201 | filter: 'kb', |
| 202 | retrievalMode: 'keyword', |
| 203 | resultCount: 2, |
| 204 | degradationCategory: 'semantic-request-failed', |
| 205 | strongMatch: null, |
| 206 | statusCode: 200, |
| 207 | durationMs: 42.7, |
| 208 | keywordDurationMs: 8.2, |
| 209 | semanticDurationMs: null, |
| 210 | resultSourceTypes: ['reference', 'toolkit'], |
| 211 | previewOverlayApplied: true, |
| 212 | }, { |
| 213 | apiKey: 'phc_project_token', |
| 214 | host: 'https://us.i.posthog.com/', |
| 215 | }); |
| 216 | |
| 217 | expect(capture).toEqual({ |
| 218 | url: 'https://us.i.posthog.com/i/v0/e/', |
| 219 | body: { |
| 220 | api_key: 'phc_project_token', |
| 221 | event: 'kb_search_executed', |
| 222 | properties: { |
| 223 | distinct_id: 'public-kb-search', |
| 224 | '$process_person_profile': false, |
| 225 | query: 'GITHUB_CREATE_AN_ISSUE failed token=[REDACTED]', |
| 226 | filter: 'kb', |
| 227 | retrieval_mode: 'keyword', |
| 228 | result_count: 2, |
| 229 | degradation_category: 'semantic-request-failed', |
| 230 | strong_match: null, |
| 231 | status_code: 200, |
| 232 | duration_ms: 43, |
| 233 | keyword_duration_ms: 8, |
| 234 | semantic_duration_ms: null, |
| 235 | result_source_types: ['reference', 'toolkit'], |
| 236 | preview_overlay_applied: true, |
| 237 | }, |
| 238 | }, |
| 239 | }); |
| 240 | }); |
| 241 | |
| 242 | test('removes the KB query from generic PostHog pageview URLs', () => { |
| 243 | expect(getPostHogPageViewUrl( |
| 244 | 'https://docs.composio.dev', |
| 245 | '/kb/search', |
| 246 | new URLSearchParams('q=github+oauth&filter=kb'), |
| 247 | )).toBe('https://docs.composio.dev/kb/search?filter=kb'); |
| 248 | expect(getPostHogPageViewUrl( |
| 249 | 'https://docs.composio.dev', |
| 250 | '/docs', |
| 251 | new URLSearchParams('framework=nextjs'), |
| 252 | )).toBe('https://docs.composio.dev/docs?framework=nextjs'); |
| 253 | }); |
| 254 | |
| 255 | test('delivers the capture payload to the configured PostHog ingestion host', async () => { |
| 256 | let receivedPath = ''; |
| 257 | let receivedBody: unknown; |
| 258 | const server = Bun.serve({ |
| 259 | port: 0, |
| 260 | async fetch(request) { |
| 261 | receivedPath = new URL(request.url).pathname; |
| 262 | receivedBody = await request.json(); |
| 263 | return new Response(null, { status: 202 }); |
| 264 | }, |
| 265 | }); |
| 266 | try { |
| 267 | expect(await sendKnowledgeSearchAnalytics({ |
| 268 | query: 'github oauth delivery test', |
| 269 | filter: 'kb', |
| 270 | retrievalMode: 'hybrid', |
| 271 | resultCount: 3, |
| 272 | degradationCategory: null, |
| 273 | strongMatch: true, |
| 274 | statusCode: 200, |
| 275 | durationMs: 25, |
| 276 | keywordDurationMs: 5, |
| 277 | semanticDurationMs: 12, |
| 278 | resultSourceTypes: ['kb'], |
| 279 | previewOverlayApplied: false, |
| 280 | }, { |
| 281 | apiKey: 'phc_project_token', |
| 282 | host: `http://127.0.0.1:${server.port}`, |
| 283 | timeoutMs: 1_000, |
| 284 | })).toBe(true); |
| 285 | expect(receivedPath).toBe('/i/v0/e/'); |
| 286 | expect(receivedBody).toMatchObject({ |
| 287 | event: 'kb_search_executed', |
| 288 | properties: { query: 'github oauth delivery test' }, |
| 289 | }); |
| 290 | } finally { |
| 291 | server.stop(true); |
| 292 | } |
| 293 | }); |
| 294 | }); |