@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
ts 294 lines 10.4 KB
Raw
1 import { describe, expect, test } from 'bun:test';
2 import { createKnowledgeSearchHandler } from '@/app/api/knowledge-search/route';
3 import { getPostHogPageViewUrl } from '@/components/posthog-provider';
4 import {
5 buildKnowledgeSearchCapture,
6 redactKnowledgeSearchQuery,
7 sendKnowledgeSearchAnalytics,
8 } from '@/lib/knowledge/query-analytics';
9 import type { PublicKnowledgeCandidateRecord } from '@/lib/knowledge/hybrid-search';
10
11 interface CapturedSearchEvent {
12 query: string;
13 filter: string;
14 retrievalMode: string;
15 resultCount: number;
16 degradationCategory: string | null;
17 strongMatch: boolean | null;
18 statusCode: number;
19 durationMs: number;
20 keywordDurationMs: number;
21 semanticDurationMs: number | null;
22 resultSourceTypes: string[];
23 previewOverlayApplied: boolean;
24 }
25
26 function weakKeywordCandidate(): PublicKnowledgeCandidateRecord {
27 return {
28 objectID: 'weak-keyword',
29 sourceType: 'kb',
30 sourceLabel: 'Knowledge Base',
31 pageID: '/kb/weak-keyword',
32 title: 'GitHub OAuth guide',
33 section: null,
34 description: 'General GitHub OAuth guidance.',
35 content: 'Reconnect GitHub access when authorization expires.',
36 canonicalUrl: '/kb/weak-keyword',
37 breadcrumbs: ['Knowledge Base'],
38 productAreas: [],
39 toolkitSlugs: ['github'],
40 keywords: [],
41 slug: 'github-oauth-guide',
42 toolNames: [],
43 toolSlugs: [],
44 pageRank: 1_900,
45 sectionRank: 100,
46 lastVerifiedAt: '2026-08-18',
47 };
48 }
49
50 describe('knowledge search analytics', () => {
51 test.each([
52 ['x-api-key header', 'x-api-key: api-value-123', 'x-api-key: [REDACTED]'],
53 ['X-API-KEY environment field', 'X-API-KEY=api-value-123', 'X-API-KEY=[REDACTED]'],
54 ['X_API_KEY environment field', 'X_API_KEY=api-value-123', 'X_API_KEY=[REDACTED]'],
55 ])('redacts a %s', (_case, query, expected) => {
56 expect(redactKnowledgeSearchQuery(query)).toBe(expected);
57 });
58
59 test.each([
60 ['ordinary error code', 'Error code: NONEXISTENT_VERSION'],
61 ['ordinary HTTP state', 'HTTP state=failed'],
62 ])('preserves an %s', (_case, query) => {
63 expect(redactKnowledgeSearchQuery(query)).toBe(query);
64 });
65
66 test('redacts credential fields in raw and URL-shaped queries without deleting prose', () => {
67 expect(redactKnowledgeSearchQuery(
68 "refresh_token=refresh-value-123 access-token: access-value-123 client_secret='client-value-123'",
69 )).toBe(
70 'refresh_token=[REDACTED] access-token: [REDACTED] client_secret=[REDACTED]',
71 );
72 expect(redactKnowledgeSearchQuery(
73 'https://example.com/callback?refresh-token=refresh-value-123&access_token=access-value-123&client-secret=client-value-123&page=1',
74 )).toBe(
75 'https://example.com/callback?refresh-token=[REDACTED]&access_token=[REDACTED]&client-secret=[REDACTED]&page=1',
76 );
77 expect(redactKnowledgeSearchQuery(
78 'https://example.com/callback?authorization=Bearer-value-123&code=oauth-code-123&state=opaque-state-123&page=1',
79 )).toBe(
80 'https://example.com/callback?authorization=[REDACTED]&code=[REDACTED]&state=[REDACTED]&page=1',
81 );
82 expect(redactKnowledgeSearchQuery(
83 '{"code":"oauth-code-123","state":"opaque-state-123","status":"pending"}',
84 )).toBe(
85 '{"code":[REDACTED],"state":[REDACTED],"status":"pending"}',
86 );
87 expect(redactKnowledgeSearchQuery(
88 'authorization: Bearer short-secret',
89 )).toBe(
90 'authorization: [REDACTED]',
91 );
92 expect(redactKnowledgeSearchQuery(
93 'Authorization code flow keeps state between browser redirects, and refresh token rotation is automatic.',
94 )).toBe(
95 'Authorization code flow keeps state between browser redirects, and refresh token rotation is automatic.',
96 );
97 });
98
99 test('records each completed API search with its retrieval outcome', async () => {
100 const events: CapturedSearchEvent[] = [];
101 const handler = createKnowledgeSearchHandler({
102 hybridEnabled: () => true,
103 searchKeywordCandidates: async () => ({ candidates: [] }),
104 searchSemanticCandidates: async () => [],
105 captureSearch: event => events.push(event),
106 });
107
108 const response = await handler(new Request(
109 'http://localhost/api/knowledge-search?q=why+did+github+oauth+fail&filter=kb',
110 ));
111
112 expect(response.status).toBe(200);
113 expect(events).toHaveLength(1);
114 expect(events[0]).toMatchObject({
115 query: 'why did github oauth fail',
116 filter: 'kb',
117 retrievalMode: 'hybrid',
118 resultCount: 0,
119 degradationCategory: null,
120 strongMatch: false,
121 statusCode: 200,
122 resultSourceTypes: [],
123 previewOverlayApplied: false,
124 });
125 expect(events[0]?.durationMs).toBeGreaterThanOrEqual(0);
126 expect(events[0]?.keywordDurationMs).toBeGreaterThanOrEqual(0);
127 expect(events[0]?.semanticDurationMs).toBeGreaterThanOrEqual(0);
128 });
129
130 test('records semantic guardrail hits without failing the search request', async () => {
131 const events: CapturedSearchEvent[] = [];
132 const handler = createKnowledgeSearchHandler({
133 hybridEnabled: () => true,
134 searchKeywordCandidates: async () => ({ candidates: [] }),
135 searchSemanticCandidates: async () => {
136 throw new Error('semantic search should not run after admission is denied');
137 },
138 acquireSemanticSearch: () => ({
139 allowed: false,
140 reason: 'semantic-capacity-limited',
141 }),
142 captureSearch: event => events.push(event),
143 });
144
145 const response = await handler(new Request(
146 'http://localhost/api/knowledge-search?q=github+oauth&filter=kb',
147 ));
148
149 expect(response.status).toBe(200);
150 const responseBody = await response.json() as Record<string, unknown>;
151 expect(responseBody).toMatchObject({ mode: 'keyword' });
152 expect(responseBody).not.toHaveProperty('degradedReason');
153 expect(events).toHaveLength(1);
154 expect(events[0]).toMatchObject({
155 retrievalMode: 'keyword',
156 degradationCategory: 'semantic-capacity-limited',
157 statusCode: 200,
158 });
159 });
160
161 for (const guardrail of [
162 'semantic-timeout',
163 'semantic-rate-limited',
164 'semantic-capacity-limited',
165 ] as const) {
166 test(`keeps ${guardrail} visible when the preview overlay also fails`, async () => {
167 const events: CapturedSearchEvent[] = [];
168 const handler = createKnowledgeSearchHandler({
169 hybridEnabled: () => true,
170 searchKeywordCandidates: async () => ({ candidates: [weakKeywordCandidate()] }),
171 searchSemanticCandidates: async () => guardrail === 'semantic-timeout'
172 ? new Promise<never>(() => {})
173 : Promise.reject(new Error('semantic search should not run after admission is denied')),
174 searchLocalCandidates: async () => { throw new Error('preview-overlay-failed'); },
175 searchPreviewCandidates: async () => { throw new Error('preview-overlay-failed'); },
176 previewOverlayEnabled: () => true,
177 ...(guardrail === 'semantic-timeout'
178 ? { semanticTimeoutMs: () => 1 }
179 : {
180 acquireSemanticSearch: () => ({
181 allowed: false as const,
182 reason: guardrail,
183 }),
184 }),
185 captureSearch: event => events.push(event),
186 });
187
188 const response = await handler(new Request(
189 'http://localhost/api/knowledge-search?q=why+did+authorization+fail+again&filter=kb',
190 ));
191
192 expect(response.status).toBe(200);
193 expect(events).toHaveLength(1);
194 expect(events[0]).toMatchObject({ degradationCategory: guardrail });
195 });
196 }
197
198 test('builds an anonymous PostHog event with a lightly redacted query', () => {
199 const capture = buildKnowledgeSearchCapture({
200 query: 'GITHUB_CREATE_AN_ISSUE failed token=sk-secretvalue123456',
201 filter: 'kb',
202 retrievalMode: 'keyword',
203 resultCount: 2,
204 degradationCategory: 'semantic-request-failed',
205 strongMatch: null,
206 statusCode: 200,
207 durationMs: 42.7,
208 keywordDurationMs: 8.2,
209 semanticDurationMs: null,
210 resultSourceTypes: ['reference', 'toolkit'],
211 previewOverlayApplied: true,
212 }, {
213 apiKey: 'phc_project_token',
214 host: 'https://us.i.posthog.com/',
215 });
216
217 expect(capture).toEqual({
218 url: 'https://us.i.posthog.com/i/v0/e/',
219 body: {
220 api_key: 'phc_project_token',
221 event: 'kb_search_executed',
222 properties: {
223 distinct_id: 'public-kb-search',
224 '$process_person_profile': false,
225 query: 'GITHUB_CREATE_AN_ISSUE failed token=[REDACTED]',
226 filter: 'kb',
227 retrieval_mode: 'keyword',
228 result_count: 2,
229 degradation_category: 'semantic-request-failed',
230 strong_match: null,
231 status_code: 200,
232 duration_ms: 43,
233 keyword_duration_ms: 8,
234 semantic_duration_ms: null,
235 result_source_types: ['reference', 'toolkit'],
236 preview_overlay_applied: true,
237 },
238 },
239 });
240 });
241
242 test('removes the KB query from generic PostHog pageview URLs', () => {
243 expect(getPostHogPageViewUrl(
244 'https://docs.composio.dev',
245 '/kb/search',
246 new URLSearchParams('q=github+oauth&filter=kb'),
247 )).toBe('https://docs.composio.dev/kb/search?filter=kb');
248 expect(getPostHogPageViewUrl(
249 'https://docs.composio.dev',
250 '/docs',
251 new URLSearchParams('framework=nextjs'),
252 )).toBe('https://docs.composio.dev/docs?framework=nextjs');
253 });
254
255 test('delivers the capture payload to the configured PostHog ingestion host', async () => {
256 let receivedPath = '';
257 let receivedBody: unknown;
258 const server = Bun.serve({
259 port: 0,
260 async fetch(request) {
261 receivedPath = new URL(request.url).pathname;
262 receivedBody = await request.json();
263 return new Response(null, { status: 202 });
264 },
265 });
266 try {
267 expect(await sendKnowledgeSearchAnalytics({
268 query: 'github oauth delivery test',
269 filter: 'kb',
270 retrievalMode: 'hybrid',
271 resultCount: 3,
272 degradationCategory: null,
273 strongMatch: true,
274 statusCode: 200,
275 durationMs: 25,
276 keywordDurationMs: 5,
277 semanticDurationMs: 12,
278 resultSourceTypes: ['kb'],
279 previewOverlayApplied: false,
280 }, {
281 apiKey: 'phc_project_token',
282 host: `http://127.0.0.1:${server.port}`,
283 timeoutMs: 1_000,
284 })).toBe(true);
285 expect(receivedPath).toBe('/i/v0/e/');
286 expect(receivedBody).toMatchObject({
287 event: 'kb_search_executed',
288 properties: { query: 'github oauth delivery test' },
289 });
290 } finally {
291 server.stop(true);
292 }
293 });
294 });