@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
ts 181 lines 5.74 KB
Raw
1 import { afterEach, describe, expect, test } from 'bun:test';
2 import { generateText, streamText } from 'ai';
3 import { createInceptionChatModel } from '../../agent/agent';
4
5 /**
6 * Provider-level regression tests for the system prompt disclosure finding.
7 *
8 * These drive a real model call through the configured `inception` provider
9 * with a stubbed `fetch`, because the AI SDK builds its own `APICallError`
10 * objects that carry `requestBodyValues` (the chat payload, system prompt
11 * included). Calling `safeInceptionFetch` directly does not exercise that.
12 */
13
14 const SYSTEM_PROMPT = 'CONFIDENTIAL-EVE-SYSTEM-PROMPT-CANARY';
15 const SAFE_MESSAGE = 'Docs agent model request failed.';
16
17 const originalFetch = globalThis.fetch;
18 const originalInceptionApiKey = process.env.INCEPTION_API_KEY;
19
20 afterEach(() => {
21 globalThis.fetch = originalFetch;
22 process.env.INCEPTION_API_KEY = originalInceptionApiKey;
23 });
24
25 /**
26 * Walks own properties (including non-enumerable ones such as `message`) and
27 * the `cause` chain. `JSON.stringify(error, propertyNames)` filters keys at
28 * every depth and silently misses nested payloads like `requestBodyValues`.
29 */
30 function serializeDeep(value: unknown, seen = new Set<unknown>()): string {
31 if (value === null || typeof value !== 'object') return String(value);
32 if (seen.has(value)) return '[circular]';
33 seen.add(value);
34
35 if (Array.isArray(value)) {
36 return value.map(entry => serializeDeep(entry, seen)).join('|');
37 }
38
39 return Object.getOwnPropertyNames(value)
40 .map(key => {
41 let nested: unknown;
42 try {
43 nested = (value as Record<string, unknown>)[key];
44 } catch {
45 return `${key}=[unreadable]`;
46 }
47 return `${key}=${serializeDeep(nested, seen)}`;
48 })
49 .join('|');
50 }
51
52 function expectNoPromptLeak(error: unknown) {
53 const serialized = serializeDeep(error);
54 expect(serialized).not.toContain(SYSTEM_PROMPT);
55 expect(JSON.stringify(error ?? null)).not.toContain(SYSTEM_PROMPT);
56 expect(JSON.stringify((error as { requestBodyValues?: unknown })?.requestBodyValues ?? null)).not.toContain(
57 SYSTEM_PROMPT
58 );
59 expect(serializeDeep((error as { cause?: unknown })?.cause)).not.toContain(SYSTEM_PROMPT);
60 }
61
62 function stubFetch(handler: () => Promise<Response>) {
63 process.env.INCEPTION_API_KEY = 'test-key';
64 globalThis.fetch = handler as typeof fetch;
65 }
66
67 function sseResponse(...frames: string[]) {
68 return new Response(`${frames.map(frame => `data: ${frame}`).join('\n\n')}\n\n`, {
69 status: 200,
70 headers: { 'content-type': 'text/event-stream' },
71 });
72 }
73
74 const ERROR_FRAME =
75 '{"error":{"message":"quota exceeded","type":"insufficient_quota","code":"insufficient_quota"}}';
76 const TEXT_FRAME =
77 '{"id":"1","object":"chat.completion.chunk","created":1,"model":"mercury-2","choices":[{"index":0,"delta":{"role":"assistant","content":"hi"},"finish_reason":null}]}';
78
79 async function captureGenerateError() {
80 try {
81 await generateText({
82 model: createInceptionChatModel(),
83 system: SYSTEM_PROMPT,
84 prompt: 'How do Composio sessions work?',
85 maxRetries: 0,
86 });
87 return undefined;
88 } catch (error) {
89 return error;
90 }
91 }
92
93 async function captureStreamError() {
94 let captured: unknown;
95 const result = streamText({
96 model: createInceptionChatModel(),
97 system: SYSTEM_PROMPT,
98 prompt: 'How do Composio sessions work?',
99 maxRetries: 0,
100 onError: ({ error }) => {
101 captured ??= error;
102 },
103 });
104
105 for await (const part of result.fullStream) {
106 if (part.type === 'error') captured ??= part.error;
107 }
108
109 return captured;
110 }
111
112 describe('Inception model error sanitization', () => {
113 test('non-2xx upstream responses do not leak the system prompt', async () => {
114 stubFetch(async () => new Response('upstream exploded', { status: 500 }));
115
116 const error = await captureGenerateError();
117
118 expectNoPromptLeak(error);
119 expect((error as Error)?.message).toContain(SAFE_MESSAGE);
120 });
121
122 test('200 responses with a JSON error payload do not leak the system prompt', async () => {
123 stubFetch(
124 async () =>
125 new Response(JSON.stringify({ error: { message: 'quota exceeded' } }), {
126 status: 200,
127 headers: { 'content-type': 'application/json' },
128 })
129 );
130
131 const error = await captureGenerateError();
132
133 expectNoPromptLeak(error);
134 expect((error as Error)?.message).toContain(SAFE_MESSAGE);
135 });
136
137 test('transport failures do not leak the system prompt', async () => {
138 stubFetch(async () => {
139 const cause = Object.assign(new Error('connect ECONNREFUSED 127.0.0.1:443'), {
140 code: 'ECONNREFUSED',
141 });
142 throw Object.assign(new TypeError('fetch failed'), { cause });
143 });
144
145 const error = await captureGenerateError();
146
147 expectNoPromptLeak(error);
148 expect((error as Error)?.message).toContain(SAFE_MESSAGE);
149 });
150
151 test('streamed error frames before any output do not leak the system prompt', async () => {
152 stubFetch(async () => sseResponse(ERROR_FRAME, '[DONE]'));
153
154 const error = await captureStreamError();
155
156 expect(error).toBeDefined();
157 expectNoPromptLeak(error);
158 expect((error as Error)?.message).toContain(SAFE_MESSAGE);
159 });
160
161 test('streamed error frames after output has started do not leak the system prompt', async () => {
162 stubFetch(async () => sseResponse(TEXT_FRAME, ERROR_FRAME, '[DONE]'));
163
164 const error = await captureStreamError();
165
166 expect(error).toBeDefined();
167 expectNoPromptLeak(error);
168 expect((error as Error)?.message).toContain(SAFE_MESSAGE);
169 });
170
171 test('abort errors still reach the AI SDK unchanged', async () => {
172 const abortError = new DOMException('The operation was aborted.', 'AbortError');
173 stubFetch(async () => {
174 throw abortError;
175 });
176
177 const error = await captureGenerateError();
178
179 expect(error).toBe(abortError);
180 });
181 });