Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | import { afterEach, describe, expect, test } from 'bun:test'; |
| 2 | import { generateText, streamText } from 'ai'; |
| 3 | import { createInceptionChatModel } from '../../agent/agent'; |
| 4 | |
| 5 | /** |
| 6 | * Provider-level regression tests for the system prompt disclosure finding. |
| 7 | * |
| 8 | * These drive a real model call through the configured `inception` provider |
| 9 | * with a stubbed `fetch`, because the AI SDK builds its own `APICallError` |
| 10 | * objects that carry `requestBodyValues` (the chat payload, system prompt |
| 11 | * included). Calling `safeInceptionFetch` directly does not exercise that. |
| 12 | */ |
| 13 | |
| 14 | const SYSTEM_PROMPT = 'CONFIDENTIAL-EVE-SYSTEM-PROMPT-CANARY'; |
| 15 | const SAFE_MESSAGE = 'Docs agent model request failed.'; |
| 16 | |
| 17 | const originalFetch = globalThis.fetch; |
| 18 | const originalInceptionApiKey = process.env.INCEPTION_API_KEY; |
| 19 | |
| 20 | afterEach(() => { |
| 21 | globalThis.fetch = originalFetch; |
| 22 | process.env.INCEPTION_API_KEY = originalInceptionApiKey; |
| 23 | }); |
| 24 | |
| 25 | /** |
| 26 | * Walks own properties (including non-enumerable ones such as `message`) and |
| 27 | * the `cause` chain. `JSON.stringify(error, propertyNames)` filters keys at |
| 28 | * every depth and silently misses nested payloads like `requestBodyValues`. |
| 29 | */ |
| 30 | function serializeDeep(value: unknown, seen = new Set<unknown>()): string { |
| 31 | if (value === null || typeof value !== 'object') return String(value); |
| 32 | if (seen.has(value)) return '[circular]'; |
| 33 | seen.add(value); |
| 34 | |
| 35 | if (Array.isArray(value)) { |
| 36 | return value.map(entry => serializeDeep(entry, seen)).join('|'); |
| 37 | } |
| 38 | |
| 39 | return Object.getOwnPropertyNames(value) |
| 40 | .map(key => { |
| 41 | let nested: unknown; |
| 42 | try { |
| 43 | nested = (value as Record<string, unknown>)[key]; |
| 44 | } catch { |
| 45 | return `${key}=[unreadable]`; |
| 46 | } |
| 47 | return `${key}=${serializeDeep(nested, seen)}`; |
| 48 | }) |
| 49 | .join('|'); |
| 50 | } |
| 51 | |
| 52 | function expectNoPromptLeak(error: unknown) { |
| 53 | const serialized = serializeDeep(error); |
| 54 | expect(serialized).not.toContain(SYSTEM_PROMPT); |
| 55 | expect(JSON.stringify(error ?? null)).not.toContain(SYSTEM_PROMPT); |
| 56 | expect(JSON.stringify((error as { requestBodyValues?: unknown })?.requestBodyValues ?? null)).not.toContain( |
| 57 | SYSTEM_PROMPT |
| 58 | ); |
| 59 | expect(serializeDeep((error as { cause?: unknown })?.cause)).not.toContain(SYSTEM_PROMPT); |
| 60 | } |
| 61 | |
| 62 | function stubFetch(handler: () => Promise<Response>) { |
| 63 | process.env.INCEPTION_API_KEY = 'test-key'; |
| 64 | globalThis.fetch = handler as typeof fetch; |
| 65 | } |
| 66 | |
| 67 | function sseResponse(...frames: string[]) { |
| 68 | return new Response(`${frames.map(frame => `data: ${frame}`).join('\n\n')}\n\n`, { |
| 69 | status: 200, |
| 70 | headers: { 'content-type': 'text/event-stream' }, |
| 71 | }); |
| 72 | } |
| 73 | |
| 74 | const ERROR_FRAME = |
| 75 | '{"error":{"message":"quota exceeded","type":"insufficient_quota","code":"insufficient_quota"}}'; |
| 76 | const TEXT_FRAME = |
| 77 | '{"id":"1","object":"chat.completion.chunk","created":1,"model":"mercury-2","choices":[{"index":0,"delta":{"role":"assistant","content":"hi"},"finish_reason":null}]}'; |
| 78 | |
| 79 | async function captureGenerateError() { |
| 80 | try { |
| 81 | await generateText({ |
| 82 | model: createInceptionChatModel(), |
| 83 | system: SYSTEM_PROMPT, |
| 84 | prompt: 'How do Composio sessions work?', |
| 85 | maxRetries: 0, |
| 86 | }); |
| 87 | return undefined; |
| 88 | } catch (error) { |
| 89 | return error; |
| 90 | } |
| 91 | } |
| 92 | |
| 93 | async function captureStreamError() { |
| 94 | let captured: unknown; |
| 95 | const result = streamText({ |
| 96 | model: createInceptionChatModel(), |
| 97 | system: SYSTEM_PROMPT, |
| 98 | prompt: 'How do Composio sessions work?', |
| 99 | maxRetries: 0, |
| 100 | onError: ({ error }) => { |
| 101 | captured ??= error; |
| 102 | }, |
| 103 | }); |
| 104 | |
| 105 | for await (const part of result.fullStream) { |
| 106 | if (part.type === 'error') captured ??= part.error; |
| 107 | } |
| 108 | |
| 109 | return captured; |
| 110 | } |
| 111 | |
| 112 | describe('Inception model error sanitization', () => { |
| 113 | test('non-2xx upstream responses do not leak the system prompt', async () => { |
| 114 | stubFetch(async () => new Response('upstream exploded', { status: 500 })); |
| 115 | |
| 116 | const error = await captureGenerateError(); |
| 117 | |
| 118 | expectNoPromptLeak(error); |
| 119 | expect((error as Error)?.message).toContain(SAFE_MESSAGE); |
| 120 | }); |
| 121 | |
| 122 | test('200 responses with a JSON error payload do not leak the system prompt', async () => { |
| 123 | stubFetch( |
| 124 | async () => |
| 125 | new Response(JSON.stringify({ error: { message: 'quota exceeded' } }), { |
| 126 | status: 200, |
| 127 | headers: { 'content-type': 'application/json' }, |
| 128 | }) |
| 129 | ); |
| 130 | |
| 131 | const error = await captureGenerateError(); |
| 132 | |
| 133 | expectNoPromptLeak(error); |
| 134 | expect((error as Error)?.message).toContain(SAFE_MESSAGE); |
| 135 | }); |
| 136 | |
| 137 | test('transport failures do not leak the system prompt', async () => { |
| 138 | stubFetch(async () => { |
| 139 | const cause = Object.assign(new Error('connect ECONNREFUSED 127.0.0.1:443'), { |
| 140 | code: 'ECONNREFUSED', |
| 141 | }); |
| 142 | throw Object.assign(new TypeError('fetch failed'), { cause }); |
| 143 | }); |
| 144 | |
| 145 | const error = await captureGenerateError(); |
| 146 | |
| 147 | expectNoPromptLeak(error); |
| 148 | expect((error as Error)?.message).toContain(SAFE_MESSAGE); |
| 149 | }); |
| 150 | |
| 151 | test('streamed error frames before any output do not leak the system prompt', async () => { |
| 152 | stubFetch(async () => sseResponse(ERROR_FRAME, '[DONE]')); |
| 153 | |
| 154 | const error = await captureStreamError(); |
| 155 | |
| 156 | expect(error).toBeDefined(); |
| 157 | expectNoPromptLeak(error); |
| 158 | expect((error as Error)?.message).toContain(SAFE_MESSAGE); |
| 159 | }); |
| 160 | |
| 161 | test('streamed error frames after output has started do not leak the system prompt', async () => { |
| 162 | stubFetch(async () => sseResponse(TEXT_FRAME, ERROR_FRAME, '[DONE]')); |
| 163 | |
| 164 | const error = await captureStreamError(); |
| 165 | |
| 166 | expect(error).toBeDefined(); |
| 167 | expectNoPromptLeak(error); |
| 168 | expect((error as Error)?.message).toContain(SAFE_MESSAGE); |
| 169 | }); |
| 170 | |
| 171 | test('abort errors still reach the AI SDK unchanged', async () => { |
| 172 | const abortError = new DOMException('The operation was aborted.', 'AbortError'); |
| 173 | stubFetch(async () => { |
| 174 | throw abortError; |
| 175 | }); |
| 176 | |
| 177 | const error = await captureGenerateError(); |
| 178 | |
| 179 | expect(error).toBe(abortError); |
| 180 | }); |
| 181 | }); |