@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 04:45 UTC

next
ts 590 lines 20.4 KB
Raw
1 import { describe, it, expect, vi, beforeEach, afterEach, Mock } from 'vitest';
2 import * as path from 'node:path';
3 import * as fs from 'node:fs';
4 import { getFileDataAfterUploadingToS3, downloadFileFromS3 } from '../../src/utils/fileUtils.node';
5 import ComposioClient from '@composio/client';
6 import { ComposioSensitiveFilePathBlockedError } from '../../src/errors/FileModifierErrors';
7
8 // Mock the uuid module
9 vi.mock('../../src/utils/uuid', () => ({
10 getRandomShortId: vi.fn(() => 'abc12345'),
11 getRandomUUID: vi.fn(() => '12345678-1234-1234-1234-123456789012'),
12 }));
13
14 // Mock fs module
15 vi.mock('fs', () => ({
16 default: {
17 readFileSync: vi.fn(),
18 writeFileSync: vi.fn(),
19 existsSync: vi.fn(() => true),
20 mkdirSync: vi.fn(),
21 },
22 readFileSync: vi.fn(),
23 writeFileSync: vi.fn(),
24 existsSync: vi.fn(() => true),
25 mkdirSync: vi.fn(),
26 }));
27
28 // Mock os module
29 vi.mock('os', () => ({
30 default: {
31 homedir: vi.fn(() => '/home/test'),
32 },
33 homedir: vi.fn(() => '/home/test'),
34 }));
35
36 // Mock path module (keep resolve/join for sensitive path checks; basename stays mocked for fileUtils)
37 vi.mock('path', async importOriginal => {
38 const actual = await importOriginal<typeof import('node:path')>();
39 return {
40 ...actual,
41 default: actual,
42 basename: vi.fn((p: string) => p.split('/').pop() ?? p),
43 };
44 });
45
46 // Mock DNS resolution so the SSRF guard treats test hosts as public without
47 // hitting the network. (URL uploads route through ssrfSafeFetch, which resolves
48 // the host and rejects private/internal addresses.)
49 vi.mock('node:dns/promises', () => ({
50 lookup: vi.fn().mockResolvedValue([{ address: '93.184.216.34', family: 4 }]),
51 }));
52
53 // eslint-disable-next-line no-restricted-imports
54 import { lookup } from 'node:dns/promises';
55 import { ComposioBlockedInternalUrlError } from '../../src/errors';
56
57 const mockLookup = vi.mocked(lookup);
58
59 // Mock global fetch
60 const mockFetch = vi.fn();
61 global.fetch = mockFetch;
62
63 const fetchedFileResponse = (contentType: string = 'application/pdf') =>
64 new Response(new Uint8Array(10), {
65 status: 200,
66 headers: { 'content-type': contentType },
67 });
68
69 /** A small, well-formed S3 download response (no `content-length` header). */
70 const downloadedFileResponse = (body: Uint8Array = new Uint8Array(10)) =>
71 new Response(body, { status: 200 });
72
73 describe('fileUtils', () => {
74 let mockClient: ComposioClient;
75
76 beforeEach(() => {
77 mockClient = {
78 files: {
79 createPresignedURL: vi.fn(),
80 },
81 } as unknown as ComposioClient;
82
83 vi.clearAllMocks();
84
85 // Mock Date.now to return a consistent timestamp
86 vi.spyOn(Date, 'now').mockReturnValue(1640995200000); // 2022-01-01 00:00:00
87 });
88
89 afterEach(() => {
90 vi.restoreAllMocks();
91 });
92
93 describe('URL filename generation with query parameters', () => {
94 beforeEach(() => {
95 // Mock successful fetch response
96 mockFetch.mockResolvedValue(fetchedFileResponse());
97
98 // Mock successful S3 upload
99 (mockClient.files.createPresignedURL as unknown as Mock).mockResolvedValue({
100 key: 'test-key',
101 type: 'new',
102 new_presigned_url: 'https://s3.example.com/upload',
103 });
104
105 // Mock successful PUT to S3
106 mockFetch.mockImplementation((url, options) => {
107 if (options?.method === 'PUT') {
108 return Promise.resolve({ ok: true });
109 }
110 // For the initial file fetch
111 return Promise.resolve(fetchedFileResponse());
112 });
113 });
114
115 it('should handle URLs with query parameters correctly', async () => {
116 const urlWithQuery =
117 'https://example.com/document.pdf?param1=value1&param2=value2&token=abc123';
118
119 const result = await getFileDataAfterUploadingToS3(urlWithQuery, {
120 toolSlug: 'test-tool',
121 toolkitSlug: 'test-toolkit',
122 client: mockClient,
123 });
124
125 expect(result.name).toBe('document.pdf');
126 expect(mockFetch).toHaveBeenCalledWith(
127 urlWithQuery,
128 // Plus a `dispatcher` pinned to the address the guard validated.
129 expect.objectContaining({ signal: undefined, redirect: 'manual' })
130 );
131 });
132
133 it('should generate filename when URL has no filename', async () => {
134 const urlWithoutFilename = 'https://example.com/?download=true&format=pdf';
135
136 const result = await getFileDataAfterUploadingToS3(urlWithoutFilename, {
137 toolSlug: 'test-tool',
138 toolkitSlug: 'test-toolkit',
139 client: mockClient,
140 });
141
142 expect(result.name).toBe('file_ts1640995200000abc12345.pdf');
143 });
144
145 it('should generate filename when URL path ends with slash', async () => {
146 const urlEndingWithSlash = 'https://example.com/folder/?type=document';
147
148 const result = await getFileDataAfterUploadingToS3(urlEndingWithSlash, {
149 toolSlug: 'test-tool',
150 toolkitSlug: 'test-toolkit',
151 client: mockClient,
152 });
153
154 expect(result.name).toBe('file_ts1640995200000abc12345.pdf');
155 });
156
157 it('should add extension to filename without extension based on MIME type', async () => {
158 const urlWithoutExtension = 'https://example.com/document?format=pdf';
159
160 const result = await getFileDataAfterUploadingToS3(urlWithoutExtension, {
161 toolSlug: 'test-tool',
162 toolkitSlug: 'test-toolkit',
163 client: mockClient,
164 });
165
166 expect(result.name).toBe('file_ts1640995200000abc12345.pdf');
167 });
168
169 it('should reject an oversized URL response before requesting an upload URL', async () => {
170 mockFetch.mockReset();
171 mockFetch.mockResolvedValueOnce(
172 new Response('small body', {
173 status: 200,
174 headers: {
175 'content-type': 'application/pdf',
176 'content-length': String(100 * 1024 * 1024 + 1),
177 },
178 })
179 );
180
181 await expect(
182 getFileDataAfterUploadingToS3('https://example.com/oversized.pdf', {
183 toolSlug: 'test-tool',
184 toolkitSlug: 'test-toolkit',
185 client: mockClient,
186 })
187 ).rejects.toThrow('exceeds maximum allowed size');
188
189 expect(mockClient.files.createPresignedURL).not.toHaveBeenCalled();
190 });
191 });
192
193 describe('MIME type extension handling', () => {
194 it('should handle structured MIME types with + correctly', async () => {
195 const testCases = [
196 { mimeType: 'application/vnd.api+json', expectedExt: 'json' },
197 { mimeType: 'application/ld+json', expectedExt: 'json' },
198 { mimeType: 'image/svg+xml', expectedExt: 'svg' },
199 { mimeType: 'application/atom+xml', expectedExt: 'atom' },
200 { mimeType: 'application/rss+xml', expectedExt: 'rss' },
201 { mimeType: 'application/hal+json', expectedExt: 'json' },
202 { mimeType: 'application/vnd.collection+json', expectedExt: 'json' },
203 { mimeType: 'application/vnd.custom+zip', expectedExt: 'zip' },
204 ];
205
206 for (const { mimeType, expectedExt } of testCases) {
207 // Reset mocks for each iteration
208 vi.clearAllMocks();
209
210 // Mock successful fetch response
211 mockFetch.mockResolvedValueOnce(fetchedFileResponse(mimeType));
212
213 // Mock successful S3 upload
214 (mockClient.files.createPresignedURL as unknown as Mock).mockResolvedValueOnce({
215 key: 'test-key',
216 type: 'new',
217 new_presigned_url: 'https://s3.example.com/upload',
218 });
219
220 // Mock successful PUT to S3
221 mockFetch.mockResolvedValueOnce({ ok: true });
222
223 const result = await getFileDataAfterUploadingToS3('https://example.com/file', {
224 toolSlug: 'test-tool',
225 toolkitSlug: 'test-toolkit',
226 client: mockClient,
227 });
228
229 expect(result.name).toBe(`file_ts1640995200000abc12345.${expectedExt}`);
230 }
231 });
232
233 it('should handle MIME types with parameters', async () => {
234 // Reset mocks
235 vi.clearAllMocks();
236
237 mockFetch.mockResolvedValueOnce(fetchedFileResponse('text/plain; charset=utf-8'));
238
239 // Mock successful S3 upload
240 (mockClient.files.createPresignedURL as unknown as Mock).mockResolvedValueOnce({
241 key: 'test-key',
242 type: 'new',
243 new_presigned_url: 'https://s3.example.com/upload',
244 });
245
246 // Mock successful PUT to S3
247 mockFetch.mockResolvedValueOnce({ ok: true });
248
249 const result = await getFileDataAfterUploadingToS3('https://example.com/file', {
250 toolSlug: 'test-tool',
251 toolkitSlug: 'test-toolkit',
252 client: mockClient,
253 });
254
255 expect(result.name).toBe('file_ts1640995200000abc12345.txt');
256 });
257
258 it('should handle common MIME types', async () => {
259 const testCases = [
260 { mimeType: 'application/pdf', expectedExt: 'pdf' },
261 { mimeType: 'image/jpeg', expectedExt: 'jpg' },
262 { mimeType: 'image/png', expectedExt: 'png' },
263 { mimeType: 'text/html', expectedExt: 'html' },
264 { mimeType: 'application/json', expectedExt: 'json' },
265 { mimeType: 'video/mp4', expectedExt: 'mp4' },
266 ];
267
268 for (const { mimeType, expectedExt } of testCases) {
269 // Reset mocks for each iteration
270 vi.clearAllMocks();
271
272 mockFetch.mockResolvedValueOnce(fetchedFileResponse(mimeType));
273
274 // Mock successful S3 upload
275 (mockClient.files.createPresignedURL as unknown as Mock).mockResolvedValueOnce({
276 key: 'test-key',
277 type: 'new',
278 new_presigned_url: 'https://s3.example.com/upload',
279 });
280
281 // Mock successful PUT to S3
282 mockFetch.mockResolvedValueOnce({ ok: true });
283
284 const result = await getFileDataAfterUploadingToS3('https://example.com/file', {
285 toolSlug: 'test-tool',
286 toolkitSlug: 'test-toolkit',
287 client: mockClient,
288 });
289
290 expect(result.name).toBe(`file_ts1640995200000abc12345.${expectedExt}`);
291 }
292 });
293
294 it('should fallback to generic extraction for unknown MIME types', async () => {
295 // Reset mocks
296 vi.clearAllMocks();
297
298 mockFetch.mockResolvedValueOnce(fetchedFileResponse('application/custom-format'));
299
300 // Mock successful S3 upload
301 (mockClient.files.createPresignedURL as unknown as Mock).mockResolvedValueOnce({
302 key: 'test-key',
303 type: 'new',
304 new_presigned_url: 'https://s3.example.com/upload',
305 });
306
307 // Mock successful PUT to S3
308 mockFetch.mockResolvedValueOnce({ ok: true });
309
310 const result = await getFileDataAfterUploadingToS3('https://example.com/file', {
311 toolSlug: 'test-tool',
312 toolkitSlug: 'test-toolkit',
313 client: mockClient,
314 });
315
316 expect(result.name).toBe('file_ts1640995200000abc12345.custom-format');
317 });
318 });
319
320 describe('downloadFileFromS3', () => {
321 it('should generate filename with tool slug prefix', async () => {
322 mockFetch.mockResolvedValue(downloadedFileResponse());
323
324 const result = await downloadFileFromS3({
325 toolSlug: 'github',
326 s3Url: 'https://s3.example.com/file.txt',
327 mimeType: 'text/plain',
328 });
329
330 expect(result.name).toBe('github_1640995200000abc12345.txt');
331 });
332
333 it('should handle MIME types with + in downloadFileFromS3', async () => {
334 mockFetch.mockResolvedValue(downloadedFileResponse());
335
336 const result = await downloadFileFromS3({
337 toolSlug: 'api-tool',
338 s3Url: 'https://s3.example.com/data.json',
339 mimeType: 'application/vnd.api+json',
340 });
341
342 expect(result.name).toBe('api-tool_1640995200000abc12345.json');
343 });
344
345 it('should handle download failure', async () => {
346 mockFetch.mockResolvedValue({
347 ok: false,
348 statusText: 'Not Found',
349 });
350
351 await expect(
352 downloadFileFromS3({
353 toolSlug: 'test-tool',
354 s3Url: 'https://s3.example.com/nonexistent.txt',
355 mimeType: 'text/plain',
356 })
357 ).rejects.toThrow('Failed to download file: Not Found');
358 });
359
360 it('synthesizes the filename server-side and ignores the s3Url path', async () => {
361 // Regression: lock the invariant that the saved filename comes from
362 // (toolSlug, mimeType, timestamp, random id) — never from the s3Url.
363 // If a future change started reflecting the s3Url path or query into
364 // the local filename, an attacker-controlled CDN response could write
365 // outside the download dir or stomp on existing files.
366 mockFetch.mockResolvedValue(downloadedFileResponse());
367
368 const result = await downloadFileFromS3({
369 toolSlug: 'github',
370 // The URL path contains traversal-looking segments and a totally
371 // different "filename"; none of this should leak into result.name.
372 s3Url:
373 'https://s3.example.com/../../../etc/passwd?response-content-disposition=attachment%3Bfilename%3Devil.sh',
374 mimeType: 'text/plain',
375 });
376
377 expect(result.name).toBe('github_1640995200000abc12345.txt');
378 expect(result.name).not.toContain('..');
379 expect(result.name).not.toContain('/');
380 expect(result.name).not.toContain('passwd');
381 expect(result.name).not.toContain('evil');
382 });
383
384 it('strips path components from a path-laden toolSlug before saving', async () => {
385 // Regression: the prefix is `${toolSlug}_`, so a slug containing `/`
386 // could in theory smuggle path components into the saved filename.
387 // `saveFile` defends against that by running the assembled filename
388 // through `path.basename` before joining with the download dir.
389 mockFetch.mockResolvedValue(downloadedFileResponse());
390
391 const result = await downloadFileFromS3({
392 toolSlug: 'AAA/../etc/passwd',
393 s3Url: 'https://s3.example.com/file.bin',
394 mimeType: 'text/plain',
395 });
396
397 // `result.name` is the pre-basename string (raw composed name); the
398 // actual on-disk path goes through `path.basename`. We assert that
399 // the on-disk path stays under the download dir by checking the
400 // returned filePath.
401 expect(result.filePath).toBeDefined();
402 expect(path.dirname(result.filePath as string)).not.toContain('etc');
403 expect(path.basename(result.filePath as string)).toBe('passwd_1640995200000abc12345.txt');
404 });
405
406 it('rejects a stream that exceeds the download limit and writes nothing', async () => {
407 // `s3Url` is an API-response field, so the body behind it is untrusted:
408 // without a cap, a dishonest (or absent) Content-Length lets the server
409 // stream unbounded bytes into the host process's heap and onto disk.
410 mockFetch.mockResolvedValue(downloadedFileResponse(new Uint8Array(2048)));
411
412 await expect(
413 downloadFileFromS3({
414 toolSlug: 'github',
415 s3Url: 'https://s3.example.com/huge.bin',
416 mimeType: 'text/plain',
417 maxDownloadBytes: 1024,
418 })
419 ).rejects.toThrow(/exceeds maximum allowed size/);
420
421 expect(fs.writeFileSync).not.toHaveBeenCalled();
422 });
423
424 it('downloads a body within the limit', async () => {
425 const body = new Uint8Array(512).fill(7);
426 mockFetch.mockResolvedValue(downloadedFileResponse(body));
427
428 const result = await downloadFileFromS3({
429 toolSlug: 'github',
430 s3Url: 'https://s3.example.com/small.bin',
431 mimeType: 'text/plain',
432 maxDownloadBytes: 1024,
433 });
434
435 expect(result.filePath).toBeDefined();
436 expect(fs.writeFileSync).toHaveBeenCalledWith(result.filePath, body);
437 });
438 });
439
440 describe('File object handling', () => {
441 it('should handle File objects correctly', async () => {
442 // Reset mocks
443 vi.clearAllMocks();
444
445 // Mock successful S3 upload
446 (mockClient.files.createPresignedURL as unknown as Mock).mockResolvedValueOnce({
447 key: 'test-key',
448 type: 'new',
449 new_presigned_url: 'https://s3.example.com/upload',
450 });
451
452 // Mock successful PUT to S3
453 mockFetch.mockResolvedValueOnce({ ok: true });
454
455 const fileContent = 'test file content';
456 const file = new File([fileContent], 'test.txt', { type: 'text/plain' });
457
458 const result = await getFileDataAfterUploadingToS3(file, {
459 toolSlug: 'test-tool',
460 toolkitSlug: 'test-toolkit',
461 client: mockClient,
462 });
463
464 expect(result.name).toBe('test.txt');
465 expect(result.mimetype).toBe('text/plain');
466 });
467 });
468
469 // Note: Local file handling tests are covered in the existing fileModifiers.test.ts
470
471 describe('Error handling', () => {
472 it('refuses sensitive local paths before read or S3 (built-in denylist)', async () => {
473 await expect(
474 getFileDataAfterUploadingToS3(path.join('/home/test', '.aws', 'creds'), {
475 toolSlug: 'test-tool',
476 toolkitSlug: 'test-toolkit',
477 client: mockClient,
478 })
479 ).rejects.toThrow(ComposioSensitiveFilePathBlockedError);
480 });
481
482 it('treats a local path that merely starts with "http" as a path, not a URL', async () => {
483 // Regression test: a naive `.startsWith('http')` check would misclassify
484 // this as a URL and skip the local-path denylist entirely.
485 await expect(
486 getFileDataAfterUploadingToS3(path.join('http_export', '.aws', 'creds'), {
487 toolSlug: 'test-tool',
488 toolkitSlug: 'test-toolkit',
489 client: mockClient,
490 })
491 ).rejects.toThrow(ComposioSensitiveFilePathBlockedError);
492
493 // It must not have been routed through the URL-fetch branch.
494 expect(mockFetch).not.toHaveBeenCalled();
495 });
496
497 it('should handle fetch errors for URLs', async () => {
498 const cancel = vi.fn();
499 const response = new Response(new ReadableStream({ cancel }), {
500 status: 500,
501 statusText: 'Internal Server Error',
502 });
503 mockFetch.mockResolvedValue(response);
504
505 await expect(
506 getFileDataAfterUploadingToS3('https://example.com/file.pdf', {
507 toolSlug: 'test-tool',
508 toolkitSlug: 'test-toolkit',
509 client: mockClient,
510 })
511 ).rejects.toThrow('Failed to fetch file: Internal Server Error');
512
513 expect(cancel).toHaveBeenCalledOnce();
514 expect(response.bodyUsed).toBe(true);
515 });
516
517 it('should handle S3 upload errors', async () => {
518 mockFetch.mockResolvedValueOnce(fetchedFileResponse());
519
520 (mockClient.files.createPresignedURL as unknown as Mock).mockResolvedValue({
521 key: 'test-key',
522 type: 'new',
523 new_presigned_url: 'https://s3.example.com/upload',
524 });
525
526 // Mock failed S3 upload
527 mockFetch.mockResolvedValueOnce({
528 ok: false,
529 statusText: 'Upload Failed',
530 });
531
532 await expect(
533 getFileDataAfterUploadingToS3('https://example.com/file.pdf', {
534 toolSlug: 'test-tool',
535 toolkitSlug: 'test-toolkit',
536 client: mockClient,
537 })
538 ).rejects.toThrow('Failed to upload file to S3: Upload Failed');
539 });
540
541 it('should handle invalid file types', async () => {
542 await expect(
543 getFileDataAfterUploadingToS3(123 as unknown as string, {
544 toolSlug: 'test-tool',
545 toolkitSlug: 'test-toolkit',
546 client: mockClient,
547 })
548 ).rejects.toThrow('Invalid file type');
549 });
550 });
551 describe('SSRF guard on response-supplied URLs', () => {
552 // The S3 URLs below are API response fields, not caller input. The SDK
553 // treats a response as untrusted, so both directions are validated: a
554 // download would otherwise fetch an internal address and write the bytes
555 // to disk, and an upload would PUT the user's file to one.
556 it('should block a download whose s3Url resolves internally', async () => {
557 mockLookup.mockResolvedValueOnce([{ address: '169.254.169.254', family: 4 }] as never);
558
559 await expect(
560 downloadFileFromS3({
561 toolSlug: 'github',
562 s3Url: 'http://metadata.example/latest/meta-data/',
563 mimeType: 'text/plain',
564 })
565 ).rejects.toBeInstanceOf(ComposioBlockedInternalUrlError);
566
567 expect(mockFetch).not.toHaveBeenCalled();
568 });
569
570 it('should block an upload whose presigned URL resolves internally', async () => {
571 vi.mocked(mockClient.files.createPresignedURL).mockResolvedValue({
572 key: 'test-key',
573 type: 'new',
574 new_presigned_url: 'http://127.0.0.1:9000/upload',
575 } as never);
576 // The file content is read first, then the presigned URL is validated.
577 mockLookup.mockResolvedValueOnce([{ address: '127.0.0.1', family: 4 }] as never);
578
579 await expect(
580 getFileDataAfterUploadingToS3(new File(['data'], 'report.txt', { type: 'text/plain' }), {
581 toolSlug: 'test-tool',
582 toolkitSlug: 'test-toolkit',
583 client: mockClient,
584 })
585 ).rejects.toBeInstanceOf(ComposioBlockedInternalUrlError);
586
587 expect(mockFetch).not.toHaveBeenCalled();
588 });
589 });
590 });