Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 501 lines 16.4 KB
Raw
1 /* eslint-disable-next-line no-restricted-imports */
2 import crypto from 'node:crypto'; // we're in a Node.js-specific module
3 import { platform } from '#platform';
4 import ComposioClient from '@composio/client';
5 import { COMPOSIO_DIR, TEMP_FILES_DIRECTORY_NAME } from './constants';
6 import logger from './logger';
7 import { getRandomShortId } from './uuid';
8 import { base64ToUint8Array, uint8ArrayToBase64 } from './buffer';
9 import type { FileDownloadData, FileUploadData } from '../types/files.types';
10 import { assertSafeFileUploadPath } from './sensitiveFileUploadPaths';
11 import { assertPathInsideUploadDirs } from './uploadDirAllowlist.node';
12 import { ssrfSafeFetch } from './ssrfGuard.node';
13 import { readResponseBodyWithLimit } from './readResponseBody';
14 import { getMimeTypeFromFileName } from './mime';
15
16 /**
17 * Options for {@link getFileDataAfterUploadingToS3} (S3 presigned upload from local path, URL, or File).
18 */
19 export type GetFileDataAfterUploadingToS3Options = {
20 toolSlug: string;
21 toolkitSlug: string;
22 client: ComposioClient;
23 /**
24 * When false, skips the sensitive-path blocklist. URLs and {@link File} objects are never checked.
25 * @default true
26 */
27 sensitiveFileUploadProtection?: boolean;
28 /**
29 * Extra path segments (one directory/file component) to block anywhere in the resolved path;
30 * merged with the built-in list.
31 */
32 fileUploadPathDenySegments?: string[];
33 /**
34 * When provided, the local path must resolve inside one of these directories
35 * (after symlink resolution, on a component boundary) or upload is rejected.
36 * Intended for automatic upload during tool execution; manual
37 * `composio.files.upload()` calls pass `undefined` here to skip enforcement.
38 *
39 * An empty array means "no paths are allowed" (fail-closed).
40 */
41 fileUploadAllowlist?: string[];
42 signal?: AbortSignal;
43 };
44
45 /**
46 * Checks whether `value` is an absolute http(s) URL rather than a local path.
47 *
48 * A bare `startsWith('http')` check also matches local paths like
49 * `http_export/report.csv` or `httpdocs/.env`, which routes them into the URL
50 * fetch branch and skips the allowlist/denylist checks in
51 * {@link getFileDataAfterUploadingToS3}. `new URL()` requires a real scheme
52 * (`http:` or `https:`) followed by `//`, so relative-looking paths correctly
53 * fail to parse and are treated as local.
54 */
55 export const isHttpUrl = (value: string): boolean => {
56 try {
57 const url = new URL(value);
58 return url.protocol === 'http:' || url.protocol === 'https:';
59 } catch {
60 return false;
61 }
62 };
63
64 // Helper function to get file extension from MIME type
65 const getExtensionFromMimeType = (mimeType: string): string => {
66 const mimeToExt: Record<string, string> = {
67 'text/plain': 'txt',
68 'text/html': 'html',
69 'text/css': 'css',
70 'text/csv': 'csv',
71 'text/javascript': 'js',
72 'application/json': 'json',
73 'application/xml': 'xml',
74 'application/pdf': 'pdf',
75 'application/zip': 'zip',
76 'application/x-zip-compressed': 'zip',
77 'application/gzip': 'gz',
78 'application/x-tar': 'tar',
79 'application/msword': 'doc',
80 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': 'docx',
81 'application/vnd.ms-excel': 'xls',
82 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': 'xlsx',
83 'application/vnd.ms-powerpoint': 'ppt',
84 'application/vnd.openxmlformats-officedocument.presentationml.presentation': 'pptx',
85 'image/jpeg': 'jpg',
86 'image/jpg': 'jpg',
87 'image/png': 'png',
88 'image/gif': 'gif',
89 'image/svg+xml': 'svg',
90 'image/webp': 'webp',
91 'image/bmp': 'bmp',
92 'image/tiff': 'tiff',
93 'audio/mpeg': 'mp3',
94 'audio/wav': 'wav',
95 'audio/ogg': 'ogg',
96 'video/mp4': 'mp4',
97 'video/mpeg': 'mpeg',
98 'video/quicktime': 'mov',
99 'video/x-msvideo': 'avi',
100 'video/webm': 'webm',
101 };
102
103 // Clean the MIME type by removing parameters (e.g., "text/plain; charset=utf-8" -> "text/plain")
104 const cleanMimeType = mimeType.split(';')[0].toLowerCase().trim();
105
106 // Try exact match first
107 // An own property only: `constructor` or `__proto__` must not reach Object.prototype.
108 if (Object.hasOwn(mimeToExt, cleanMimeType)) {
109 return mimeToExt[cleanMimeType];
110 }
111
112 // Try to extract from the MIME type (e.g., "image/png" -> "png")
113 const parts = cleanMimeType.split('/');
114 if (parts.length === 2) {
115 const subtype = parts[1].toLowerCase();
116 const cleanSubtype = subtype; // Already cleaned above
117
118 // Handle structured MIME types with + (e.g., "application/vnd.api+json" -> "json")
119 if (cleanSubtype.includes('+')) {
120 const plusParts = cleanSubtype.split('+');
121 const prefix = plusParts[0];
122 const suffix = plusParts[plusParts.length - 1]; // Get the part after the last +
123
124 // For specific formats, prefer the prefix (e.g., "atom+xml" -> "atom")
125 const knownPrefixes = ['svg', 'atom', 'rss'];
126 if (knownPrefixes.includes(prefix)) {
127 return prefix;
128 }
129
130 // For common structured suffixes, use the suffix (e.g., "vnd.api+json" -> "json")
131 const structuredSuffixes = ['json', 'xml', 'yaml', 'zip', 'gzip'];
132 if (structuredSuffixes.includes(suffix)) {
133 return suffix;
134 }
135
136 // Default to the suffix for other structured types
137 return suffix;
138 }
139
140 return cleanSubtype || 'txt';
141 }
142
143 return 'txt'; // Default fallback
144 };
145
146 // Helper function to generate a filename with timestamp and random ID
147 const generateTimestampedFilename = (extension: string, prefix?: string): string => {
148 const basePrefix = prefix || 'file_ts';
149 return `${basePrefix}${Date.now()}${getRandomShortId()}.${extension}`;
150 };
151
152 const readFileContent = async (
153 filePath: string
154 ): Promise<{ fileName: string; content: string; mimeType: string }> => {
155 try {
156 if (!platform.supportsFileSystem) {
157 throw new Error('File system operations are not supported in this runtime environment');
158 }
159 const content = platform.readFileSync(filePath);
160 return {
161 fileName: generateTimestampedFilename(filePath.split('.').pop() || 'txt'),
162 content:
163 content instanceof Uint8Array
164 ? uint8ArrayToBase64(content)
165 : uint8ArrayToBase64(new TextEncoder().encode(content)),
166 mimeType: getMimeTypeFromFileName(filePath),
167 };
168 } catch (error) {
169 throw new Error(`Error reading file at ${filePath}: ${error}`);
170 }
171 };
172
173 const readFileContentFromURL = async (
174 path: string,
175 signal?: AbortSignal
176 ): Promise<{ fileName: string; content: string; mimeType: string }> => {
177 // SSRF guard: `path` is user-supplied (and can come from an LLM-produced tool
178 // argument), so it must not be allowed to reach internal/private addresses or
179 // redirect into them. See ssrfGuard.node.ts.
180 const response = await ssrfSafeFetch(path, { signal });
181 if (!response.ok) {
182 // The error path never reads the body, so release it explicitly (mirrors
183 // `readResponseBodyWithLimit`) instead of leaving it to the garbage collector.
184 await response.body?.cancel().catch(() => undefined);
185 throw new Error(`Failed to fetch file: ${response.statusText}`);
186 }
187 const content = await readResponseBodyWithLimit(response);
188 const mimeType = response.headers.get('content-type') || 'application/octet-stream';
189
190 // Extract clean filename from URL, removing query parameters
191 const url = new URL(path);
192 const pathname = url.pathname;
193 let fileName = platform.basename(pathname);
194
195 // If no filename from URL, generate one with appropriate extension
196 if (!fileName || fileName === '/') {
197 // Try to get extension from mimeType
198 const extension = getExtensionFromMimeType(mimeType);
199 fileName = generateTimestampedFilename(extension);
200 } else {
201 // If filename has no extension, try to add one from mimeType
202 const hasExtension = fileName.includes('.');
203 if (!hasExtension) {
204 const extension = getExtensionFromMimeType(mimeType);
205 fileName = generateTimestampedFilename(extension);
206 }
207 }
208
209 return {
210 content: uint8ArrayToBase64(content),
211 mimeType,
212 fileName,
213 };
214 };
215
216 const uploadFileToS3 = async (
217 fileName: string,
218 content: string,
219 toolSlug: string,
220 toolkitSlug: string,
221 mimeType: string,
222 client: ComposioClient,
223 signal?: AbortSignal
224 ): Promise<string> => {
225 const contentBytes = base64ToUint8Array(content);
226 const response = await client.files.createPresignedURL(
227 {
228 filename: fileName,
229 mimetype: mimeType,
230 md5: crypto.createHash('md5').update(contentBytes).digest('hex'),
231 tool_slug: toolSlug,
232 toolkit_slug: toolkitSlug,
233 },
234 { signal }
235 );
236
237 const { key, new_presigned_url: signedURL } = response;
238
239 // Upload file using presigned URL
240 // Note: API now always returns type: 'new' with new_presigned_url
241 logger.debug(`Uploading ${key} file to S3: ${key}`);
242
243 // Create a new ArrayBuffer to ensure compatibility with fetch API
244 const uploadBuffer = new Uint8Array(contentBytes.byteLength);
245 uploadBuffer.set(contentBytes);
246
247 // Note: do not set `Content-Length` manually. Node.js's built-in fetch (undici)
248 // computes it from `body` and rejects any user-supplied value with
249 // `InvalidArgumentError: invalid content-length header`, which surfaces as
250 // `TypeError: fetch failed` and breaks every `file_uploadable` connector tool
251 // on Node 22+.
252 // SSRF guard: `new_presigned_url` comes from the API response, which the SDK
253 // treats as untrusted — a response naming an internal address would otherwise
254 // have the file's bytes PUT to it. See ssrfGuard.node.ts.
255 const uploadResponse = await ssrfSafeFetch(signedURL, {
256 method: 'PUT',
257 body: uploadBuffer,
258 signal,
259 headers: {
260 'Content-Type': mimeType,
261 },
262 });
263
264 if (!uploadResponse.ok) {
265 throw new Error(`Failed to upload file to S3: ${uploadResponse.statusText}`);
266 }
267
268 return key;
269 };
270
271 const readFile = async (
272 file: File | string,
273 signal?: AbortSignal
274 ): Promise<{ fileName: string; content: string; mimeType: string }> => {
275 if (file instanceof File) {
276 const content = await file.arrayBuffer();
277 return {
278 fileName: file.name,
279 content: uint8ArrayToBase64(new Uint8Array(content)),
280 mimeType: file.type,
281 };
282 } else if (typeof file === 'string') {
283 if (isHttpUrl(file)) {
284 return await readFileContentFromURL(file, signal);
285 } else {
286 return await readFileContent(file);
287 }
288 }
289 throw new Error('Invalid file type');
290 };
291
292 export const getFileDataAfterUploadingToS3 = async (
293 file: File | string,
294 {
295 toolSlug,
296 toolkitSlug,
297 client,
298 sensitiveFileUploadProtection,
299 fileUploadPathDenySegments,
300 fileUploadAllowlist,
301 signal,
302 }: GetFileDataAfterUploadingToS3Options
303 ): Promise<FileUploadData> => {
304 if (!file) {
305 throw new Error('Either path or blob must be provided');
306 }
307
308 const isLocalPath = typeof file === 'string' && !isHttpUrl(file);
309
310 if (isLocalPath && fileUploadAllowlist !== undefined) {
311 assertPathInsideUploadDirs(file as string, fileUploadAllowlist);
312 }
313
314 if (sensitiveFileUploadProtection !== false && isLocalPath) {
315 assertSafeFileUploadPath(file as string, {
316 additionalDenySegments: fileUploadPathDenySegments,
317 });
318 }
319
320 const fileData = await readFile(file, signal);
321 logger.debug(`Uploading file to S3...`);
322 const s3key = await uploadFileToS3(
323 platform.basename(fileData.fileName),
324 fileData.content,
325 toolSlug,
326 toolkitSlug,
327 fileData.mimeType,
328 client,
329 signal
330 );
331
332 logger.debug(`Done! File uploaded to S3: ${s3key}`, JSON.stringify(fileData, null, 2));
333 return {
334 name: fileData.fileName,
335 mimetype: fileData.mimeType,
336 s3key: s3key,
337 };
338 };
339
340 export const downloadFileFromS3 = async ({
341 toolSlug,
342 s3Url,
343 mimeType,
344 fileDownloadDir,
345 signal,
346 maxDownloadBytes,
347 }: {
348 toolSlug: string;
349 s3Url: string;
350 mimeType: string;
351 /**
352 * Absolute path to the directory to save the file into. When omitted, falls
353 * back to `<home>/.composio/files` (the legacy default).
354 */
355 fileDownloadDir?: string;
356 signal?: AbortSignal;
357 /**
358 * Maximum number of bytes to read from the response before rejecting.
359 * Defaults to {@link MAX_URL_UPLOAD_SIZE_BYTES} (100 MiB).
360 */
361 maxDownloadBytes?: number;
362 }): Promise<FileDownloadData> => {
363 // SSRF guard: `s3Url` is a field of the tool-execution response, so it is no
364 // more trusted than a user-supplied URL — and the bytes it returns are
365 // written to disk. See ssrfGuard.node.ts.
366 const response = await ssrfSafeFetch(s3Url, { signal });
367 if (!response.ok) {
368 throw new Error(`Failed to download file: ${response.statusText}`);
369 }
370 // The response is attacker-influencable and streamed to disk, so the body is
371 // read through the shared size guard rather than buffered wholesale.
372 const data = await readResponseBodyWithLimit(response, maxDownloadBytes);
373
374 const extension = getExtensionFromMimeType(mimeType);
375 const fileName = generateTimestampedFilename(extension, `${toolSlug}_`);
376 const filePath = saveFile(fileName, data, {
377 isTempFile: fileDownloadDir === undefined,
378 outputDir: fileDownloadDir,
379 });
380 return {
381 name: fileName,
382 mimeType: mimeType,
383 s3Url: s3Url,
384
385 /**
386 * @todo: fix in follow-up PR.
387 */
388 filePath: filePath as string,
389 };
390 };
391
392 /**
393 * Gets the Composio directory.
394 * @param createDirIfNotExists - Whether to create the directory if it doesn't exist.
395 * @returns The path to the Composio directory.
396 */
397 export const getComposioDir = (createDirIfNotExists: boolean = false) => {
398 try {
399 const homeDir = platform.homedir();
400 if (!homeDir) {
401 return null;
402 }
403 const composioDir = platform.joinPath(homeDir, COMPOSIO_DIR);
404 if (createDirIfNotExists && platform.supportsFileSystem && !platform.existsSync(composioDir)) {
405 platform.mkdirSync(composioDir);
406 }
407 return composioDir;
408 } catch (_error) {
409 return null;
410 }
411 };
412
413 /**
414 * Gets the Composio temporary files directory.
415 * @param createDirIfNotExists - Whether to create the directory if it doesn't exist.
416 * @returns The path to the Composio temporary files directory.
417 */
418 export const getComposioTempFilesDir = (createDirIfNotExists: boolean = false) => {
419 try {
420 const homeDir = platform.homedir();
421 if (!homeDir) {
422 return null;
423 }
424 const composioFilesDir = platform.joinPath(homeDir, COMPOSIO_DIR, TEMP_FILES_DIRECTORY_NAME);
425 if (
426 createDirIfNotExists &&
427 platform.supportsFileSystem &&
428 !platform.existsSync(composioFilesDir)
429 ) {
430 platform.mkdirSync(composioFilesDir);
431 }
432 return composioFilesDir;
433 } catch (_error) {
434 return null;
435 }
436 };
437
438 /**
439 * Saves a file to a local directory.
440 *
441 * Directory precedence (highest first):
442 * 1. `options.outputDir` (explicit, absolute or relative; created if missing)
443 * 2. `options.isTempFile === true` → `<home>/.composio/files`
444 * 3. fallback → `<home>/.composio`
445 *
446 * @returns The absolute path to the saved file, or `null` if the runtime has
447 * no filesystem (e.g. Cloudflare Workers) or the save failed.
448 */
449 export const saveFile = (
450 file: string,
451 content: string | Uint8Array,
452 options: { isTempFile?: boolean; outputDir?: string } | boolean = {}
453 ) => {
454 // Back-compat: legacy callers passed a boolean `isTempFile` positional.
455 const opts: { isTempFile?: boolean; outputDir?: string } =
456 typeof options === 'boolean' ? { isTempFile: options } : options;
457
458 try {
459 if (!platform.supportsFileSystem) {
460 logger.debug('File system operations are not supported in this runtime environment');
461 return null;
462 }
463
464 let composioFilesDir: string | null | undefined;
465 if (opts.outputDir) {
466 const absDir = platform.joinPath(opts.outputDir); // platform.joinPath normalizes
467 try {
468 if (!platform.existsSync(absDir)) {
469 platform.mkdirSync(absDir);
470 }
471 composioFilesDir = absDir;
472 } catch (err) {
473 logger.warn(
474 `fileDownloadDir "${absDir}" is not writable, falling back to the default ` +
475 `(~/.composio/files). Error: ${err}`
476 );
477 composioFilesDir = getComposioTempFilesDir(true);
478 }
479 } else {
480 composioFilesDir = opts.isTempFile ? getComposioTempFilesDir(true) : getComposioDir(true);
481 }
482
483 if (!composioFilesDir) {
484 return null;
485 }
486 const filePath = platform.joinPath(composioFilesDir, platform.basename(file));
487
488 logger.info(`Saving file to: ${filePath}`);
489
490 if (content instanceof Uint8Array) {
491 platform.writeFileSync(filePath, content);
492 } else {
493 platform.writeFileSync(filePath, content, 'utf8');
494 }
495
496 return filePath;
497 } catch (_error) {
498 logger.debug(`Error saving file: ${_error}`);
499 return null;
500 }
501 };