Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | from __future__ import annotations |
| 2 | |
| 3 | import contextlib |
| 4 | import functools |
| 5 | import hashlib |
| 6 | import os |
| 7 | import typing as t |
| 8 | from pathlib import Path |
| 9 | from urllib.parse import unquote, urlparse |
| 10 | import uuid |
| 11 | import datetime |
| 12 | |
| 13 | import requests |
| 14 | import typing_extensions as te |
| 15 | from composio_client.types.file_create_presigned_url_response import ( |
| 16 | FileCreatePresignedURLResponse, |
| 17 | ) |
| 18 | from pydantic import BaseModel, ConfigDict, Field |
| 19 | |
| 20 | from composio.client import HttpClient |
| 21 | from composio.client.types import Tool |
| 22 | from composio.exceptions import ( |
| 23 | ErrorDownloadingFile, |
| 24 | ErrorUploadingFile, |
| 25 | FileUploadAbortedError, |
| 26 | ResponseTooLargeError, |
| 27 | SDKFileNotFoundError, |
| 28 | UnsafePathComponentError, |
| 29 | ) |
| 30 | from composio.utils import mimetypes |
| 31 | from composio.utils.json_schema import dereference_json_schema |
| 32 | from composio.utils.safe_path import open_unique_file, secure_basename_join, secure_join |
| 33 | from composio.utils.url_safety import ( |
| 34 | parse_content_length, |
| 35 | safe_get, |
| 36 | safe_request, |
| 37 | ) |
| 38 | from composio.utils.sensitive_file_upload_paths import ( |
| 39 | assert_safe_local_file_upload_path, |
| 40 | ) |
| 41 | from composio.utils.upload_dir_allowlist import ( |
| 42 | assert_path_inside_upload_dirs, |
| 43 | ) |
| 44 | from composio.utils.logging import WithLogger |
| 45 | |
| 46 | if t.TYPE_CHECKING: |
| 47 | from .tools import ToolExecutionResponse |
| 48 | from ._modifiers import BeforeFileUploadContextCallable # noqa: F401 |
| 49 | |
| 50 | _DEFAULT_CHUNK_SIZE = 1024 * 1024 |
| 51 | _MAX_FILENAME_LENGTH = 100 |
| 52 | """ |
| 53 | Maximum filename length to prevent issues with long URLs from public buckets. |
| 54 | Long filenames (containing hashes, UUIDs, or encoded metadata) are replaced |
| 55 | with timestamped filenames to match TypeScript SDK behavior. |
| 56 | """ |
| 57 | |
| 58 | _MAX_RESPONSE_SIZE = 100 * 1024 * 1024 # 100 MB default limit |
| 59 | """ |
| 60 | Maximum response size in bytes when fetching files from URLs. |
| 61 | Prevents memory exhaustion attacks from malicious URLs pointing to large files. |
| 62 | """ |
| 63 | |
| 64 | _CONNECT_TIMEOUT = 5 # seconds |
| 65 | _READ_TIMEOUT = 60 # seconds |
| 66 | """ |
| 67 | Separate connect and read timeouts for URL fetching. |
| 68 | Connect timeout is short to fail fast on unreachable hosts. |
| 69 | Read timeout is longer to allow for slower file transfers. |
| 70 | """ |
| 71 | |
| 72 | _DELETE_VALUE: t.Final = object() |
| 73 | """ |
| 74 | Sentinel returned by the upload walker to signal that a value should be dropped |
| 75 | from its parent container. ``None`` and ``""`` are both legal payload values, so |
| 76 | the walker cannot use either to mean "remove this key/item". |
| 77 | """ |
| 78 | |
| 79 | LOCAL_CACHE_DIRECTORY_NAME = ".composio" |
| 80 | """ |
| 81 | Local cache directory name for composio CLI |
| 82 | """ |
| 83 | |
| 84 | ENV_LOCAL_CACHE_DIRECTORY = "COMPOSIO_CACHE_DIR" |
| 85 | """ |
| 86 | Environment to set the composio caching directory. |
| 87 | """ |
| 88 | |
| 89 | LOCAL_OUTPUT_FILE_DIRECTORY_NAME = "files" |
| 90 | """ |
| 91 | Name of the cache sub-directory into which files downloaded during tool |
| 92 | execution are written. Previously ``outputs``; now ``files`` for parity with |
| 93 | the TypeScript SDK. |
| 94 | """ |
| 95 | |
| 96 | |
| 97 | def get_cache_directory() -> Path: |
| 98 | """Resolve the local caching directory without touching the filesystem. |
| 99 | |
| 100 | ``COMPOSIO_CACHE_DIR`` is read on every call, so it can be set after |
| 101 | ``composio`` has already been imported. ``Path.home()`` is only consulted |
| 102 | when the variable is unset: it can raise ``RuntimeError`` when there is no |
| 103 | resolvable home directory, which is exactly the situation |
| 104 | ``COMPOSIO_CACHE_DIR`` exists to work around, so it must not be evaluated |
| 105 | eagerly as a fallback argument. |
| 106 | """ |
| 107 | configured = os.environ.get(ENV_LOCAL_CACHE_DIRECTORY) |
| 108 | if configured: |
| 109 | return Path(configured) |
| 110 | |
| 111 | try: |
| 112 | home = Path.home() |
| 113 | except RuntimeError as e: |
| 114 | raise RuntimeError( |
| 115 | "Could not determine a home directory to store the Composio cache " |
| 116 | f"in. Provide a writable path using the {ENV_LOCAL_CACHE_DIRECTORY} " |
| 117 | "environment variable." |
| 118 | ) from e |
| 119 | return home / LOCAL_CACHE_DIRECTORY_NAME |
| 120 | |
| 121 | |
| 122 | def get_output_file_directory() -> Path: |
| 123 | """Default local directory into which files downloaded during tool |
| 124 | execution are written. Override by passing ``file_download_dir=...`` to |
| 125 | Composio, or by setting ``outdir`` on ``FileHelper`` directly. |
| 126 | """ |
| 127 | return get_cache_directory() / LOCAL_OUTPUT_FILE_DIRECTORY_NAME |
| 128 | |
| 129 | |
| 130 | def ensure_cache_directory() -> Path: |
| 131 | """Create the cache directory on first use and check that it is writable. |
| 132 | |
| 133 | This used to run at module import time, so a bare ``import composio`` |
| 134 | raised ``RuntimeError`` on any read-only filesystem -- AWS Lambda, |
| 135 | distroless containers, ``ProtectHome=true`` systemd units -- even for |
| 136 | programs that never touched a file. Deferring it to first use keeps the |
| 137 | same check, and the same error message, for the callers that actually |
| 138 | need the directory. |
| 139 | """ |
| 140 | directory = get_cache_directory() |
| 141 | try: |
| 142 | directory.mkdir(parents=True, exist_ok=True) |
| 143 | if not os.access(directory, os.W_OK): |
| 144 | raise OSError |
| 145 | except OSError as e: |
| 146 | raise RuntimeError( |
| 147 | f"Cache directory {directory} is not writable please " |
| 148 | f"provide a path that is writable using {ENV_LOCAL_CACHE_DIRECTORY} " |
| 149 | "environment variable." |
| 150 | ) from e |
| 151 | return directory |
| 152 | |
| 153 | |
| 154 | def __getattr__(name: str) -> Path: |
| 155 | """Keep the historical module-level path constants working, but lazily. |
| 156 | |
| 157 | ``LOCAL_CACHE_DIRECTORY`` and ``LOCAL_OUTPUT_FILE_DIRECTORY`` used to be |
| 158 | computed at import time. They are now resolved on attribute access |
| 159 | instead (PEP 562), so importing this module no longer touches the |
| 160 | filesystem or depends on the environment, and both constants observe a |
| 161 | ``COMPOSIO_CACHE_DIR`` that was set after import. |
| 162 | """ |
| 163 | if name == "LOCAL_CACHE_DIRECTORY": |
| 164 | return get_cache_directory() |
| 165 | if name == "LOCAL_OUTPUT_FILE_DIRECTORY": |
| 166 | return get_output_file_directory() |
| 167 | raise AttributeError(f"module {__name__!r} has no attribute {name!r}") |
| 168 | |
| 169 | |
| 170 | def get_md5(file: Path) -> str: |
| 171 | """Calculate MD5 hash of a file for integrity verification. |
| 172 | |
| 173 | Note: MD5 is used here for file integrity checking and deduplication, |
| 174 | not for cryptographic security. The Composio API requires MD5 hashes |
| 175 | for file upload verification. For security-critical applications, |
| 176 | consider using SHA-256 for additional integrity checks. |
| 177 | |
| 178 | Args: |
| 179 | file: Path to file to hash |
| 180 | |
| 181 | Returns: |
| 182 | Hexadecimal MD5 hash string |
| 183 | """ |
| 184 | # `usedforsecurity=False` lets this run on FIPS-mode systems, where |
| 185 | # `hashlib.md5()` without the flag raises `ValueError: [digital envelope |
| 186 | # routines] unsupported`. We're hashing for integrity / deduplication, |
| 187 | # not security — the API just needs the digest for upload verification. |
| 188 | obj = hashlib.md5(usedforsecurity=False) |
| 189 | with file.open("rb") as fp: |
| 190 | while True: |
| 191 | line = fp.read(_DEFAULT_CHUNK_SIZE) |
| 192 | if not line: |
| 193 | break |
| 194 | obj.update(line) |
| 195 | return obj.hexdigest() |
| 196 | |
| 197 | |
| 198 | def _upload_to_presigned_url( |
| 199 | url: str, data: t.Union[bytes, t.IO[bytes]], mimetype: str |
| 200 | ) -> None: |
| 201 | """PUT ``data`` to a presigned S3 URL with the content type it was signed with. |
| 202 | |
| 203 | The presign request carries ``mimetype``, so the PUT must send the same |
| 204 | value as ``Content-Type``: when the signature covers the content type, a |
| 205 | mismatched or missing header is rejected with ``403 SignatureDoesNotMatch``. |
| 206 | Routing every presigned PUT through one helper keeps the file and bytes |
| 207 | upload paths from drifting apart again, mirroring ``uploadFileToS3`` in |
| 208 | the TypeScript SDK, which funnels path, URL, and File inputs through a |
| 209 | single uploader. |
| 210 | |
| 211 | Raises: |
| 212 | ErrorUploadingFile: On transport failure or a non-200 response, |
| 213 | including the HTTP status when one was received. |
| 214 | """ |
| 215 | try: |
| 216 | response = safe_request( |
| 217 | "PUT", |
| 218 | url, |
| 219 | data=data, |
| 220 | headers={"Content-Type": mimetype}, |
| 221 | timeout=(_CONNECT_TIMEOUT, _READ_TIMEOUT), |
| 222 | ) |
| 223 | except requests.exceptions.RequestException as e: |
| 224 | raise ErrorUploadingFile( |
| 225 | "Failed to upload to S3: " |
| 226 | f"{_sanitize_url_for_logging(url)}. Error: {type(e).__name__}" |
| 227 | ) from e |
| 228 | if response.status_code != 200: |
| 229 | raise ErrorUploadingFile( |
| 230 | f"Failed to upload to S3. Status: {response.status_code}. " |
| 231 | "This may indicate an expired presigned URL or permission issue." |
| 232 | ) |
| 233 | |
| 234 | |
| 235 | def upload(url: str, file: Path, mimetype: t.Optional[str] = None) -> bool: |
| 236 | """Upload file to presigned S3 URL. |
| 237 | |
| 238 | Args: |
| 239 | url: Presigned S3 upload URL |
| 240 | file: Path to file to upload |
| 241 | mimetype: Content type to send with the upload. Defaults to the type |
| 242 | guessed from ``file``. This must match the ``mimetype`` the |
| 243 | presigned URL was requested with, otherwise S3 rejects the PUT |
| 244 | with ``403 SignatureDoesNotMatch`` when the signature covers the |
| 245 | content type. |
| 246 | |
| 247 | Returns: |
| 248 | True if the upload succeeded. |
| 249 | |
| 250 | Raises: |
| 251 | ErrorUploadingFile: If the upload fails; the message includes the |
| 252 | HTTP status when one was received. |
| 253 | """ |
| 254 | if mimetype is None: |
| 255 | mimetype = mimetypes.guess(file=file) |
| 256 | with file.open("rb") as data: |
| 257 | _upload_to_presigned_url(url=url, data=data, mimetype=mimetype) |
| 258 | return True |
| 259 | |
| 260 | |
| 261 | def _request_presigned_upload( |
| 262 | client: HttpClient, |
| 263 | *, |
| 264 | filename: str, |
| 265 | md5: str, |
| 266 | mimetype: str, |
| 267 | tool: str, |
| 268 | toolkit: str, |
| 269 | ) -> FileCreatePresignedURLResponse: |
| 270 | """Request a presigned S3 upload URL from the backend. |
| 271 | |
| 272 | Single-sources the presign wire shape so the file and bytes upload paths |
| 273 | request the same fields they later send. |
| 274 | """ |
| 275 | return client.files.create_presigned_url( |
| 276 | md5=md5, |
| 277 | filename=filename, |
| 278 | mimetype=mimetype, |
| 279 | tool_slug=tool, |
| 280 | toolkit_slug=toolkit, |
| 281 | ) |
| 282 | |
| 283 | |
| 284 | def _is_url(value: str) -> bool: |
| 285 | """Check if a string is a valid HTTP/HTTPS URL.""" |
| 286 | try: |
| 287 | parsed = urlparse(value) |
| 288 | return parsed.scheme in ("http", "https") and bool(parsed.netloc) |
| 289 | except Exception: |
| 290 | return False |
| 291 | |
| 292 | |
| 293 | def _get_extension_from_mimetype(mimetype: str) -> str: |
| 294 | """Return the dotted extension for a known content type, or ``""``. |
| 295 | |
| 296 | Looks the type up in the table shared with ``get_extension_from_mime_type`` |
| 297 | but skips that helper's subtype fallback: an unknown or generic binary type |
| 298 | leaves the generated name extensionless instead of inventing an extension. |
| 299 | """ |
| 300 | clean = mimetype.lower() |
| 301 | if clean == "application/octet-stream": |
| 302 | return "" |
| 303 | extension = mimetypes._MIME_TO_EXT.get(clean) |
| 304 | return f".{extension}" if extension else "" |
| 305 | |
| 306 | |
| 307 | def _generate_timestamped_filename(extension: str) -> str: |
| 308 | """Generate a unique filename with timestamp.""" |
| 309 | timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") |
| 310 | unique_id = uuid.uuid4().hex[:8] |
| 311 | return f"file_{timestamp}_{unique_id}{extension}" |
| 312 | |
| 313 | |
| 314 | def _truncate_filename(filename: str, max_length: int = _MAX_FILENAME_LENGTH) -> str: |
| 315 | """Truncate filename if it exceeds max length by generating a timestamped name. |
| 316 | |
| 317 | Long filenames are common with public bucket URLs containing hashes or UUIDs. |
| 318 | These can cause issues, so we replace them with timestamped filenames while |
| 319 | preserving the extension. |
| 320 | |
| 321 | Args: |
| 322 | filename: The original filename |
| 323 | max_length: Maximum allowed length for the filename |
| 324 | |
| 325 | Returns: |
| 326 | The original filename if within limits, or a timestamped filename |
| 327 | with the extension preserved if the original is too long |
| 328 | """ |
| 329 | if len(filename) <= max_length: |
| 330 | return filename |
| 331 | |
| 332 | # Extract extension |
| 333 | if "." in filename: |
| 334 | _, ext = filename.rsplit(".", 1) |
| 335 | ext = f".{ext}" |
| 336 | else: |
| 337 | ext = "" |
| 338 | |
| 339 | # Generate a timestamped filename (matches TypeScript SDK behavior) |
| 340 | return _generate_timestamped_filename(ext) |
| 341 | |
| 342 | |
| 343 | def _sanitize_url_for_logging(url: str) -> str: |
| 344 | """Sanitize URL for safe logging (remove sensitive query params). |
| 345 | |
| 346 | Args: |
| 347 | url: URL to sanitize |
| 348 | |
| 349 | Returns: |
| 350 | URL with query parameters hidden |
| 351 | """ |
| 352 | parsed = urlparse(url) |
| 353 | if parsed.query: |
| 354 | return f"{parsed.scheme}://{parsed.netloc}{parsed.path}?[REDACTED]" |
| 355 | return url |
| 356 | |
| 357 | |
| 358 | def _fetch_file_from_url( |
| 359 | url: str, |
| 360 | max_size: int = _MAX_RESPONSE_SIZE, |
| 361 | ) -> t.Tuple[str, bytes, str]: |
| 362 | """Fetch file content from a URL with security protections. |
| 363 | |
| 364 | Security features: |
| 365 | - Response size limiting (prevents memory exhaustion) |
| 366 | - Redirects disabled (prevents redirect-based attacks) |
| 367 | - Separate connect/read timeouts |
| 368 | |
| 369 | Args: |
| 370 | url: URL to fetch file from |
| 371 | max_size: Maximum response size in bytes (default: 100MB) |
| 372 | |
| 373 | Returns: |
| 374 | Tuple of (filename, content_bytes, mimetype) |
| 375 | |
| 376 | Raises: |
| 377 | ResponseTooLargeError: If response exceeds max_size |
| 378 | ErrorUploadingFile: If fetch fails for other reasons |
| 379 | """ |
| 380 | # `safe_get` validates the target, connects to the address it validated |
| 381 | # (so DNS cannot rebind between the two), and never follows redirects. |
| 382 | try: |
| 383 | response = safe_get( |
| 384 | url, |
| 385 | stream=True, # Enable streaming for size limiting |
| 386 | timeout=(_CONNECT_TIMEOUT, _READ_TIMEOUT), |
| 387 | ) |
| 388 | except requests.exceptions.Timeout: |
| 389 | raise ErrorUploadingFile( |
| 390 | f"Request timed out fetching URL: {_sanitize_url_for_logging(url)}" |
| 391 | ) |
| 392 | except requests.exceptions.RequestException as e: |
| 393 | raise ErrorUploadingFile( |
| 394 | f"Failed to fetch file from URL: {_sanitize_url_for_logging(url)}. Error: {e}" |
| 395 | ) |
| 396 | |
| 397 | # Reject redirects - require direct URL to resource |
| 398 | if response.status_code in (301, 302, 303, 307, 308): |
| 399 | location = response.headers.get("Location", "unknown") |
| 400 | response.close() |
| 401 | raise ErrorUploadingFile( |
| 402 | f"URL returned redirect to {_sanitize_url_for_logging(location)}. " |
| 403 | f"Please provide a direct URL to the file." |
| 404 | ) |
| 405 | |
| 406 | # Check for successful response |
| 407 | if not response.ok: |
| 408 | response.close() |
| 409 | raise ErrorUploadingFile( |
| 410 | f"Failed to fetch file from URL: {_sanitize_url_for_logging(url)}. " |
| 411 | f"Status: {response.status_code}" |
| 412 | ) |
| 413 | |
| 414 | # Check Content-Length header first (early abort for oversized files). |
| 415 | # The header is a hint from the remote server: `parse_content_length` |
| 416 | # returns None for anything untrustworthy, and the streaming guard below |
| 417 | # is the authoritative limit. |
| 418 | content_length = parse_content_length(response.headers.get("Content-Length")) |
| 419 | if content_length is not None and content_length > max_size: |
| 420 | response.close() |
| 421 | raise ResponseTooLargeError( |
| 422 | f"File size ({content_length} bytes) exceeds maximum allowed " |
| 423 | f"size ({max_size} bytes)" |
| 424 | ) |
| 425 | |
| 426 | # Stream response with size tracking |
| 427 | chunks: t.List[bytes] = [] |
| 428 | total_bytes = 0 |
| 429 | chunk_size = 8192 # 8 KB chunks |
| 430 | |
| 431 | try: |
| 432 | for chunk in response.iter_content(chunk_size=chunk_size): |
| 433 | if chunk: |
| 434 | total_bytes += len(chunk) |
| 435 | if total_bytes > max_size: |
| 436 | raise ResponseTooLargeError( |
| 437 | f"Response size exceeds maximum allowed size ({max_size} bytes)" |
| 438 | ) |
| 439 | chunks.append(chunk) |
| 440 | except requests.exceptions.RequestException as e: |
| 441 | raise ErrorUploadingFile( |
| 442 | f"Failed to fetch file from URL: {_sanitize_url_for_logging(url)}. " |
| 443 | f"Error: {e}" |
| 444 | ) from e |
| 445 | finally: |
| 446 | response.close() |
| 447 | |
| 448 | content = b"".join(chunks) |
| 449 | |
| 450 | # Extract mimetype |
| 451 | mimetype = response.headers.get("content-type", "application/octet-stream") |
| 452 | # Handle mimetypes with charset or other parameters (e.g., "text/html; charset=utf-8") |
| 453 | mimetype = mimetype.split(";")[0].strip() |
| 454 | |
| 455 | # Extract filename from URL (decode percent-encoded characters) |
| 456 | parsed_url = urlparse(url) |
| 457 | pathname = unquote(parsed_url.path) |
| 458 | filename = os.path.basename(pathname) if pathname else "" |
| 459 | |
| 460 | # If no filename from URL or no extension, generate one |
| 461 | if not filename: |
| 462 | extension = _get_extension_from_mimetype(mimetype) |
| 463 | filename = _generate_timestamped_filename(extension) |
| 464 | else: |
| 465 | # If filename has no extension, try to add one from mimetype |
| 466 | if "." not in filename: |
| 467 | extension = _get_extension_from_mimetype(mimetype) |
| 468 | filename = _generate_timestamped_filename(extension) |
| 469 | |
| 470 | # Truncate long filenames (common with public bucket URLs containing hashes) |
| 471 | filename = _truncate_filename(filename) |
| 472 | |
| 473 | return filename, content, mimetype |
| 474 | |
| 475 | |
| 476 | def _upload_bytes_to_s3( |
| 477 | client: HttpClient, |
| 478 | filename: str, |
| 479 | content: bytes, |
| 480 | mimetype: str, |
| 481 | tool: str, |
| 482 | toolkit: str, |
| 483 | ) -> str: |
| 484 | """Upload bytes content to S3 and return the S3 key.""" |
| 485 | s3meta = _request_presigned_upload( |
| 486 | client, |
| 487 | filename=filename, |
| 488 | md5=hashlib.md5(content, usedforsecurity=False).hexdigest(), |
| 489 | mimetype=mimetype, |
| 490 | tool=tool, |
| 491 | toolkit=toolkit, |
| 492 | ) |
| 493 | _upload_to_presigned_url( |
| 494 | url=s3meta.new_presigned_url, data=content, mimetype=mimetype |
| 495 | ) |
| 496 | return s3meta.key |
| 497 | |
| 498 | |
| 499 | class FileUploadable(BaseModel): |
| 500 | model_config = ConfigDict(json_schema_extra={"file_uploadable": True}) |
| 501 | |
| 502 | name: str |
| 503 | mimetype: str |
| 504 | s3key: str |
| 505 | |
| 506 | @classmethod |
| 507 | def from_url( |
| 508 | cls, |
| 509 | client: HttpClient, |
| 510 | url: str, |
| 511 | tool: str, |
| 512 | toolkit: str, |
| 513 | ) -> te.Self: |
| 514 | """Create a FileUploadable from a public URL. |
| 515 | |
| 516 | Fetches the file content from the URL and uploads it to S3. |
| 517 | |
| 518 | :param client: The HTTP client for API calls |
| 519 | :param url: The public URL to fetch the file from |
| 520 | :param tool: The tool slug |
| 521 | :param toolkit: The toolkit slug |
| 522 | :return: FileUploadable instance with S3 key |
| 523 | """ |
| 524 | filename, content, mimetype = _fetch_file_from_url(url) |
| 525 | |
| 526 | s3key = _upload_bytes_to_s3( |
| 527 | client=client, |
| 528 | filename=filename, |
| 529 | content=content, |
| 530 | mimetype=mimetype, |
| 531 | tool=tool, |
| 532 | toolkit=toolkit, |
| 533 | ) |
| 534 | |
| 535 | return cls(name=filename, mimetype=mimetype, s3key=s3key) |
| 536 | |
| 537 | @classmethod |
| 538 | def from_path( |
| 539 | cls, |
| 540 | client: HttpClient, |
| 541 | file: t.Union[str, Path], |
| 542 | tool: str, |
| 543 | toolkit: str, |
| 544 | *, |
| 545 | sensitive_file_upload_protection: bool = True, |
| 546 | file_upload_path_deny_segments: t.Optional[t.Sequence[str]] = None, |
| 547 | file_upload_allowlist: t.Optional[t.Sequence[Path]] = None, |
| 548 | before_file_upload: t.Optional["BeforeFileUploadContextCallable"] = None, |
| 549 | ) -> te.Self: |
| 550 | """Create a FileUploadable from a local file path or public URL. |
| 551 | |
| 552 | If the file parameter is a URL (starts with http:// or https://), |
| 553 | it will fetch the file content from the URL and upload it to S3. |
| 554 | Otherwise, it treats it as a local file path. |
| 555 | |
| 556 | :param client: The HTTP client for API calls |
| 557 | :param file: Local file path or public URL |
| 558 | :param tool: The tool slug |
| 559 | :param toolkit: The toolkit slug |
| 560 | :param sensitive_file_upload_protection: When True, block paths on the built-in denylist. |
| 561 | :param file_upload_path_deny_segments: Extra path segment names to merge with the built-in list. |
| 562 | :param file_upload_allowlist: When provided (not None), local paths must |
| 563 | resolve inside one of these directories on a component boundary. |
| 564 | Pass ``None`` to skip the check (e.g. manual upload APIs). URLs are |
| 565 | never checked against the allowlist. An empty sequence means |
| 566 | "no paths are allowed" (fail-closed). |
| 567 | :param before_file_upload: Optional context-form hook produced by |
| 568 | :func:`composio.core.models._modifiers.merge_before_file_upload`. |
| 569 | Receives ``{"path", "source", "tool", "toolkit"}`` where ``source`` |
| 570 | is ``"url"`` for ``http(s)://...`` inputs and ``"path"`` for local |
| 571 | filesystem paths. Return a new string to substitute, or ``False`` |
| 572 | to abort. |
| 573 | :return: FileUploadable instance with S3 key |
| 574 | """ |
| 575 | file_str = str(file) if isinstance(file, Path) else file |
| 576 | path_in = file_str |
| 577 | source: t.Literal["url", "path"] = ( |
| 578 | "url" if isinstance(file_str, str) and _is_url(file_str) else "path" |
| 579 | ) |
| 580 | |
| 581 | if before_file_upload is not None: |
| 582 | out = before_file_upload( |
| 583 | { |
| 584 | "path": path_in, |
| 585 | "source": source, |
| 586 | "tool": tool, |
| 587 | "toolkit": toolkit, |
| 588 | } |
| 589 | ) |
| 590 | if out is False: |
| 591 | raise FileUploadAbortedError( |
| 592 | "File upload was aborted because before_file_upload returned False." |
| 593 | ) |
| 594 | if isinstance(out, str): |
| 595 | path_in = out |
| 596 | |
| 597 | # Re-decide routing on the post-hook value: a URL-source hook may return |
| 598 | # a local path (and vice versa). Re-checking with `_is_url` keeps the |
| 599 | # URL fetch path and the local-file path properly separated, so a hook |
| 600 | # cannot, for example, smuggle `/etc/passwd` past the URL branch's |
| 601 | # missing allowlist/denylist by rewriting the URL into a path. |
| 602 | if isinstance(path_in, str) and _is_url(path_in): |
| 603 | return cls.from_url(client=client, url=path_in, tool=tool, toolkit=toolkit) |
| 604 | |
| 605 | # Allowlist check runs BEFORE the denylist / existence checks when enabled, |
| 606 | # so the "configure file_upload_dirs" hint fires first for the common case |
| 607 | # (user has auto-upload on but hasn't configured dirs). Caller passes |
| 608 | # ``None`` to bypass the allowlist (manual upload APIs). |
| 609 | if file_upload_allowlist is not None: |
| 610 | assert_path_inside_upload_dirs(path_in, file_upload_allowlist) |
| 611 | |
| 612 | assert_safe_local_file_upload_path( |
| 613 | path_in, |
| 614 | enabled=sensitive_file_upload_protection, |
| 615 | additional_deny_segments=file_upload_path_deny_segments, |
| 616 | ) |
| 617 | |
| 618 | # Handle as local file path |
| 619 | file = Path(path_in) |
| 620 | if not file.exists(): |
| 621 | raise SDKFileNotFoundError( |
| 622 | f"File not found: {file}. Please provide a valid file path." |
| 623 | ) |
| 624 | |
| 625 | if not file.is_file(): |
| 626 | raise SDKFileNotFoundError( |
| 627 | f"Not a file: {file}. Please provide a valid file path." |
| 628 | ) |
| 629 | |
| 630 | if not os.access(file, os.R_OK): |
| 631 | raise SDKFileNotFoundError( |
| 632 | f"File not readable: {file}. Please check the file permissions." |
| 633 | ) |
| 634 | |
| 635 | mimetype = mimetypes.guess(file=file) |
| 636 | s3meta = _request_presigned_upload( |
| 637 | client, |
| 638 | filename=file.name, |
| 639 | md5=get_md5(file=file), |
| 640 | mimetype=mimetype, |
| 641 | tool=tool, |
| 642 | toolkit=toolkit, |
| 643 | ) |
| 644 | upload(url=s3meta.new_presigned_url, file=file, mimetype=mimetype) |
| 645 | return cls(name=file.name, mimetype=mimetype, s3key=s3meta.key) |
| 646 | |
| 647 | |
| 648 | def _discard_partial_download(outfile: Path) -> None: |
| 649 | """Remove a half-written download so it is never mistaken for the file. |
| 650 | |
| 651 | Cleanup failures are swallowed on purpose: the error that triggered the |
| 652 | cleanup is what the caller needs to see, and an ``OSError`` raised from |
| 653 | here would replace it. |
| 654 | """ |
| 655 | with contextlib.suppress(OSError): |
| 656 | outfile.unlink(missing_ok=True) |
| 657 | |
| 658 | |
| 659 | class FileDownloadable(BaseModel): |
| 660 | model_config = ConfigDict(json_schema_extra={"file_downloadable": True}) |
| 661 | |
| 662 | name: str = Field(..., description="Name of the file") |
| 663 | mimetype: str = Field(..., description="Mime type of the file.") |
| 664 | s3url: str = Field(..., description="URL of the file.") |
| 665 | |
| 666 | def download( |
| 667 | self, |
| 668 | outdir: Path, |
| 669 | chunk_size: int = _DEFAULT_CHUNK_SIZE, |
| 670 | *, |
| 671 | root: Path, |
| 672 | max_size: int = _MAX_RESPONSE_SIZE, |
| 673 | ) -> Path: |
| 674 | """Fetch the file into ``outdir``. |
| 675 | |
| 676 | :param outdir: Directory to write into. May be derived from untrusted |
| 677 | input, provided it was produced by :func:`secure_join`. |
| 678 | :param root: Trusted containment anchor, configured locally and never |
| 679 | derived from an API response. Required rather than defaulted: |
| 680 | checking containment against a directory that untrusted input has |
| 681 | already relocated is not a check at all, and ``outdir`` may be |
| 682 | exactly such a directory. Callers must name the anchor explicitly. |
| 683 | :param max_size: Maximum number of bytes to write to disk. ``s3url`` is |
| 684 | an API-response field, so the body behind it is untrusted: the |
| 685 | streamed byte count is authoritative because ``Content-Length`` can |
| 686 | be absent or dishonest. |
| 687 | """ |
| 688 | # SEC-316: `self.name` also comes from the (potentially compromised or |
| 689 | # MITM'd) API response. Collapsed to a bare filename and checked against |
| 690 | # `root` — not `outdir` — so a name like `output_evil/foo` |
| 691 | # (sibling-prefix attack) is rejected too. |
| 692 | try: |
| 693 | outfile = secure_basename_join(outdir, self.name, root=root) |
| 694 | except UnsafePathComponentError as e: |
| 695 | raise ErrorDownloadingFile(str(e)) from e |
| 696 | # A presigned storage URL can legitimately answer with a redirect (an S3 |
| 697 | # region redirect, say), so it is followed, with every hop validated. |
| 698 | try: |
| 699 | response = safe_request( |
| 700 | "GET", |
| 701 | self.s3url, |
| 702 | stream=True, |
| 703 | timeout=(_CONNECT_TIMEOUT, _READ_TIMEOUT), |
| 704 | ) |
| 705 | except requests.exceptions.RequestException as e: |
| 706 | raise ErrorDownloadingFile( |
| 707 | "Error downloading file: " |
| 708 | f"{_sanitize_url_for_logging(self.s3url)}. Error: {type(e).__name__}" |
| 709 | ) from e |
| 710 | if response.status_code != 200: |
| 711 | response.close() |
| 712 | raise ErrorDownloadingFile( |
| 713 | f"Error downloading file: {_sanitize_url_for_logging(self.s3url)}" |
| 714 | ) |
| 715 | |
| 716 | # Early abort for a self-declared oversized body. The header is only a |
| 717 | # hint — `parse_content_length` returns None for anything untrustworthy |
| 718 | # and the streaming counter below is the authoritative limit. |
| 719 | content_length = parse_content_length(response.headers.get("Content-Length")) |
| 720 | if content_length is not None and content_length > max_size: |
| 721 | response.close() |
| 722 | raise ResponseTooLargeError( |
| 723 | f"File size ({content_length} bytes) exceeds maximum allowed " |
| 724 | f"size ({max_size} bytes)" |
| 725 | ) |
| 726 | |
| 727 | total_bytes = 0 |
| 728 | created = False |
| 729 | try: |
| 730 | # Only once the fetch is validated and connected, so a blocked URL |
| 731 | # leaves no directory behind — and inside the `try`, so a failure |
| 732 | # here still closes the response. |
| 733 | outdir.mkdir(exist_ok=True, parents=True) |
| 734 | # A literal server name can equal a digest-tagged name. Claim the |
| 735 | # path exclusively, then choose a numbered name if it exists. |
| 736 | outfile, fd = open_unique_file(outfile) |
| 737 | created = True |
| 738 | with fd: |
| 739 | for chunk in response.iter_content(chunk_size=chunk_size): |
| 740 | if chunk: |
| 741 | total_bytes += len(chunk) |
| 742 | if total_bytes > max_size: |
| 743 | raise ResponseTooLargeError( |
| 744 | "Response size exceeds maximum allowed size " |
| 745 | f"({max_size} bytes)" |
| 746 | ) |
| 747 | fd.write(chunk) |
| 748 | except ResponseTooLargeError: |
| 749 | # Propagates uncaught — callers must see the limit hit — but the |
| 750 | # truncated file must not be left behind as if it were the download. |
| 751 | if created: |
| 752 | _discard_partial_download(outfile) |
| 753 | raise |
| 754 | except OSError as e: |
| 755 | # `requests.exceptions.RequestException` subclasses `OSError`, so a |
| 756 | # mid-stream transport failure and a failing `fd.write`/`mkdir` |
| 757 | # (disk full, permissions) both land here — and both owe the caller |
| 758 | # the `ErrorDownloadingFile` this method documents. |
| 759 | if created: |
| 760 | _discard_partial_download(outfile) |
| 761 | raise ErrorDownloadingFile( |
| 762 | "Error downloading file: " |
| 763 | f"{_sanitize_url_for_logging(self.s3url)}. Error: {type(e).__name__}" |
| 764 | ) from e |
| 765 | finally: |
| 766 | response.close() |
| 767 | return outfile |
| 768 | |
| 769 | |
| 770 | # Internal alias — ``FileHelper`` receives the already-adapted context-form |
| 771 | # callable produced by :func:`merge_before_file_upload`. |
| 772 | from composio.core.models._modifiers import ( # noqa: E402 |
| 773 | BeforeFileUploadContextCallable as BeforeFileUpload, |
| 774 | ) |
| 775 | |
| 776 | |
| 777 | class FileHelper(WithLogger): |
| 778 | def __init__( |
| 779 | self, |
| 780 | client: HttpClient, |
| 781 | outdir: t.Optional[str] = None, |
| 782 | *, |
| 783 | sensitive_file_upload_protection: bool = True, |
| 784 | file_upload_path_deny_segments: t.Optional[t.Sequence[str]] = None, |
| 785 | file_upload_allowlist: t.Optional[t.Sequence[Path]] = None, |
| 786 | ) -> None: |
| 787 | """ |
| 788 | :param outdir: Where files downloaded during tool execution are written. |
| 789 | Defaults to ``~/.composio/files``. |
| 790 | :param file_upload_allowlist: Directories from which local files may be |
| 791 | auto-uploaded. ``None`` disables the allowlist check (used when |
| 792 | auto-upload is off, or for manual upload APIs). An empty list means |
| 793 | "no paths allowed" (fail-closed). |
| 794 | """ |
| 795 | super().__init__() |
| 796 | self._client = client |
| 797 | # Falsy, not just `is None`: an empty string falls through to the |
| 798 | # same default directory below, and must be treated as such here too |
| 799 | # or `ensure_cache_directory()` silently stops firing for it. |
| 800 | self._outdir_is_default = not outdir |
| 801 | self._outdir = Path(outdir) if outdir else get_output_file_directory() |
| 802 | self._sensitive_file_upload_protection = sensitive_file_upload_protection |
| 803 | self._file_upload_path_deny_segments = file_upload_path_deny_segments |
| 804 | self._file_upload_allowlist: t.Optional[t.Sequence[Path]] = ( |
| 805 | list(file_upload_allowlist) if file_upload_allowlist is not None else None |
| 806 | ) |
| 807 | |
| 808 | def _has_file_property( |
| 809 | self, schema: t.Dict, property_name: str = "file_uploadable" |
| 810 | ) -> bool: |
| 811 | """Check if a schema (or any of its variants) contains a file property. |
| 812 | |
| 813 | Recursively checks anyOf, oneOf, allOf, nested properties, and array items. |
| 814 | """ |
| 815 | if not isinstance(schema, dict): |
| 816 | return False |
| 817 | |
| 818 | # Direct property check |
| 819 | if schema.get(property_name, False): |
| 820 | return True |
| 821 | |
| 822 | # The default execution path uses this walker as a cheap gate before |
| 823 | # dereferencing. Inspect both modern and legacy definition containers |
| 824 | # so a file flag reachable only through ``$ref`` still passes the gate. |
| 825 | for definitions_key in ("$defs", "definitions"): |
| 826 | definitions = schema.get(definitions_key) |
| 827 | if not isinstance(definitions, dict): |
| 828 | continue |
| 829 | for definition in definitions.values(): |
| 830 | if self._has_file_property(definition, property_name): |
| 831 | return True |
| 832 | |
| 833 | # Check anyOf variants |
| 834 | if "anyOf" in schema: |
| 835 | for variant in schema["anyOf"]: |
| 836 | if self._has_file_property(variant, property_name): |
| 837 | return True |
| 838 | |
| 839 | # Check oneOf variants |
| 840 | if "oneOf" in schema: |
| 841 | for variant in schema["oneOf"]: |
| 842 | if self._has_file_property(variant, property_name): |
| 843 | return True |
| 844 | |
| 845 | # Check allOf variants |
| 846 | if "allOf" in schema: |
| 847 | for variant in schema["allOf"]: |
| 848 | if self._has_file_property(variant, property_name): |
| 849 | return True |
| 850 | |
| 851 | # Check nested properties |
| 852 | if "properties" in schema: |
| 853 | for prop in schema["properties"].values(): |
| 854 | if self._has_file_property(prop, property_name): |
| 855 | return True |
| 856 | |
| 857 | # Check array items |
| 858 | if "items" in schema: |
| 859 | items = schema["items"] |
| 860 | if isinstance(items, list): |
| 861 | for item in items: |
| 862 | if self._has_file_property(item, property_name): |
| 863 | return True |
| 864 | elif isinstance(items, dict): |
| 865 | if self._has_file_property(items, property_name): |
| 866 | return True |
| 867 | |
| 868 | return False |
| 869 | |
| 870 | def _file_uploadable(self, schema: t.Dict) -> bool: |
| 871 | """Check if a schema has file_uploadable property.""" |
| 872 | return self._has_file_property(schema, "file_uploadable") |
| 873 | |
| 874 | def _process_file_uploadable(self, schema: t.Dict) -> t.Dict: |
| 875 | return { |
| 876 | "type": "string", |
| 877 | "format": "path", |
| 878 | "description": schema.get("description", "Path to file."), |
| 879 | "title": schema.get("title"), |
| 880 | "file_uploadable": True, |
| 881 | } |
| 882 | |
| 883 | def _transform_schema_for_file_upload(self, schema: t.Dict) -> t.Dict: |
| 884 | """Recursively transform a schema, converting file_uploadable fields to path format. |
| 885 | |
| 886 | Handles anyOf, oneOf, allOf, nested properties, and array items. |
| 887 | """ |
| 888 | if not isinstance(schema, dict): |
| 889 | return schema |
| 890 | |
| 891 | # Direct file_uploadable - transform it |
| 892 | if schema.get("file_uploadable", False): |
| 893 | return self._process_file_uploadable(schema) |
| 894 | |
| 895 | # Create a copy to avoid mutating the original |
| 896 | new_schema = dict(schema) |
| 897 | |
| 898 | # Transform anyOf variants |
| 899 | if "anyOf" in schema: |
| 900 | new_schema["anyOf"] = [ |
| 901 | self._transform_schema_for_file_upload(variant) |
| 902 | for variant in schema["anyOf"] |
| 903 | ] |
| 904 | |
| 905 | # Transform oneOf variants |
| 906 | if "oneOf" in schema: |
| 907 | new_schema["oneOf"] = [ |
| 908 | self._transform_schema_for_file_upload(variant) |
| 909 | for variant in schema["oneOf"] |
| 910 | ] |
| 911 | |
| 912 | # Transform allOf variants |
| 913 | if "allOf" in schema: |
| 914 | new_schema["allOf"] = [ |
| 915 | self._transform_schema_for_file_upload(variant) |
| 916 | for variant in schema["allOf"] |
| 917 | ] |
| 918 | |
| 919 | # Transform nested properties |
| 920 | if "properties" in schema: |
| 921 | new_schema["properties"] = { |
| 922 | key: self._transform_schema_for_file_upload(prop) |
| 923 | for key, prop in schema["properties"].items() |
| 924 | } |
| 925 | |
| 926 | # Transform array items |
| 927 | if "items" in schema: |
| 928 | items = schema["items"] |
| 929 | if isinstance(items, list): |
| 930 | new_schema["items"] = [ |
| 931 | self._transform_schema_for_file_upload(item) for item in items |
| 932 | ] |
| 933 | elif isinstance(items, dict): |
| 934 | new_schema["items"] = self._transform_schema_for_file_upload(items) |
| 935 | |
| 936 | return new_schema |
| 937 | |
| 938 | def enhance_schema_descriptions(self, schema: t.Dict) -> t.Dict: |
| 939 | """Add type hints and required notes to parameter descriptions. |
| 940 | |
| 941 | This method enhances parameter descriptions by adding: |
| 942 | - Type hints ("Please provide a value of type...") |
| 943 | - Required notes ("This parameter is required.") |
| 944 | |
| 945 | This is separate from file processing and should always run |
| 946 | regardless of `dangerously_allow_auto_upload_download_files`. |
| 947 | |
| 948 | Schemas with no `properties` key (e.g. `{}` for tools that declare |
| 949 | no input parameters — common with MCP-backed toolkits) are returned |
| 950 | unchanged, matching the sibling `process_file_uploadable_schema`. |
| 951 | """ |
| 952 | if "properties" not in schema: |
| 953 | return schema |
| 954 | required = schema.get("required") or [] |
| 955 | for _param, _schema in schema["properties"].items(): |
| 956 | if not isinstance(_schema, dict): |
| 957 | # Boolean schemas have no description to enhance. They are |
| 958 | # valid property schemas and reach this unconditional helper |
| 959 | # before schema conversion. |
| 960 | continue |
| 961 | if _schema.get("type") in ["string", "integer", "number", "boolean"]: |
| 962 | ext = f"Please provide a value of type {_schema['type']}." |
| 963 | description = _schema.get("description", "").rstrip(".") |
| 964 | _schema["description"] = f"{description}. {ext}" if description else ext |
| 965 | |
| 966 | if _param in required: |
| 967 | description = _schema.get("description") |
| 968 | _schema["description"] = ( |
| 969 | (f"{description.rstrip('.')}. This parameter is required.") |
| 970 | if description |
| 971 | else "This parameter is required." |
| 972 | ) |
| 973 | return schema |
| 974 | |
| 975 | def process_file_uploadable_schema(self, schema: t.Dict) -> t.Dict: |
| 976 | """Process file_uploadable fields in schema. |
| 977 | |
| 978 | This method converts file_uploadable fields to path format. |
| 979 | Should only be called when the caller opted in via |
| 980 | `dangerously_allow_auto_upload_download_files=True`. |
| 981 | Recursively handles anyOf, oneOf, allOf, nested properties, and array items. |
| 982 | |
| 983 | ``$ref``/``$defs`` indirection is inlined first so file_uploadable flags |
| 984 | reachable only through a reference are visible to the transform below |
| 985 | (https://github.com/ComposioHQ/composio/issues/3506). Tool schemas come |
| 986 | from the API, which may emit a ``$ref`` without a ``$defs`` block |
| 987 | (https://github.com/ComposioHQ/composio/issues/3307); ``"sentinel"`` |
| 988 | degrades that gracefully instead of raising. |
| 989 | """ |
| 990 | resolved = dereference_json_schema(schema, on_unresolved="sentinel") |
| 991 | if "properties" not in resolved: |
| 992 | return schema |
| 993 | |
| 994 | # Preserve the original object's identity (callers such as |
| 995 | # ``process_schema_recursively`` mutate in place) while adopting the |
| 996 | # dereferenced, ``$defs``-stripped shape. |
| 997 | schema.clear() |
| 998 | schema.update(resolved) |
| 999 | schema["properties"] = { |
| 1000 | key: self._transform_schema_for_file_upload(prop) |
| 1001 | for key, prop in schema["properties"].items() |
| 1002 | } |
| 1003 | return schema |
| 1004 | |
| 1005 | def process_schema_recursively(self, schema: t.Dict) -> t.Dict: |
| 1006 | """Process schema for both file handling and description enhancements. |
| 1007 | |
| 1008 | This method is kept for backward compatibility. It calls both |
| 1009 | process_file_uploadable_schema and enhance_schema_descriptions. |
| 1010 | """ |
| 1011 | self.process_file_uploadable_schema(schema) |
| 1012 | self.enhance_schema_descriptions(schema) |
| 1013 | return schema |
| 1014 | |
| 1015 | def _schema_variants(self, schema: t.Dict) -> t.List[t.Dict]: |
| 1016 | """Return composed schema variants in the SDK's historical order.""" |
| 1017 | variants: t.List[t.Dict] = [] |
| 1018 | for key in ("anyOf", "oneOf", "allOf"): |
| 1019 | schema_variants = schema.get(key) |
| 1020 | if not isinstance(schema_variants, list): |
| 1021 | continue |
| 1022 | variants.extend(v for v in schema_variants if isinstance(v, dict)) |
| 1023 | return variants |
| 1024 | |
| 1025 | def _json_schema_type_matches_value(self, schema: t.Dict, value: t.Any) -> bool: |
| 1026 | """Best-effort runtime shape match for selecting composed schemas.""" |
| 1027 | schema_type = schema.get("type") |
| 1028 | if isinstance(schema_type, list): |
| 1029 | return any( |
| 1030 | self._json_schema_type_matches_value({**schema, "type": tpe}, value) |
| 1031 | for tpe in schema_type |
| 1032 | ) |
| 1033 | |
| 1034 | if schema_type is None: |
| 1035 | if "properties" in schema: |
| 1036 | return isinstance(value, dict) |
| 1037 | if "items" in schema: |
| 1038 | return isinstance(value, list) |
| 1039 | return False |
| 1040 | |
| 1041 | if schema_type == "array": |
| 1042 | return isinstance(value, list) |
| 1043 | if schema_type == "object": |
| 1044 | return isinstance(value, dict) |
| 1045 | if schema_type == "string": |
| 1046 | return isinstance(value, str) |
| 1047 | if schema_type == "integer": |
| 1048 | return isinstance(value, int) and not isinstance(value, bool) |
| 1049 | if schema_type == "number": |
| 1050 | return isinstance(value, (int, float)) and not isinstance(value, bool) |
| 1051 | if schema_type == "boolean": |
| 1052 | return isinstance(value, bool) |
| 1053 | if schema_type == "null": |
| 1054 | return value is None |
| 1055 | |
| 1056 | return False |
| 1057 | |
| 1058 | @staticmethod |
| 1059 | def _is_array_shaped_schema(schema: t.Dict) -> bool: |
| 1060 | """Return whether JSON Schema declares or infers an array shape.""" |
| 1061 | schema_type = schema.get("type") |
| 1062 | if isinstance(schema_type, list): |
| 1063 | return "array" in schema_type |
| 1064 | return schema_type == "array" or (schema_type is None and "items" in schema) |
| 1065 | |
| 1066 | def _find_schema_variant_with_file_property( |
| 1067 | self, |
| 1068 | schema: t.Dict, |
| 1069 | property_name: str, |
| 1070 | value: t.Any = None, |
| 1071 | ) -> t.Optional[t.Dict]: |
| 1072 | """Find the composed schema variant that should process a file value. |
| 1073 | |
| 1074 | ``anyOf``, ``oneOf`` and ``allOf`` are intentionally treated the same |
| 1075 | here, matching the SDK's existing convention. When a runtime value is |
| 1076 | available, prefer the file-bearing variant with the matching JSON Schema |
| 1077 | shape; otherwise keep the historical first-match behavior. |
| 1078 | """ |
| 1079 | candidates = [ |
| 1080 | variant |
| 1081 | for variant in self._schema_variants(schema) |
| 1082 | if self._has_file_property(variant, property_name) |
| 1083 | ] |
| 1084 | if not candidates: |
| 1085 | return None |
| 1086 | |
| 1087 | if value is not None: |
| 1088 | for candidate in candidates: |
| 1089 | if self._json_schema_type_matches_value(candidate, value): |
| 1090 | return candidate |
| 1091 | |
| 1092 | return candidates[0] |
| 1093 | |
| 1094 | def _find_uploadable_schema_variant( |
| 1095 | self, schema: t.Dict, value: t.Any = None |
| 1096 | ) -> t.Optional[t.Dict]: |
| 1097 | """Find a schema variant that contains file_uploadable properties.""" |
| 1098 | return self._find_schema_variant_with_file_property( |
| 1099 | schema=schema, |
| 1100 | property_name="file_uploadable", |
| 1101 | value=value, |
| 1102 | ) |
| 1103 | |
| 1104 | @staticmethod |
| 1105 | def _drop_empty_file_value(value: t.Any) -> t.Any: |
| 1106 | """Omit an empty string from a file input when auto-upload is disabled. |
| 1107 | |
| 1108 | An explicit ``None`` remains part of the payload so nullable file fields, |
| 1109 | including required nullable fields, keep their JSON Schema semantics. |
| 1110 | """ |
| 1111 | return _DELETE_VALUE if value == "" else value |
| 1112 | |
| 1113 | def _upload_file_value( |
| 1114 | self, |
| 1115 | value: t.Any, |
| 1116 | tool: Tool, |
| 1117 | before_file_upload: t.Optional[BeforeFileUpload], |
| 1118 | ) -> t.Any: |
| 1119 | if value is None or self._drop_empty_file_value(value) is _DELETE_VALUE: |
| 1120 | return _DELETE_VALUE |
| 1121 | |
| 1122 | return FileUploadable.from_path( |
| 1123 | client=self._client, |
| 1124 | file=value, |
| 1125 | tool=tool.slug, |
| 1126 | toolkit=tool.toolkit.slug, |
| 1127 | sensitive_file_upload_protection=self._sensitive_file_upload_protection, |
| 1128 | file_upload_path_deny_segments=self._file_upload_path_deny_segments, |
| 1129 | file_upload_allowlist=self._file_upload_allowlist, |
| 1130 | before_file_upload=before_file_upload, |
| 1131 | ).model_dump() |
| 1132 | |
| 1133 | def _substitute_file_upload_value( |
| 1134 | self, |
| 1135 | value: t.Any, |
| 1136 | schema: t.Optional[t.Dict], |
| 1137 | leaf: t.Callable[[t.Any], t.Any], |
| 1138 | ) -> t.Any: |
| 1139 | """Return ``value`` with every ``file_uploadable`` leaf passed through ``leaf``. |
| 1140 | |
| 1141 | A leaf may return ``_DELETE_VALUE`` to omit the key/item from its |
| 1142 | parent container. |
| 1143 | """ |
| 1144 | if not isinstance(schema, dict): |
| 1145 | return value |
| 1146 | |
| 1147 | if schema.get("file_uploadable", False): |
| 1148 | return leaf(value) |
| 1149 | |
| 1150 | # Array-only file inputs historically omit null in upload mode even |
| 1151 | # though null cannot be traversed as an array. Keep that behavior |
| 1152 | # without treating an optional object that merely contains a nested |
| 1153 | # file property as a file leaf itself. |
| 1154 | if ( |
| 1155 | value is None |
| 1156 | and self._is_array_shaped_schema(schema) |
| 1157 | and self._file_uploadable(schema) |
| 1158 | ): |
| 1159 | return leaf(value) |
| 1160 | |
| 1161 | uploadable_variant = self._find_uploadable_schema_variant( |
| 1162 | schema=schema, |
| 1163 | value=value, |
| 1164 | ) |
| 1165 | if uploadable_variant is not None: |
| 1166 | # An empty string cannot match an array-only file input, but it may |
| 1167 | # match another non-file string variant in the same composition. |
| 1168 | if value == "" and self._is_array_shaped_schema(uploadable_variant): |
| 1169 | matching_non_file_variant = any( |
| 1170 | not self._file_uploadable(variant) |
| 1171 | and self._json_schema_type_matches_value(variant, value) |
| 1172 | for variant in self._schema_variants(schema) |
| 1173 | ) |
| 1174 | if matching_non_file_variant: |
| 1175 | return value |
| 1176 | return leaf(value) |
| 1177 | return self._substitute_file_upload_value( |
| 1178 | value=value, |
| 1179 | schema=uploadable_variant, |
| 1180 | leaf=leaf, |
| 1181 | ) |
| 1182 | |
| 1183 | if ( |
| 1184 | value == "" |
| 1185 | and self._is_array_shaped_schema(schema) |
| 1186 | and self._file_uploadable(schema) |
| 1187 | ): |
| 1188 | return leaf(value) |
| 1189 | |
| 1190 | if isinstance(value, dict) and "properties" in schema: |
| 1191 | processed: t.Dict[str, t.Any] = {} |
| 1192 | properties = schema["properties"] |
| 1193 | for key, item in value.items(): |
| 1194 | item_schema = properties.get(key) |
| 1195 | processed_item = self._substitute_file_upload_value( |
| 1196 | value=item, |
| 1197 | schema=item_schema, |
| 1198 | leaf=leaf, |
| 1199 | ) |
| 1200 | if processed_item is not _DELETE_VALUE: |
| 1201 | processed[key] = processed_item |
| 1202 | return processed |
| 1203 | |
| 1204 | if isinstance(value, list) and "items" in schema: |
| 1205 | items_schema = schema["items"] |
| 1206 | if isinstance(items_schema, list): |
| 1207 | items_schema = items_schema[0] if items_schema else None |
| 1208 | |
| 1209 | processed_items: t.List[t.Any] = [] |
| 1210 | for item in value: |
| 1211 | processed_item = self._substitute_file_upload_value( |
| 1212 | value=item, |
| 1213 | schema=items_schema, |
| 1214 | leaf=leaf, |
| 1215 | ) |
| 1216 | if processed_item is not _DELETE_VALUE: |
| 1217 | processed_items.append(processed_item) |
| 1218 | return processed_items |
| 1219 | |
| 1220 | return value |
| 1221 | |
| 1222 | def _substitute_file_uploads_recursively( |
| 1223 | self, |
| 1224 | tool: Tool, |
| 1225 | schema: t.Dict, |
| 1226 | request: t.Dict, |
| 1227 | *, |
| 1228 | before_file_upload: t.Optional[BeforeFileUpload] = None, |
| 1229 | leaf: t.Optional[t.Callable[[t.Any], t.Any]] = None, |
| 1230 | ) -> t.Dict: |
| 1231 | if leaf is None: |
| 1232 | leaf = functools.partial( |
| 1233 | self._upload_file_value, |
| 1234 | tool=tool, |
| 1235 | before_file_upload=before_file_upload, |
| 1236 | ) |
| 1237 | |
| 1238 | processed = self._substitute_file_upload_value( |
| 1239 | value=request, |
| 1240 | schema=schema, |
| 1241 | leaf=leaf, |
| 1242 | ) |
| 1243 | if processed is request: |
| 1244 | return request |
| 1245 | if isinstance(processed, dict): |
| 1246 | request.clear() |
| 1247 | request.update(processed) |
| 1248 | return request |
| 1249 | assert isinstance(processed, dict), ( |
| 1250 | "expected dict from _substitute_file_upload_value at the root; " |
| 1251 | f"got {type(processed).__name__}" |
| 1252 | ) |
| 1253 | return processed |
| 1254 | |
| 1255 | def substitute_file_uploads( |
| 1256 | self, |
| 1257 | tool: Tool, |
| 1258 | request: t.Dict, |
| 1259 | *, |
| 1260 | before_file_upload: t.Optional[BeforeFileUpload] = None, |
| 1261 | ) -> t.Dict: |
| 1262 | """Stage file-uploadable leaves in ``request`` and return it. |
| 1263 | |
| 1264 | Mutation contract: the top-level ``request`` dict is mutated in place |
| 1265 | and its identity is preserved (the return value is the same object). |
| 1266 | Nested dicts inside ``request`` may be replaced with fresh dicts |
| 1267 | rather than mutated, so callers should not retain references to |
| 1268 | nested values across this call. |
| 1269 | """ |
| 1270 | return self._substitute_file_uploads_recursively( |
| 1271 | tool=tool, |
| 1272 | # Inline $ref/$defs once at the boundary so the walker sees |
| 1273 | # file_uploadable flags hidden behind a reference. Non-mutating: |
| 1274 | # ``tool.input_parameters`` is left untouched. |
| 1275 | schema=dereference_json_schema( |
| 1276 | tool.input_parameters, on_unresolved="sentinel" |
| 1277 | ), |
| 1278 | request=request, |
| 1279 | before_file_upload=before_file_upload, |
| 1280 | ) |
| 1281 | |
| 1282 | def drop_empty_file_uploads(self, tool: Tool, request: t.Dict) -> t.Dict: |
| 1283 | """Omit ``""`` at ``file_uploadable`` leaves without uploading. |
| 1284 | |
| 1285 | This is the half of :meth:`substitute_file_uploads` that needs no |
| 1286 | filesystem access, so it runs even when automatic upload is disabled: |
| 1287 | an empty file value is never a valid staged descriptor and the backend |
| 1288 | would reject it (issue #4233). Explicit ``None`` and every non-empty |
| 1289 | value are forwarded untouched. File-bearing request containers are |
| 1290 | rebuilt; unlike :meth:`substitute_file_uploads`, this method never |
| 1291 | mutates caller-owned arguments or copies arbitrary leaf objects. |
| 1292 | """ |
| 1293 | # Every execution takes this path by default, so leave requests for |
| 1294 | # tools without a file input completely alone. Check the raw schema |
| 1295 | # first to avoid paying the dereference cost for non-file tools. |
| 1296 | if not self._file_uploadable(tool.input_parameters): |
| 1297 | return request |
| 1298 | |
| 1299 | schema = dereference_json_schema( |
| 1300 | tool.input_parameters, on_unresolved="sentinel" |
| 1301 | ) |
| 1302 | processed = self._substitute_file_upload_value( |
| 1303 | value=request, |
| 1304 | schema=schema, |
| 1305 | leaf=self._drop_empty_file_value, |
| 1306 | ) |
| 1307 | assert isinstance(processed, dict), ( |
| 1308 | "expected dict from _substitute_file_upload_value at the root; " |
| 1309 | f"got {type(processed).__name__}" |
| 1310 | ) |
| 1311 | return processed |
| 1312 | |
| 1313 | def _is_file_downloadable(self, schema: t.Dict) -> bool: |
| 1314 | """Check if a schema has file_downloadable property.""" |
| 1315 | return self._has_file_property(schema, "file_downloadable") |
| 1316 | |
| 1317 | def _find_downloadable_schema_variant( |
| 1318 | self, schema: t.Dict, value: t.Any = None |
| 1319 | ) -> t.Optional[t.Dict]: |
| 1320 | """Find a schema variant that contains file_downloadable properties.""" |
| 1321 | return self._find_schema_variant_with_file_property( |
| 1322 | schema=schema, |
| 1323 | property_name="file_downloadable", |
| 1324 | value=value, |
| 1325 | ) |
| 1326 | |
| 1327 | def _download_file_value(self, value: t.Any, tool: Tool) -> t.Any: |
| 1328 | if isinstance(value, dict) and "s3url" in value: |
| 1329 | if self._outdir_is_default: |
| 1330 | # First point at which the cache directory is genuinely |
| 1331 | # needed. Raises the same RuntimeError that used to be raised |
| 1332 | # at import time, pointing at COMPOSIO_CACHE_DIR. |
| 1333 | ensure_cache_directory() |
| 1334 | # `tool.toolkit.slug` and `tool.slug` come from the API response and |
| 1335 | # are untrusted, so joining them directly would let the response pick |
| 1336 | # the download directory. `secure_join` validates each component and |
| 1337 | # anchors containment on `self._outdir`, which is configured locally |
| 1338 | # and never derived from the response. |
| 1339 | return str( |
| 1340 | FileDownloadable(**value).download( |
| 1341 | outdir=secure_join(self._outdir, tool.toolkit.slug, tool.slug), |
| 1342 | root=self._outdir, |
| 1343 | ) |
| 1344 | ) |
| 1345 | return value |
| 1346 | |
| 1347 | def _substitute_file_download_value( |
| 1348 | self, |
| 1349 | value: t.Any, |
| 1350 | schema: t.Optional[t.Dict], |
| 1351 | tool: Tool, |
| 1352 | ) -> t.Any: |
| 1353 | """Return ``value`` with file-downloadable leaves saved locally.""" |
| 1354 | if not isinstance(schema, dict): |
| 1355 | return value |
| 1356 | |
| 1357 | if schema.get("file_downloadable", False): |
| 1358 | return self._download_file_value(value=value, tool=tool) |
| 1359 | |
| 1360 | downloadable_variant = self._find_downloadable_schema_variant( |
| 1361 | schema=schema, |
| 1362 | value=value, |
| 1363 | ) |
| 1364 | if downloadable_variant is not None: |
| 1365 | return self._substitute_file_download_value( |
| 1366 | value=value, |
| 1367 | schema=downloadable_variant, |
| 1368 | tool=tool, |
| 1369 | ) |
| 1370 | |
| 1371 | if isinstance(value, dict) and "properties" in schema: |
| 1372 | properties = schema["properties"] |
| 1373 | return { |
| 1374 | key: self._substitute_file_download_value( |
| 1375 | value=item, |
| 1376 | schema=properties.get(key), |
| 1377 | tool=tool, |
| 1378 | ) |
| 1379 | for key, item in value.items() |
| 1380 | } |
| 1381 | |
| 1382 | if isinstance(value, list) and "items" in schema: |
| 1383 | items_schema = schema["items"] |
| 1384 | if isinstance(items_schema, list): |
| 1385 | items_schema = items_schema[0] if items_schema else None |
| 1386 | return [ |
| 1387 | self._substitute_file_download_value( |
| 1388 | value=item, |
| 1389 | schema=items_schema, |
| 1390 | tool=tool, |
| 1391 | ) |
| 1392 | for item in value |
| 1393 | ] |
| 1394 | |
| 1395 | return value |
| 1396 | |
| 1397 | def _substitute_file_downloads_recursively( |
| 1398 | self, |
| 1399 | tool: Tool, |
| 1400 | schema: t.Dict, |
| 1401 | request: t.Dict, |
| 1402 | ) -> t.Dict: |
| 1403 | processed = self._substitute_file_download_value( |
| 1404 | value=request, |
| 1405 | schema=schema, |
| 1406 | tool=tool, |
| 1407 | ) |
| 1408 | if processed is request: |
| 1409 | return request |
| 1410 | if isinstance(processed, dict): |
| 1411 | request.clear() |
| 1412 | request.update(processed) |
| 1413 | return request |
| 1414 | assert isinstance(processed, dict), ( |
| 1415 | "expected dict from _substitute_file_download_value at the root; " |
| 1416 | f"got {type(processed).__name__}" |
| 1417 | ) |
| 1418 | return processed |
| 1419 | |
| 1420 | def substitute_file_downloads( |
| 1421 | self, |
| 1422 | tool: Tool, |
| 1423 | response: ToolExecutionResponse, |
| 1424 | ) -> ToolExecutionResponse: |
| 1425 | """Materialize file-downloadable leaves in ``response`` and return it. |
| 1426 | |
| 1427 | Mutation contract: the top-level ``response`` dict is mutated in |
| 1428 | place and its identity is preserved (the return value is the same |
| 1429 | object). Nested dicts inside ``response`` may be replaced with |
| 1430 | fresh dicts rather than mutated, so callers should not retain |
| 1431 | references to nested values across this call. |
| 1432 | """ |
| 1433 | return t.cast( |
| 1434 | "ToolExecutionResponse", |
| 1435 | self._substitute_file_downloads_recursively( |
| 1436 | tool=tool, |
| 1437 | # Inline $ref/$defs once at the boundary so the walker sees |
| 1438 | # file_downloadable flags hidden behind a reference (e.g. |
| 1439 | # GMAIL_GET_ATTACHMENT). Non-mutating: ``tool.output_parameters`` |
| 1440 | # is left untouched. |
| 1441 | schema=dereference_json_schema( |
| 1442 | tool.output_parameters, on_unresolved="sentinel" |
| 1443 | ), |
| 1444 | request=t.cast(dict, response), |
| 1445 | ), |
| 1446 | ) |