@devries-cameron20 / composio Mirror · read-only / commits / 03429c740
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 16:00 UTC

fix(python): create the cache directory on first use instead of at import time (#4162)

Alberto Schiabel committed Aug 18, 2026 at 15:45 UTC 03429c74044b09c2c71fdcfa3bc59579744a7219
4 files changed +344 -33
python/composio/core/models/_files.py
+84 -25
@@ -82,35 +82,85 @@ ENV_LOCAL_CACHE_DIRECTORY = "COMPOSIO_CACHE_DIR"
82 Environment to set the composio caching directory.
83 """
84
85 -LOCAL_CACHE_DIRECTORY = Path(
86 - os.environ.get(
87 - ENV_LOCAL_CACHE_DIRECTORY,
88 - Path.home() / LOCAL_CACHE_DIRECTORY_NAME, # Fallback to user directory
89 - )
90 -)
85 +LOCAL_OUTPUT_FILE_DIRECTORY_NAME = "files"
86 """
92 -Path to local caching directory.
87 +Name of the cache sub-directory into which files downloaded during tool
88 +execution are written. Previously ``outputs``; now ``files`` for parity with
89 +the TypeScript SDK.
90 """
91
95 -try:
96 - LOCAL_CACHE_DIRECTORY.mkdir(parents=True, exist_ok=True)
97 - if not os.access(LOCAL_CACHE_DIRECTORY, os.W_OK):
98 - raise OSError
99 -except OSError as e:
100 - raise RuntimeError(
101 - f"Cache directory {LOCAL_CACHE_DIRECTORY} is not writable please "
102 - f"provide a path that is writable using {ENV_LOCAL_CACHE_DIRECTORY} "
103 - "environment variable."
104 - ) from e
92
93 +def get_cache_directory() -> Path:
94 + """Resolve the local caching directory without touching the filesystem.
95
107 -LOCAL_OUTPUT_FILE_DIRECTORY = LOCAL_CACHE_DIRECTORY / "files"
108 -"""
109 -Default local directory into which files downloaded during tool execution are
110 -written. Previously ``<cache>/outputs``; now ``<cache>/files`` for parity with
111 -the TypeScript SDK. Override by passing ``file_download_dir=...`` to Composio,
112 -or by setting ``outdir`` on ``FileHelper`` directly.
113 -"""
96 + ``COMPOSIO_CACHE_DIR`` is read on every call, so it can be set after
97 + ``composio`` has already been imported. ``Path.home()`` is only consulted
98 + when the variable is unset: it can raise ``RuntimeError`` when there is no
99 + resolvable home directory, which is exactly the situation
100 + ``COMPOSIO_CACHE_DIR`` exists to work around, so it must not be evaluated
101 + eagerly as a fallback argument.
102 + """
103 + configured = os.environ.get(ENV_LOCAL_CACHE_DIRECTORY)
104 + if configured:
105 + return Path(configured)
106 +
107 + try:
108 + home = Path.home()
109 + except RuntimeError as e:
110 + raise RuntimeError(
111 + "Could not determine a home directory to store the Composio cache "
112 + f"in. Provide a writable path using the {ENV_LOCAL_CACHE_DIRECTORY} "
113 + "environment variable."
114 + ) from e
115 + return home / LOCAL_CACHE_DIRECTORY_NAME
116 +
117 +
118 +def get_output_file_directory() -> Path:
119 + """Default local directory into which files downloaded during tool
120 + execution are written. Override by passing ``file_download_dir=...`` to
121 + Composio, or by setting ``outdir`` on ``FileHelper`` directly.
122 + """
123 + return get_cache_directory() / LOCAL_OUTPUT_FILE_DIRECTORY_NAME
124 +
125 +
126 +def ensure_cache_directory() -> Path:
127 + """Create the cache directory on first use and check that it is writable.
128 +
129 + This used to run at module import time, so a bare ``import composio``
130 + raised ``RuntimeError`` on any read-only filesystem -- AWS Lambda,
131 + distroless containers, ``ProtectHome=true`` systemd units -- even for
132 + programs that never touched a file. Deferring it to first use keeps the
133 + same check, and the same error message, for the callers that actually
134 + need the directory.
135 + """
136 + directory = get_cache_directory()
137 + try:
138 + directory.mkdir(parents=True, exist_ok=True)
139 + if not os.access(directory, os.W_OK):
140 + raise OSError
141 + except OSError as e:
142 + raise RuntimeError(
143 + f"Cache directory {directory} is not writable please "
144 + f"provide a path that is writable using {ENV_LOCAL_CACHE_DIRECTORY} "
145 + "environment variable."
146 + ) from e
147 + return directory
148 +
149 +
150 +def __getattr__(name: str) -> Path:
151 + """Keep the historical module-level path constants working, but lazily.
152 +
153 + ``LOCAL_CACHE_DIRECTORY`` and ``LOCAL_OUTPUT_FILE_DIRECTORY`` used to be
154 + computed at import time. They are now resolved on attribute access
155 + instead (PEP 562), so importing this module no longer touches the
156 + filesystem or depends on the environment, and both constants observe a
157 + ``COMPOSIO_CACHE_DIR`` that was set after import.
158 + """
159 + if name == "LOCAL_CACHE_DIRECTORY":
160 + return get_cache_directory()
161 + if name == "LOCAL_OUTPUT_FILE_DIRECTORY":
162 + return get_output_file_directory()
163 + raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
164
165
166 def get_md5(file: Path) -> str:
@@ -670,7 +720,11 @@ class FileHelper(WithLogger):
720 """
721 super().__init__()
722 self._client = client
673 - self._outdir = Path(outdir) if outdir else LOCAL_OUTPUT_FILE_DIRECTORY
723 + # Falsy, not just `is None`: an empty string falls through to the
724 + # same default directory below, and must be treated as such here too
725 + # or `ensure_cache_directory()` silently stops firing for it.
726 + self._outdir_is_default = not outdir
727 + self._outdir = Path(outdir) if outdir else get_output_file_directory()
728 self._sensitive_file_upload_protection = sensitive_file_upload_protection
729 self._file_upload_path_deny_segments = file_upload_path_deny_segments
730 self._file_upload_allowlist: t.Optional[t.Sequence[Path]] = (
@@ -1103,6 +1157,11 @@ class FileHelper(WithLogger):
1157
1158 def _download_file_value(self, value: t.Any, tool: Tool) -> t.Any:
1159 if isinstance(value, dict) and "s3url" in value:
1160 + if self._outdir_is_default:
1161 + # First point at which the cache directory is genuinely
1162 + # needed. Raises the same RuntimeError that used to be raised
1163 + # at import time, pointing at COMPOSIO_CACHE_DIR.
1164 + ensure_cache_directory()
1165 # `tool.toolkit.slug` and `tool.slug` come from the API response and
1166 # are untrusted, so joining them directly would let the response pick
1167 # the download directory. `secure_join` validates each component and
python/tests/test_files.py
+9 -2
@@ -52,8 +52,15 @@ def mock_client():
52
53
54 @pytest.fixture
55 -def file_helper(mock_client):
56 - """Create a FileHelper instance with a mock client."""
55 +def file_helper(mock_client, monkeypatch, tmp_path):
56 + """Create a FileHelper instance with a mock client.
57 +
58 + Sandboxes ``COMPOSIO_CACHE_DIR`` so the default-outdir download path
59 + (``ensure_cache_directory()``) never touches the real home directory,
60 + even though ``FileDownloadable.download`` itself is mocked in most of
61 + these tests.
62 + """
63 + monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(tmp_path / ".composio"))
64 return FileHelper(client=mock_client)
65
66
python/tests/test_lazy_cache_directory.py new
+248
@@ -0,0 +1,248 @@
1 +"""Regression tests for lazy creation of the Composio cache directory.
2 +
3 +``composio/core/models/_files.py`` used to create ``~/.composio`` and assert
4 +that it was writable at *module import time*. Because ``composio/__init__.py``
5 +imports ``tools.py``, which imports ``_files.py`` unconditionally, a bare
6 +``import composio`` raised ``RuntimeError`` on any read-only filesystem --
7 +AWS Lambda, distroless containers, ``ProtectHome=true`` systemd units -- even
8 +for programs that never touched a file (#4150).
9 +
10 +These tests pin the fix: importing the package performs no filesystem work,
11 +``COMPOSIO_CACHE_DIR`` is honoured when set after import, and the writability
12 +check still raises the same error when the directory is genuinely needed.
13 +"""
14 +
15 +import os
16 +import subprocess
17 +import sys
18 +from pathlib import Path
19 +from unittest.mock import Mock
20 +
21 +import pytest
22 +
23 +from composio.core.models import _files
24 +from composio.core.models._files import FileHelper
25 +from composio.exceptions import BlockedInternalUrlError
26 +
27 +
28 +@pytest.fixture()
29 +def unusable_cache_dir(tmp_path: Path) -> Path:
30 + """A cache path that cannot be created, without relying on permissions.
31 +
32 + ``chmod``-based read-only directories are a no-op when the test suite
33 + runs as root (``os.access(..., W_OK)`` returns ``True`` for uid 0), which
34 + is common in containerized CI. Nesting the cache directory *underneath a
35 + regular file* makes ``mkdir`` fail with ``NotADirectoryError`` for every
36 + user.
37 + """
38 + blocker = tmp_path / "blocker"
39 + blocker.write_text("not a directory")
40 + return blocker / "cache"
41 +
42 +
43 +def test_importing_composio_does_not_create_the_cache_directory(
44 + unusable_cache_dir: Path,
45 +) -> None:
46 + """The regression test: ``import composio`` must not touch the filesystem.
47 +
48 + Run in a subprocess so the import genuinely happens under the patched
49 + environment rather than hitting the already-imported module in this
50 + process.
51 + """
52 + env = dict(os.environ)
53 + env["COMPOSIO_CACHE_DIR"] = str(unusable_cache_dir)
54 +
55 + result = subprocess.run(
56 + [sys.executable, "-c", "import composio"],
57 + env=env,
58 + capture_output=True,
59 + text=True,
60 + timeout=120,
61 + )
62 +
63 + assert result.returncode == 0, (
64 + "importing composio with an uncreatable cache directory must succeed, "
65 + f"but it failed with:\n{result.stderr}"
66 + )
67 + assert not unusable_cache_dir.exists()
68 +
69 +
70 +def test_cache_directory_is_created_only_on_demand(
71 + monkeypatch: pytest.MonkeyPatch, tmp_path: Path
72 +) -> None:
73 + cache_dir = tmp_path / "not-yet-there"
74 + monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(cache_dir))
75 +
76 + # Merely resolving the path must not create it.
77 + assert _files.get_cache_directory() == cache_dir
78 + assert not cache_dir.exists()
79 +
80 + assert _files.ensure_cache_directory() == cache_dir
81 + assert cache_dir.is_dir()
82 +
83 +
84 +def test_cache_dir_env_var_is_read_after_import(
85 + monkeypatch: pytest.MonkeyPatch, tmp_path: Path
86 +) -> None:
87 + """``COMPOSIO_CACHE_DIR`` used to be frozen at import time."""
88 + first = tmp_path / "first"
89 + second = tmp_path / "second"
90 +
91 + monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(first))
92 + assert _files.get_cache_directory() == first
93 +
94 + monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(second))
95 + assert _files.get_cache_directory() == second
96 +
97 +
98 +def test_home_is_not_resolved_when_the_env_var_is_set(
99 + monkeypatch: pytest.MonkeyPatch, tmp_path: Path
100 +) -> None:
101 + """The escape hatch has to work when ``Path.home()`` itself fails.
102 +
103 + The old code passed ``Path.home() / ...`` as the *default argument* to
104 + ``os.environ.get``, which Python evaluates eagerly, so setting
105 + ``COMPOSIO_CACHE_DIR`` did not save a user whose home directory could not
106 + be resolved.
107 + """
108 +
109 + def _explode() -> Path:
110 + raise RuntimeError("Could not determine home directory")
111 +
112 + monkeypatch.setattr(Path, "home", staticmethod(_explode))
113 + monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(tmp_path / "cache"))
114 +
115 + assert _files.get_cache_directory() == tmp_path / "cache"
116 +
117 +
118 +def test_unresolvable_home_raises_a_helpful_error(
119 + monkeypatch: pytest.MonkeyPatch,
120 +) -> None:
121 + def _explode() -> Path:
122 + raise RuntimeError("Could not determine home directory")
123 +
124 + monkeypatch.setattr(Path, "home", staticmethod(_explode))
125 + monkeypatch.delenv("COMPOSIO_CACHE_DIR", raising=False)
126 +
127 + with pytest.raises(RuntimeError, match="COMPOSIO_CACHE_DIR"):
128 + _files.get_cache_directory()
129 +
130 +
131 +def test_output_file_directory_hangs_off_the_cache_directory(
132 + monkeypatch: pytest.MonkeyPatch, tmp_path: Path
133 +) -> None:
134 + monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(tmp_path / "cache"))
135 + assert _files.get_output_file_directory() == tmp_path / "cache" / "files"
136 +
137 +
138 +def test_ensure_cache_directory_still_reports_unwritable_paths(
139 + monkeypatch: pytest.MonkeyPatch, unusable_cache_dir: Path
140 +) -> None:
141 + """The writability check is deferred, not removed."""
142 + monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(unusable_cache_dir))
143 +
144 + with pytest.raises(RuntimeError, match="COMPOSIO_CACHE_DIR"):
145 + _files.ensure_cache_directory()
146 +
147 +
148 +def test_legacy_module_constants_still_resolve(
149 + monkeypatch: pytest.MonkeyPatch, tmp_path: Path
150 +) -> None:
151 + """``LOCAL_CACHE_DIRECTORY`` was public-ish; keep it importable (PEP 562)."""
152 + monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(tmp_path / "cache"))
153 +
154 + assert _files.LOCAL_CACHE_DIRECTORY == tmp_path / "cache"
155 + assert _files.LOCAL_OUTPUT_FILE_DIRECTORY == tmp_path / "cache" / "files"
156 +
157 + # And unlike the old module-level constants, they track the environment.
158 + monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(tmp_path / "other"))
159 + assert _files.LOCAL_CACHE_DIRECTORY == tmp_path / "other"
160 +
161 +
162 +def test_unknown_module_attribute_still_raises_attribute_error() -> None:
163 + """The module ``__getattr__`` must not swallow genuine typos."""
164 + with pytest.raises(AttributeError, match="no attribute"):
165 + _files.LOCAL_CACHE_DIRECTROY # noqa: B018 (deliberate typo)
166 +
167 +
168 +def test_file_helper_with_explicit_outdir_never_touches_the_default_cache(
169 + monkeypatch: pytest.MonkeyPatch, unusable_cache_dir: Path, tmp_path: Path
170 +) -> None:
171 + """A caller-supplied ``outdir`` must not force the default cache directory
172 + into existence, and downloads into it must not trigger the writability
173 + check for the (unused) default cache directory."""
174 + monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(unusable_cache_dir))
175 +
176 + outdir = tmp_path / "custom-outdir"
177 + helper = FileHelper(client=Mock(), outdir=str(outdir))
178 + assert helper._outdir_is_default is False
179 +
180 + tool = Mock()
181 + tool.toolkit.slug = "gmail"
182 + tool.slug = "GMAIL_FETCH"
183 +
184 + # The download itself fails fast on the blank s3url, but the point of
185 + # this test is what happened *before* that: the default cache directory
186 + # must never have been created.
187 + with pytest.raises(BlockedInternalUrlError):
188 + helper._download_file_value(
189 + {"name": "irrelevant.txt", "mimetype": "text/plain", "s3url": ""},
190 + tool,
191 + )
192 + assert not unusable_cache_dir.exists()
193 +
194 +
195 +def test_file_helper_default_outdir_ensures_cache_directory_on_download(
196 + monkeypatch: pytest.MonkeyPatch, tmp_path: Path
197 +) -> None:
198 + """The default-outdir path is exactly where the deferred writability
199 + check must still fire, with the original error message."""
200 + unusable = tmp_path / "blocker-file"
201 + unusable.write_text("not a directory")
202 + monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(unusable / "cache"))
203 +
204 + helper = FileHelper(client=Mock())
205 + assert helper._outdir_is_default is True
206 +
207 + tool = Mock()
208 + tool.toolkit.slug = "gmail"
209 + tool.slug = "GMAIL_FETCH"
210 +
211 + with pytest.raises(RuntimeError, match="COMPOSIO_CACHE_DIR"):
212 + helper._download_file_value(
213 + {
214 + "name": "irrelevant.txt",
215 + "mimetype": "text/plain",
216 + "s3url": "https://example.com/f",
217 + },
218 + tool,
219 + )
220 +
221 +
222 +def test_file_helper_empty_string_outdir_is_treated_as_default(
223 + monkeypatch: pytest.MonkeyPatch, tmp_path: Path
224 +) -> None:
225 + """``outdir=""`` falls through to the default directory (falsy, not just
226 + ``None``), so the default-ness flag must agree or the writability check
227 + silently stops firing for it."""
228 + unusable = tmp_path / "blocker-file"
229 + unusable.write_text("not a directory")
230 + monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(unusable / "cache"))
231 +
232 + helper = FileHelper(client=Mock(), outdir="")
233 + assert helper._outdir_is_default is True
234 + assert helper._outdir == _files.get_output_file_directory()
235 +
236 + tool = Mock()
237 + tool.toolkit.slug = "gmail"
238 + tool.slug = "GMAIL_FETCH"
239 +
240 + with pytest.raises(RuntimeError, match="COMPOSIO_CACHE_DIR"):
241 + helper._download_file_value(
242 + {
243 + "name": "irrelevant.txt",
244 + "mimetype": "text/plain",
245 + "s3url": "https://example.com/f",
246 + },
247 + tool,
248 + )
python/tests/test_path_join_guardrail.py
+3 -6
@@ -58,13 +58,10 @@ def _reviewed_path(
58 REVIEWED_JOINS: t.Dict[t.Tuple[str, str], t.Dict[str, t.Any]] = {
59 (
60 "composio/core/models/_files.py",
61 - (
62 - "Path( os.environ.get( ENV_LOCAL_CACHE_DIRECTORY, Path.home() / "
63 - "LOCAL_CACHE_DIRECTORY_NAME, # Fallback to user directory ) )"
64 - ),
61 + "Path(configured)",
62 ): _reviewed_path(
66 - "The cache root comes from local configuration or the user's home, not an "
67 - "API response."
63 + "`configured` is read from the COMPOSIO_CACHE_DIR environment variable, "
64 + "which is local configuration, not an API response."
65 ),
66 (
67 "composio/core/models/_files.py",