Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
fix(python): create the cache directory on first use instead of at import time (#4162)
Alberto Schiabel committed
Aug 18, 2026 at 15:45 UTC
03429c74044b09c2c71fdcfa3bc59579744a7219
4 files changed
+344
-33
python/composio/core/models/_files.py
+84
-25
@@ -82,35 +82,85 @@ ENV_LOCAL_CACHE_DIRECTORY = "COMPOSIO_CACHE_DIR"
82
Environment to set the composio caching directory.
83
"""
84
85
-LOCAL_CACHE_DIRECTORY = Path(
86
- os.environ.get(
87
- ENV_LOCAL_CACHE_DIRECTORY,
88
- Path.home() / LOCAL_CACHE_DIRECTORY_NAME, # Fallback to user directory
89
- )
90
-)
85
+LOCAL_OUTPUT_FILE_DIRECTORY_NAME = "files"
86
"""
92
-Path to local caching directory.
87
+Name of the cache sub-directory into which files downloaded during tool
88
+execution are written. Previously ``outputs``; now ``files`` for parity with
89
+the TypeScript SDK.
90
"""
91
95
-try:
96
- LOCAL_CACHE_DIRECTORY.mkdir(parents=True, exist_ok=True)
97
- if not os.access(LOCAL_CACHE_DIRECTORY, os.W_OK):
98
- raise OSError
99
-except OSError as e:
100
- raise RuntimeError(
101
- f"Cache directory {LOCAL_CACHE_DIRECTORY} is not writable please "
102
- f"provide a path that is writable using {ENV_LOCAL_CACHE_DIRECTORY} "
103
- "environment variable."
104
- ) from e
92
93
+def get_cache_directory() -> Path:
94
+ """Resolve the local caching directory without touching the filesystem.
95
107
-LOCAL_OUTPUT_FILE_DIRECTORY = LOCAL_CACHE_DIRECTORY / "files"
108
-"""
109
-Default local directory into which files downloaded during tool execution are
110
-written. Previously ``<cache>/outputs``; now ``<cache>/files`` for parity with
111
-the TypeScript SDK. Override by passing ``file_download_dir=...`` to Composio,
112
-or by setting ``outdir`` on ``FileHelper`` directly.
113
-"""
96
+ ``COMPOSIO_CACHE_DIR`` is read on every call, so it can be set after
97
+ ``composio`` has already been imported. ``Path.home()`` is only consulted
98
+ when the variable is unset: it can raise ``RuntimeError`` when there is no
99
+ resolvable home directory, which is exactly the situation
100
+ ``COMPOSIO_CACHE_DIR`` exists to work around, so it must not be evaluated
101
+ eagerly as a fallback argument.
102
+ """
103
+ configured = os.environ.get(ENV_LOCAL_CACHE_DIRECTORY)
104
+ if configured:
105
+ return Path(configured)
106
+
107
+ try:
108
+ home = Path.home()
109
+ except RuntimeError as e:
110
+ raise RuntimeError(
111
+ "Could not determine a home directory to store the Composio cache "
112
+ f"in. Provide a writable path using the {ENV_LOCAL_CACHE_DIRECTORY} "
113
+ "environment variable."
114
+ ) from e
115
+ return home / LOCAL_CACHE_DIRECTORY_NAME
116
+
117
+
118
+def get_output_file_directory() -> Path:
119
+ """Default local directory into which files downloaded during tool
120
+ execution are written. Override by passing ``file_download_dir=...`` to
121
+ Composio, or by setting ``outdir`` on ``FileHelper`` directly.
122
+ """
123
+ return get_cache_directory() / LOCAL_OUTPUT_FILE_DIRECTORY_NAME
124
+
125
+
126
+def ensure_cache_directory() -> Path:
127
+ """Create the cache directory on first use and check that it is writable.
128
+
129
+ This used to run at module import time, so a bare ``import composio``
130
+ raised ``RuntimeError`` on any read-only filesystem -- AWS Lambda,
131
+ distroless containers, ``ProtectHome=true`` systemd units -- even for
132
+ programs that never touched a file. Deferring it to first use keeps the
133
+ same check, and the same error message, for the callers that actually
134
+ need the directory.
135
+ """
136
+ directory = get_cache_directory()
137
+ try:
138
+ directory.mkdir(parents=True, exist_ok=True)
139
+ if not os.access(directory, os.W_OK):
140
+ raise OSError
141
+ except OSError as e:
142
+ raise RuntimeError(
143
+ f"Cache directory {directory} is not writable please "
144
+ f"provide a path that is writable using {ENV_LOCAL_CACHE_DIRECTORY} "
145
+ "environment variable."
146
+ ) from e
147
+ return directory
148
+
149
+
150
+def __getattr__(name: str) -> Path:
151
+ """Keep the historical module-level path constants working, but lazily.
152
+
153
+ ``LOCAL_CACHE_DIRECTORY`` and ``LOCAL_OUTPUT_FILE_DIRECTORY`` used to be
154
+ computed at import time. They are now resolved on attribute access
155
+ instead (PEP 562), so importing this module no longer touches the
156
+ filesystem or depends on the environment, and both constants observe a
157
+ ``COMPOSIO_CACHE_DIR`` that was set after import.
158
+ """
159
+ if name == "LOCAL_CACHE_DIRECTORY":
160
+ return get_cache_directory()
161
+ if name == "LOCAL_OUTPUT_FILE_DIRECTORY":
162
+ return get_output_file_directory()
163
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
164
165
166
def get_md5(file: Path) -> str:
@@ -670,7 +720,11 @@ class FileHelper(WithLogger):
720
"""
721
super().__init__()
722
self._client = client
673
- self._outdir = Path(outdir) if outdir else LOCAL_OUTPUT_FILE_DIRECTORY
723
+ # Falsy, not just `is None`: an empty string falls through to the
724
+ # same default directory below, and must be treated as such here too
725
+ # or `ensure_cache_directory()` silently stops firing for it.
726
+ self._outdir_is_default = not outdir
727
+ self._outdir = Path(outdir) if outdir else get_output_file_directory()
728
self._sensitive_file_upload_protection = sensitive_file_upload_protection
729
self._file_upload_path_deny_segments = file_upload_path_deny_segments
730
self._file_upload_allowlist: t.Optional[t.Sequence[Path]] = (
@@ -1103,6 +1157,11 @@ class FileHelper(WithLogger):
1157
1158
def _download_file_value(self, value: t.Any, tool: Tool) -> t.Any:
1159
if isinstance(value, dict) and "s3url" in value:
1160
+ if self._outdir_is_default:
1161
+ # First point at which the cache directory is genuinely
1162
+ # needed. Raises the same RuntimeError that used to be raised
1163
+ # at import time, pointing at COMPOSIO_CACHE_DIR.
1164
+ ensure_cache_directory()
1165
# `tool.toolkit.slug` and `tool.slug` come from the API response and
1166
# are untrusted, so joining them directly would let the response pick
1167
# the download directory. `secure_join` validates each component and
python/tests/test_files.py
+9
-2
@@ -52,8 +52,15 @@ def mock_client():
52
53
54
@pytest.fixture
55
-def file_helper(mock_client):
56
- """Create a FileHelper instance with a mock client."""
55
+def file_helper(mock_client, monkeypatch, tmp_path):
56
+ """Create a FileHelper instance with a mock client.
57
+
58
+ Sandboxes ``COMPOSIO_CACHE_DIR`` so the default-outdir download path
59
+ (``ensure_cache_directory()``) never touches the real home directory,
60
+ even though ``FileDownloadable.download`` itself is mocked in most of
61
+ these tests.
62
+ """
63
+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(tmp_path / ".composio"))
64
return FileHelper(client=mock_client)
65
66
python/tests/test_lazy_cache_directory.py
new
+248
@@ -0,0 +1,248 @@
1
+"""Regression tests for lazy creation of the Composio cache directory.
2
+
3
+``composio/core/models/_files.py`` used to create ``~/.composio`` and assert
4
+that it was writable at *module import time*. Because ``composio/__init__.py``
5
+imports ``tools.py``, which imports ``_files.py`` unconditionally, a bare
6
+``import composio`` raised ``RuntimeError`` on any read-only filesystem --
7
+AWS Lambda, distroless containers, ``ProtectHome=true`` systemd units -- even
8
+for programs that never touched a file (#4150).
9
+
10
+These tests pin the fix: importing the package performs no filesystem work,
11
+``COMPOSIO_CACHE_DIR`` is honoured when set after import, and the writability
12
+check still raises the same error when the directory is genuinely needed.
13
+"""
14
+
15
+import os
16
+import subprocess
17
+import sys
18
+from pathlib import Path
19
+from unittest.mock import Mock
20
+
21
+import pytest
22
+
23
+from composio.core.models import _files
24
+from composio.core.models._files import FileHelper
25
+from composio.exceptions import BlockedInternalUrlError
26
+
27
+
28
+@pytest.fixture()
29
+def unusable_cache_dir(tmp_path: Path) -> Path:
30
+ """A cache path that cannot be created, without relying on permissions.
31
+
32
+ ``chmod``-based read-only directories are a no-op when the test suite
33
+ runs as root (``os.access(..., W_OK)`` returns ``True`` for uid 0), which
34
+ is common in containerized CI. Nesting the cache directory *underneath a
35
+ regular file* makes ``mkdir`` fail with ``NotADirectoryError`` for every
36
+ user.
37
+ """
38
+ blocker = tmp_path / "blocker"
39
+ blocker.write_text("not a directory")
40
+ return blocker / "cache"
41
+
42
+
43
+def test_importing_composio_does_not_create_the_cache_directory(
44
+ unusable_cache_dir: Path,
45
+) -> None:
46
+ """The regression test: ``import composio`` must not touch the filesystem.
47
+
48
+ Run in a subprocess so the import genuinely happens under the patched
49
+ environment rather than hitting the already-imported module in this
50
+ process.
51
+ """
52
+ env = dict(os.environ)
53
+ env["COMPOSIO_CACHE_DIR"] = str(unusable_cache_dir)
54
+
55
+ result = subprocess.run(
56
+ [sys.executable, "-c", "import composio"],
57
+ env=env,
58
+ capture_output=True,
59
+ text=True,
60
+ timeout=120,
61
+ )
62
+
63
+ assert result.returncode == 0, (
64
+ "importing composio with an uncreatable cache directory must succeed, "
65
+ f"but it failed with:\n{result.stderr}"
66
+ )
67
+ assert not unusable_cache_dir.exists()
68
+
69
+
70
+def test_cache_directory_is_created_only_on_demand(
71
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
72
+) -> None:
73
+ cache_dir = tmp_path / "not-yet-there"
74
+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(cache_dir))
75
+
76
+ # Merely resolving the path must not create it.
77
+ assert _files.get_cache_directory() == cache_dir
78
+ assert not cache_dir.exists()
79
+
80
+ assert _files.ensure_cache_directory() == cache_dir
81
+ assert cache_dir.is_dir()
82
+
83
+
84
+def test_cache_dir_env_var_is_read_after_import(
85
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
86
+) -> None:
87
+ """``COMPOSIO_CACHE_DIR`` used to be frozen at import time."""
88
+ first = tmp_path / "first"
89
+ second = tmp_path / "second"
90
+
91
+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(first))
92
+ assert _files.get_cache_directory() == first
93
+
94
+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(second))
95
+ assert _files.get_cache_directory() == second
96
+
97
+
98
+def test_home_is_not_resolved_when_the_env_var_is_set(
99
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
100
+) -> None:
101
+ """The escape hatch has to work when ``Path.home()`` itself fails.
102
+
103
+ The old code passed ``Path.home() / ...`` as the *default argument* to
104
+ ``os.environ.get``, which Python evaluates eagerly, so setting
105
+ ``COMPOSIO_CACHE_DIR`` did not save a user whose home directory could not
106
+ be resolved.
107
+ """
108
+
109
+ def _explode() -> Path:
110
+ raise RuntimeError("Could not determine home directory")
111
+
112
+ monkeypatch.setattr(Path, "home", staticmethod(_explode))
113
+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(tmp_path / "cache"))
114
+
115
+ assert _files.get_cache_directory() == tmp_path / "cache"
116
+
117
+
118
+def test_unresolvable_home_raises_a_helpful_error(
119
+ monkeypatch: pytest.MonkeyPatch,
120
+) -> None:
121
+ def _explode() -> Path:
122
+ raise RuntimeError("Could not determine home directory")
123
+
124
+ monkeypatch.setattr(Path, "home", staticmethod(_explode))
125
+ monkeypatch.delenv("COMPOSIO_CACHE_DIR", raising=False)
126
+
127
+ with pytest.raises(RuntimeError, match="COMPOSIO_CACHE_DIR"):
128
+ _files.get_cache_directory()
129
+
130
+
131
+def test_output_file_directory_hangs_off_the_cache_directory(
132
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
133
+) -> None:
134
+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(tmp_path / "cache"))
135
+ assert _files.get_output_file_directory() == tmp_path / "cache" / "files"
136
+
137
+
138
+def test_ensure_cache_directory_still_reports_unwritable_paths(
139
+ monkeypatch: pytest.MonkeyPatch, unusable_cache_dir: Path
140
+) -> None:
141
+ """The writability check is deferred, not removed."""
142
+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(unusable_cache_dir))
143
+
144
+ with pytest.raises(RuntimeError, match="COMPOSIO_CACHE_DIR"):
145
+ _files.ensure_cache_directory()
146
+
147
+
148
+def test_legacy_module_constants_still_resolve(
149
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
150
+) -> None:
151
+ """``LOCAL_CACHE_DIRECTORY`` was public-ish; keep it importable (PEP 562)."""
152
+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(tmp_path / "cache"))
153
+
154
+ assert _files.LOCAL_CACHE_DIRECTORY == tmp_path / "cache"
155
+ assert _files.LOCAL_OUTPUT_FILE_DIRECTORY == tmp_path / "cache" / "files"
156
+
157
+ # And unlike the old module-level constants, they track the environment.
158
+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(tmp_path / "other"))
159
+ assert _files.LOCAL_CACHE_DIRECTORY == tmp_path / "other"
160
+
161
+
162
+def test_unknown_module_attribute_still_raises_attribute_error() -> None:
163
+ """The module ``__getattr__`` must not swallow genuine typos."""
164
+ with pytest.raises(AttributeError, match="no attribute"):
165
+ _files.LOCAL_CACHE_DIRECTROY # noqa: B018 (deliberate typo)
166
+
167
+
168
+def test_file_helper_with_explicit_outdir_never_touches_the_default_cache(
169
+ monkeypatch: pytest.MonkeyPatch, unusable_cache_dir: Path, tmp_path: Path
170
+) -> None:
171
+ """A caller-supplied ``outdir`` must not force the default cache directory
172
+ into existence, and downloads into it must not trigger the writability
173
+ check for the (unused) default cache directory."""
174
+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(unusable_cache_dir))
175
+
176
+ outdir = tmp_path / "custom-outdir"
177
+ helper = FileHelper(client=Mock(), outdir=str(outdir))
178
+ assert helper._outdir_is_default is False
179
+
180
+ tool = Mock()
181
+ tool.toolkit.slug = "gmail"
182
+ tool.slug = "GMAIL_FETCH"
183
+
184
+ # The download itself fails fast on the blank s3url, but the point of
185
+ # this test is what happened *before* that: the default cache directory
186
+ # must never have been created.
187
+ with pytest.raises(BlockedInternalUrlError):
188
+ helper._download_file_value(
189
+ {"name": "irrelevant.txt", "mimetype": "text/plain", "s3url": ""},
190
+ tool,
191
+ )
192
+ assert not unusable_cache_dir.exists()
193
+
194
+
195
+def test_file_helper_default_outdir_ensures_cache_directory_on_download(
196
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
197
+) -> None:
198
+ """The default-outdir path is exactly where the deferred writability
199
+ check must still fire, with the original error message."""
200
+ unusable = tmp_path / "blocker-file"
201
+ unusable.write_text("not a directory")
202
+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(unusable / "cache"))
203
+
204
+ helper = FileHelper(client=Mock())
205
+ assert helper._outdir_is_default is True
206
+
207
+ tool = Mock()
208
+ tool.toolkit.slug = "gmail"
209
+ tool.slug = "GMAIL_FETCH"
210
+
211
+ with pytest.raises(RuntimeError, match="COMPOSIO_CACHE_DIR"):
212
+ helper._download_file_value(
213
+ {
214
+ "name": "irrelevant.txt",
215
+ "mimetype": "text/plain",
216
+ "s3url": "https://example.com/f",
217
+ },
218
+ tool,
219
+ )
220
+
221
+
222
+def test_file_helper_empty_string_outdir_is_treated_as_default(
223
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
224
+) -> None:
225
+ """``outdir=""`` falls through to the default directory (falsy, not just
226
+ ``None``), so the default-ness flag must agree or the writability check
227
+ silently stops firing for it."""
228
+ unusable = tmp_path / "blocker-file"
229
+ unusable.write_text("not a directory")
230
+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(unusable / "cache"))
231
+
232
+ helper = FileHelper(client=Mock(), outdir="")
233
+ assert helper._outdir_is_default is True
234
+ assert helper._outdir == _files.get_output_file_directory()
235
+
236
+ tool = Mock()
237
+ tool.toolkit.slug = "gmail"
238
+ tool.slug = "GMAIL_FETCH"
239
+
240
+ with pytest.raises(RuntimeError, match="COMPOSIO_CACHE_DIR"):
241
+ helper._download_file_value(
242
+ {
243
+ "name": "irrelevant.txt",
244
+ "mimetype": "text/plain",
245
+ "s3url": "https://example.com/f",
246
+ },
247
+ tool,
248
+ )
python/tests/test_path_join_guardrail.py
+3
-6
@@ -58,13 +58,10 @@ def _reviewed_path(
58
REVIEWED_JOINS: t.Dict[t.Tuple[str, str], t.Dict[str, t.Any]] = {
59
(
60
"composio/core/models/_files.py",
61
- (
62
- "Path( os.environ.get( ENV_LOCAL_CACHE_DIRECTORY, Path.home() / "
63
- "LOCAL_CACHE_DIRECTORY_NAME, # Fallback to user directory ) )"
64
- ),
61
+ "Path(configured)",
62
): _reviewed_path(
66
- "The cache root comes from local configuration or the user's home, not an "
67
- "API response."
63
+ "`configured` is read from the COMPOSIO_CACHE_DIR environment variable, "
64
+ "which is local configuration, not an API response."
65
),
66
(
67
"composio/core/models/_files.py",