@devries-cameron20 / composio-1 / commits / e52078094
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

feat(py): enable public URLs

jkomyno committed Jan 27, 2026 at 03:03 UTC e52078094fd8384e5d959439d5fca58b7d67e06f
2 files changed +552 -2
python/composio/core/models/_files.py
+167
@@ -4,6 +4,9 @@ import hashlib
4 import os
5 import typing as t
6 from pathlib import Path
7 +from urllib.parse import urlparse
8 +import uuid
9 +import datetime
10
11 import requests
12 import typing_extensions as te
@@ -87,6 +90,122 @@ class _FileUploadResponse(_ComposioBaseModel):
90 new_presigned_url: str
91
92
93 +def _is_url(value: str) -> bool:
94 + """Check if a string is a valid HTTP/HTTPS URL."""
95 + try:
96 + parsed = urlparse(value)
97 + return parsed.scheme in ("http", "https") and bool(parsed.netloc)
98 + except Exception:
99 + return False
100 +
101 +
102 +def _get_extension_from_mimetype(mimetype: str) -> str:
103 + """Get file extension from mimetype."""
104 + mime_to_ext = {
105 + "image/jpeg": ".jpg",
106 + "image/png": ".png",
107 + "image/gif": ".gif",
108 + "image/webp": ".webp",
109 + "image/svg+xml": ".svg",
110 + "image/bmp": ".bmp",
111 + "image/tiff": ".tiff",
112 + "application/pdf": ".pdf",
113 + "application/json": ".json",
114 + "application/xml": ".xml",
115 + "text/plain": ".txt",
116 + "text/html": ".html",
117 + "text/css": ".css",
118 + "text/javascript": ".js",
119 + "application/zip": ".zip",
120 + "application/gzip": ".gz",
121 + "audio/mpeg": ".mp3",
122 + "audio/wav": ".wav",
123 + "video/mp4": ".mp4",
124 + "video/webm": ".webm",
125 + }
126 + return mime_to_ext.get(mimetype, "")
127 +
128 +
129 +def _generate_timestamped_filename(extension: str) -> str:
130 + """Generate a unique filename with timestamp."""
131 + timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
132 + unique_id = uuid.uuid4().hex[:8]
133 + return f"file_{timestamp}_{unique_id}{extension}"
134 +
135 +
136 +def _fetch_file_from_url(url: str) -> t.Tuple[str, bytes, str]:
137 + """Fetch file content from a URL.
138 +
139 + Returns:
140 + Tuple of (filename, content_bytes, mimetype)
141 + """
142 + response = requests.get(url, timeout=30)
143 + if not response.ok:
144 + raise ErrorUploadingFile(
145 + f"Failed to fetch file from URL: {url}. Status: {response.status_code}"
146 + )
147 +
148 + content = response.content
149 + mimetype = response.headers.get("content-type", "application/octet-stream")
150 + # Handle mimetypes with charset or other parameters (e.g., "text/html; charset=utf-8")
151 + mimetype = mimetype.split(";")[0].strip()
152 +
153 + # Extract filename from URL
154 + parsed_url = urlparse(url)
155 + pathname = parsed_url.path
156 + filename = os.path.basename(pathname) if pathname else ""
157 +
158 + # If no filename from URL or no extension, generate one
159 + if not filename or filename == "/":
160 + extension = _get_extension_from_mimetype(mimetype)
161 + filename = _generate_timestamped_filename(extension)
162 + else:
163 + # If filename has no extension, try to add one from mimetype
164 + if "." not in filename:
165 + extension = _get_extension_from_mimetype(mimetype)
166 + filename = _generate_timestamped_filename(extension)
167 +
168 + return filename, content, mimetype
169 +
170 +
171 +def _upload_bytes_to_s3(
172 + client: HttpClient,
173 + filename: str,
174 + content: bytes,
175 + mimetype: str,
176 + tool: str,
177 + toolkit: str,
178 +) -> str:
179 + """Upload bytes content to S3 and return the S3 key."""
180 + md5_hash = hashlib.md5(content).hexdigest()
181 +
182 + s3meta = client.post(
183 + path=_FILE_UPLOAD,
184 + body={
185 + "md5": md5_hash,
186 + "filename": filename,
187 + "mimetype": mimetype,
188 + "tool_slug": tool,
189 + "toolkit_slug": toolkit,
190 + },
191 + cast_to=_FileUploadResponse,
192 + )
193 +
194 + # Upload the content directly to S3
195 + upload_response = requests.put(
196 + url=s3meta.new_presigned_url,
197 + data=content,
198 + headers={"Content-Type": mimetype},
199 + )
200 +
201 + if upload_response.status_code not in (200, 403):
202 + raise ErrorUploadingFile(
203 + f"Failed to upload to S3. Status: {upload_response.status_code}"
204 + )
205 +
206 + return s3meta.key
207 +
208 +
209 class FileUploadable(BaseModel):
210 model_config = ConfigDict(json_schema_extra={"file_uploadable": True})
211
@@ -94,6 +213,37 @@ class FileUploadable(BaseModel):
213 mimetype: str
214 s3key: str
215
216 + @classmethod
217 + def from_url(
218 + cls,
219 + client: HttpClient,
220 + url: str,
221 + tool: str,
222 + toolkit: str,
223 + ) -> te.Self:
224 + """Create a FileUploadable from a public URL.
225 +
226 + Fetches the file content from the URL and uploads it to S3.
227 +
228 + :param client: The HTTP client for API calls
229 + :param url: The public URL to fetch the file from
230 + :param tool: The tool slug
231 + :param toolkit: The toolkit slug
232 + :return: FileUploadable instance with S3 key
233 + """
234 + filename, content, mimetype = _fetch_file_from_url(url)
235 +
236 + s3key = _upload_bytes_to_s3(
237 + client=client,
238 + filename=filename,
239 + content=content,
240 + mimetype=mimetype,
241 + tool=tool,
242 + toolkit=toolkit,
243 + )
244 +
245 + return cls(name=filename, mimetype=mimetype, s3key=s3key)
246 +
247 @classmethod
248 def from_path(
249 cls,
@@ -102,6 +252,23 @@ class FileUploadable(BaseModel):
252 tool: str,
253 toolkit: str,
254 ) -> te.Self:
255 + """Create a FileUploadable from a local file path or public URL.
256 +
257 + If the file parameter is a URL (starts with http:// or https://),
258 + it will fetch the file content from the URL and upload it to S3.
259 + Otherwise, it treats it as a local file path.
260 +
261 + :param client: The HTTP client for API calls
262 + :param file: Local file path or public URL
263 + :param tool: The tool slug
264 + :param toolkit: The toolkit slug
265 + :return: FileUploadable instance with S3 key
266 + """
267 + # Check if it's a URL
268 + if isinstance(file, str) and _is_url(file):
269 + return cls.from_url(client=client, url=file, tool=tool, toolkit=toolkit)
270 +
271 + # Handle as local file path
272 file = Path(file)
273 if not file.exists():
274 raise SDKFileNotFoundError(
python/tests/test_files.py
+385 -2
@@ -4,13 +4,22 @@ These tests ensure that the FileHelper class correctly handles JSON schemas
4 that use anyOf, oneOf, allOf, or $ref instead of direct 'type' properties.
5 """
6
7 -from unittest.mock import Mock
7 +from unittest.mock import Mock, patch, MagicMock
8
9 import pytest
10
11 from composio.client.types import Tool, tool_list_response
12 -from composio.core.models._files import FileHelper
12 +from composio.core.models._files import (
13 + FileHelper,
14 + FileUploadable,
15 + _is_url,
16 + _get_extension_from_mimetype,
17 + _generate_timestamped_filename,
18 + _fetch_file_from_url,
19 + _upload_bytes_to_s3,
20 +)
21 from composio.core.models.base import allow_tracking
22 +from composio.exceptions import ErrorUploadingFile
23
24
25 @pytest.fixture(autouse=True)
@@ -954,3 +963,377 @@ class TestFileUploadWithMixedSchemas:
963 # Both values should be preserved unchanged
964 assert result["metadata"] == "test data"
965 assert result["count"] == 42
966 +
967 +
968 +class TestUrlHelperFunctions:
969 + """Test cases for URL-related helper functions."""
970 +
971 + def test_is_url_with_http(self):
972 + """Test _is_url correctly identifies HTTP URLs."""
973 + assert _is_url("http://example.com/file.jpg") is True
974 + assert _is_url("http://localhost:8080/api/file") is True
975 +
976 + def test_is_url_with_https(self):
977 + """Test _is_url correctly identifies HTTPS URLs."""
978 + assert _is_url("https://example.com/file.jpg") is True
979 + assert _is_url("https://images.pexels.com/photos/123.jpg") is True
980 +
981 + def test_is_url_with_local_paths(self):
982 + """Test _is_url correctly rejects local file paths."""
983 + assert _is_url("/path/to/file.jpg") is False
984 + assert _is_url("./relative/path.txt") is False
985 + assert _is_url("file.txt") is False
986 + assert _is_url("C:\\Windows\\file.txt") is False
987 +
988 + def test_is_url_with_other_schemes(self):
989 + """Test _is_url rejects non-HTTP schemes."""
990 + assert _is_url("ftp://ftp.example.com/file.txt") is False
991 + assert _is_url("file:///local/file.txt") is False
992 + assert _is_url("mailto:test@example.com") is False
993 +
994 + def test_is_url_with_invalid_inputs(self):
995 + """Test _is_url handles invalid inputs gracefully."""
996 + assert _is_url("") is False
997 + assert _is_url("not a url") is False
998 + assert _is_url("http://") is False
999 +
1000 + def test_get_extension_from_mimetype_common_types(self):
1001 + """Test _get_extension_from_mimetype for common types."""
1002 + assert _get_extension_from_mimetype("image/jpeg") == ".jpg"
1003 + assert _get_extension_from_mimetype("image/png") == ".png"
1004 + assert _get_extension_from_mimetype("application/pdf") == ".pdf"
1005 + assert _get_extension_from_mimetype("text/plain") == ".txt"
1006 + assert _get_extension_from_mimetype("application/json") == ".json"
1007 +
1008 + def test_get_extension_from_mimetype_unknown_type(self):
1009 + """Test _get_extension_from_mimetype returns empty for unknown types."""
1010 + assert _get_extension_from_mimetype("application/unknown") == ""
1011 + assert _get_extension_from_mimetype("custom/type") == ""
1012 +
1013 + def test_generate_timestamped_filename(self):
1014 + """Test _generate_timestamped_filename generates valid filenames."""
1015 + filename = _generate_timestamped_filename(".jpg")
1016 + assert filename.startswith("file_")
1017 + assert filename.endswith(".jpg")
1018 + assert len(filename) > 15 # Should have timestamp and unique ID
1019 +
1020 + def test_generate_timestamped_filename_no_extension(self):
1021 + """Test _generate_timestamped_filename works without extension."""
1022 + filename = _generate_timestamped_filename("")
1023 + assert filename.startswith("file_")
1024 + assert not filename.endswith(".")
1025 +
1026 +
1027 +class TestFetchFileFromUrl:
1028 + """Test cases for _fetch_file_from_url function."""
1029 +
1030 + @patch("composio.core.models._files.requests.get")
1031 + def test_fetch_file_from_url_success(self, mock_get):
1032 + """Test successful file fetch from URL."""
1033 + mock_response = MagicMock()
1034 + mock_response.ok = True
1035 + mock_response.content = b"test file content"
1036 + mock_response.headers = {"content-type": "image/jpeg"}
1037 + mock_get.return_value = mock_response
1038 +
1039 + filename, content, mimetype = _fetch_file_from_url(
1040 + "https://example.com/image.jpg"
1041 + )
1042 +
1043 + assert content == b"test file content"
1044 + assert mimetype == "image/jpeg"
1045 + assert filename == "image.jpg"
1046 + mock_get.assert_called_once_with("https://example.com/image.jpg", timeout=30)
1047 +
1048 + @patch("composio.core.models._files.requests.get")
1049 + def test_fetch_file_from_url_with_charset_in_content_type(self, mock_get):
1050 + """Test that charset is stripped from content-type."""
1051 + mock_response = MagicMock()
1052 + mock_response.ok = True
1053 + mock_response.content = b"<html></html>"
1054 + mock_response.headers = {"content-type": "text/html; charset=utf-8"}
1055 + mock_get.return_value = mock_response
1056 +
1057 + filename, content, mimetype = _fetch_file_from_url(
1058 + "https://example.com/page.html"
1059 + )
1060 +
1061 + assert mimetype == "text/html"
1062 +
1063 + @patch("composio.core.models._files.requests.get")
1064 + def test_fetch_file_from_url_generates_filename_when_missing(self, mock_get):
1065 + """Test filename generation when URL has no filename."""
1066 + mock_response = MagicMock()
1067 + mock_response.ok = True
1068 + mock_response.content = b"image data"
1069 + mock_response.headers = {"content-type": "image/png"}
1070 + mock_get.return_value = mock_response
1071 +
1072 + filename, content, mimetype = _fetch_file_from_url("https://example.com/")
1073 +
1074 + assert filename.startswith("file_")
1075 + assert filename.endswith(".png")
1076 +
1077 + @patch("composio.core.models._files.requests.get")
1078 + def test_fetch_file_from_url_generates_filename_when_no_extension(self, mock_get):
1079 + """Test filename generation when URL filename has no extension."""
1080 + mock_response = MagicMock()
1081 + mock_response.ok = True
1082 + mock_response.content = b"pdf data"
1083 + mock_response.headers = {"content-type": "application/pdf"}
1084 + mock_get.return_value = mock_response
1085 +
1086 + filename, content, mimetype = _fetch_file_from_url(
1087 + "https://example.com/document"
1088 + )
1089 +
1090 + assert filename.startswith("file_")
1091 + assert filename.endswith(".pdf")
1092 +
1093 + @patch("composio.core.models._files.requests.get")
1094 + def test_fetch_file_from_url_failure(self, mock_get):
1095 + """Test error handling when URL fetch fails."""
1096 + mock_response = MagicMock()
1097 + mock_response.ok = False
1098 + mock_response.status_code = 404
1099 + mock_get.return_value = mock_response
1100 +
1101 + with pytest.raises(ErrorUploadingFile) as exc_info:
1102 + _fetch_file_from_url("https://example.com/notfound.jpg")
1103 +
1104 + assert "Failed to fetch file from URL" in str(exc_info.value)
1105 + assert "404" in str(exc_info.value)
1106 +
1107 +
1108 +class TestUploadBytesToS3:
1109 + """Test cases for _upload_bytes_to_s3 function."""
1110 +
1111 + @patch("composio.core.models._files.requests.put")
1112 + def test_upload_bytes_to_s3_success(self, mock_put):
1113 + """Test successful upload to S3."""
1114 + mock_client = MagicMock()
1115 + mock_s3_response = MagicMock()
1116 + mock_s3_response.key = "s3-key-123"
1117 + mock_s3_response.new_presigned_url = "https://s3.example.com/upload"
1118 + mock_client.post.return_value = mock_s3_response
1119 +
1120 + mock_put_response = MagicMock()
1121 + mock_put_response.status_code = 200
1122 + mock_put.return_value = mock_put_response
1123 +
1124 + result = _upload_bytes_to_s3(
1125 + client=mock_client,
1126 + filename="test.jpg",
1127 + content=b"file content",
1128 + mimetype="image/jpeg",
1129 + tool="TEST_TOOL",
1130 + toolkit="test_toolkit",
1131 + )
1132 +
1133 + assert result == "s3-key-123"
1134 + mock_client.post.assert_called_once()
1135 + mock_put.assert_called_once()
1136 +
1137 + @patch("composio.core.models._files.requests.put")
1138 + def test_upload_bytes_to_s3_failure(self, mock_put):
1139 + """Test error handling when S3 upload fails."""
1140 + mock_client = MagicMock()
1141 + mock_s3_response = MagicMock()
1142 + mock_s3_response.key = "s3-key-123"
1143 + mock_s3_response.new_presigned_url = "https://s3.example.com/upload"
1144 + mock_client.post.return_value = mock_s3_response
1145 +
1146 + mock_put_response = MagicMock()
1147 + mock_put_response.status_code = 500
1148 + mock_put.return_value = mock_put_response
1149 +
1150 + with pytest.raises(ErrorUploadingFile) as exc_info:
1151 + _upload_bytes_to_s3(
1152 + client=mock_client,
1153 + filename="test.jpg",
1154 + content=b"file content",
1155 + mimetype="image/jpeg",
1156 + tool="TEST_TOOL",
1157 + toolkit="test_toolkit",
1158 + )
1159 +
1160 + assert "Failed to upload to S3" in str(exc_info.value)
1161 +
1162 +
1163 +class TestFileUploadableFromUrl:
1164 + """Test cases for FileUploadable.from_url and from_path with URLs."""
1165 +
1166 + @patch("composio.core.models._files._upload_bytes_to_s3")
1167 + @patch("composio.core.models._files._fetch_file_from_url")
1168 + def test_from_url_success(self, mock_fetch, mock_upload):
1169 + """Test successful FileUploadable creation from URL."""
1170 + mock_fetch.return_value = ("image.jpg", b"image data", "image/jpeg")
1171 + mock_upload.return_value = "s3-key-abc"
1172 + mock_client = MagicMock()
1173 +
1174 + result = FileUploadable.from_url(
1175 + client=mock_client,
1176 + url="https://example.com/image.jpg",
1177 + tool="TEST_TOOL",
1178 + toolkit="test_toolkit",
1179 + )
1180 +
1181 + assert result.name == "image.jpg"
1182 + assert result.mimetype == "image/jpeg"
1183 + assert result.s3key == "s3-key-abc"
1184 + mock_fetch.assert_called_once_with("https://example.com/image.jpg")
1185 + mock_upload.assert_called_once()
1186 +
1187 + @patch("composio.core.models._files._upload_bytes_to_s3")
1188 + @patch("composio.core.models._files._fetch_file_from_url")
1189 + def test_from_path_detects_url(self, mock_fetch, mock_upload):
1190 + """Test that from_path correctly detects and handles URLs."""
1191 + mock_fetch.return_value = ("photo.png", b"photo data", "image/png")
1192 + mock_upload.return_value = "s3-key-xyz"
1193 + mock_client = MagicMock()
1194 +
1195 + result = FileUploadable.from_path(
1196 + client=mock_client,
1197 + file="https://images.example.com/photo.png",
1198 + tool="SEND_EMAIL",
1199 + toolkit="gmail",
1200 + )
1201 +
1202 + assert result.name == "photo.png"
1203 + assert result.mimetype == "image/png"
1204 + assert result.s3key == "s3-key-xyz"
1205 + mock_fetch.assert_called_once_with("https://images.example.com/photo.png")
1206 +
1207 + @patch("composio.core.models._files._fetch_file_from_url")
1208 + def test_from_url_propagates_fetch_error(self, mock_fetch):
1209 + """Test that fetch errors are propagated correctly."""
1210 + mock_fetch.side_effect = ErrorUploadingFile("Fetch failed")
1211 + mock_client = MagicMock()
1212 +
1213 + with pytest.raises(ErrorUploadingFile) as exc_info:
1214 + FileUploadable.from_url(
1215 + client=mock_client,
1216 + url="https://example.com/missing.jpg",
1217 + tool="TEST_TOOL",
1218 + toolkit="test_toolkit",
1219 + )
1220 +
1221 + assert "Fetch failed" in str(exc_info.value)
1222 +
1223 +
1224 +class TestFileHelperWithUrls:
1225 + """Test cases for FileHelper handling URLs in file uploads."""
1226 +
1227 + @patch("composio.core.models._files.FileUploadable.from_path")
1228 + def test_substitute_file_uploads_with_url(
1229 + self, mock_from_path, file_helper, mock_tool
1230 + ):
1231 + """Test that URLs are correctly processed in substitute_file_uploads."""
1232 + mock_tool.input_parameters = {
1233 + "type": "object",
1234 + "properties": {
1235 + "attachment": {"type": "string", "file_uploadable": True},
1236 + },
1237 + }
1238 +
1239 + mock_uploadable = MagicMock()
1240 + mock_uploadable.model_dump.return_value = {
1241 + "name": "image.jpg",
1242 + "mimetype": "image/jpeg",
1243 + "s3key": "s3-key-123",
1244 + }
1245 + mock_from_path.return_value = mock_uploadable
1246 +
1247 + result = file_helper._substitute_file_uploads_recursively(
1248 + tool=mock_tool,
1249 + schema=mock_tool.input_parameters,
1250 + request={"attachment": "https://example.com/image.jpg"},
1251 + )
1252 +
1253 + assert result["attachment"] == {
1254 + "name": "image.jpg",
1255 + "mimetype": "image/jpeg",
1256 + "s3key": "s3-key-123",
1257 + }
1258 + mock_from_path.assert_called_once()
1259 +
1260 + @patch("composio.core.models._files.FileUploadable.from_path")
1261 + def test_substitute_file_uploads_with_url_in_anyof(
1262 + self, mock_from_path, file_helper, mock_tool
1263 + ):
1264 + """Test URL handling in anyOf schema variants."""
1265 + mock_tool.input_parameters = {
1266 + "type": "object",
1267 + "properties": {
1268 + "file": {
1269 + "anyOf": [
1270 + {"type": "string", "file_uploadable": True},
1271 + {"type": "null"},
1272 + ]
1273 + },
1274 + },
1275 + }
1276 +
1277 + mock_uploadable = MagicMock()
1278 + mock_uploadable.model_dump.return_value = {
1279 + "name": "doc.pdf",
1280 + "mimetype": "application/pdf",
1281 + "s3key": "s3-key-456",
1282 + }
1283 + mock_from_path.return_value = mock_uploadable
1284 +
1285 + result = file_helper._substitute_file_uploads_recursively(
1286 + tool=mock_tool,
1287 + schema=mock_tool.input_parameters,
1288 + request={"file": "https://docs.example.com/doc.pdf"},
1289 + )
1290 +
1291 + assert result["file"] == {
1292 + "name": "doc.pdf",
1293 + "mimetype": "application/pdf",
1294 + "s3key": "s3-key-456",
1295 + }
1296 +
1297 + @patch("composio.core.models._files.FileUploadable.from_path")
1298 + def test_substitute_file_uploads_array_with_urls(
1299 + self, mock_from_path, file_helper, mock_tool
1300 + ):
1301 + """Test URL handling in arrays with file_uploadable items."""
1302 + mock_tool.input_parameters = {
1303 + "type": "object",
1304 + "properties": {
1305 + "attachments": {
1306 + "type": "array",
1307 + "items": {"type": "string", "file_uploadable": True},
1308 + },
1309 + },
1310 + }
1311 +
1312 + mock_uploadable1 = MagicMock()
1313 + mock_uploadable1.model_dump.return_value = {
1314 + "name": "file1.jpg",
1315 + "mimetype": "image/jpeg",
1316 + "s3key": "key1",
1317 + }
1318 + mock_uploadable2 = MagicMock()
1319 + mock_uploadable2.model_dump.return_value = {
1320 + "name": "file2.png",
1321 + "mimetype": "image/png",
1322 + "s3key": "key2",
1323 + }
1324 + mock_from_path.side_effect = [mock_uploadable1, mock_uploadable2]
1325 +
1326 + result = file_helper._substitute_file_uploads_recursively(
1327 + tool=mock_tool,
1328 + schema=mock_tool.input_parameters,
1329 + request={
1330 + "attachments": [
1331 + "https://example.com/file1.jpg",
1332 + "https://example.com/file2.png",
1333 + ]
1334 + },
1335 + )
1336 +
1337 + assert len(result["attachments"]) == 2
1338 + assert result["attachments"][0]["s3key"] == "key1"
1339 + assert result["attachments"][1]["s3key"] == "key2"