1560
class TestFetchFileFromUrl:
1561
"""Test cases for _fetch_file_from_url function."""
1562
1563
- @patch("composio.core.models._files.requests.get")
1563
+ @patch("composio.core.models._files.safe_get")
1564
def test_fetch_file_from_url_success(self, mock_get):
1565
"""Test successful file fetch from URL."""
1566
mock_response = MagicMock()
1581
mock_get.assert_called_once_with(
1582
"https://example.com/image.jpg",
1583
stream=True,
1584
- allow_redirects=False,
1584
timeout=(5, 60),
1585
)
1586
1588
- @patch("composio.core.models._files.requests.get")
1587
+ @patch("composio.core.models._files.safe_get")
1588
def test_fetch_file_from_url_with_charset_in_content_type(self, mock_get):
1589
"""Test that charset is stripped from content-type."""
1590
mock_response = MagicMock()
1601
1602
assert mimetype == "text/html"
1603
1605
- @patch("composio.core.models._files.requests.get")
1604
+ @patch("composio.core.models._files.safe_get")
1605
def test_fetch_file_from_url_generates_filename_when_missing(self, mock_get):
1606
"""Test filename generation when URL has no filename."""
1607
mock_response = MagicMock()
1617
assert filename.startswith("file_")
1618
assert filename.endswith(".png")
1619
1621
- @patch("composio.core.models._files.requests.get")
1620
+ @patch("composio.core.models._files.safe_get")
1621
def test_fetch_file_from_url_generates_filename_when_no_extension(self, mock_get):
1622
"""Test filename generation when URL filename has no extension."""
1623
mock_response = MagicMock()
1635
assert filename.startswith("file_")
1636
assert filename.endswith(".pdf")
1637
1639
- @patch("composio.core.models._files.requests.get")
1638
+ @patch("composio.core.models._files.safe_get")
1639
def test_fetch_file_from_url_failure(self, mock_get):
1640
"""Test error handling when URL fetch fails."""
1641
mock_response = MagicMock()
1650
assert "Failed to fetch file from URL" in str(exc_info.value)
1651
assert "404" in str(exc_info.value)
1652
1654
- @patch("composio.core.models._files.requests.get")
1653
+ @patch("composio.core.models._files.safe_get")
1654
def test_fetch_file_from_url_decodes_percent_encoded_filename(self, mock_get):
1655
"""Test that percent-encoded characters in URL filenames are decoded."""
1656
mock_response = MagicMock()
1671
assert content == b"document content"
1672
assert mimetype == "application/pdf"
1673
1675
- @patch("composio.core.models._files.requests.get")
1674
+ @patch("composio.core.models._files.safe_get")
1675
def test_fetch_file_from_url_decodes_unicode_filename(self, mock_get):
1676
"""Test that percent-encoded unicode characters in URL filenames are decoded."""
1677
mock_response = MagicMock()
1690
# Filename should be decoded to unicode
1691
assert filename == "ファイル.jpg"
1692
1694
- @patch("composio.core.models._files.requests.get")
1693
+ @patch("composio.core.models._files.safe_get")
1694
def test_fetch_file_from_url_handles_plus_sign_in_filename(self, mock_get):
1695
"""Test that plus signs in URL paths are preserved (not converted to spaces)."""
1696
mock_response = MagicMock()
2214
class TestFetchFileFromUrlWithTruncation:
2215
"""Test cases for _fetch_file_from_url with filename truncation."""
2216
2218
- @patch("composio.core.models._files.assert_safe_fetch_target")
2219
- @patch("composio.core.models._files.requests.get")
2220
- def test_fetch_truncates_long_filename(self, mock_get, _mock_safe_fetch_target):
2217
+ @patch("composio.core.models._files.safe_get")
2218
+ def test_fetch_truncates_long_filename(self, mock_get):
2219
"""Long filenames from URLs should be truncated."""
2220
mock_response = MagicMock()
2221
mock_response.ok = True
2237
assert filename.endswith(".pdf")
2238
assert content == b"test content"
2239
2242
- @patch("composio.core.models._files.requests.get")
2240
+ @patch("composio.core.models._files.safe_get")
2241
def test_fetch_preserves_short_filename(self, mock_get):
2242
"""Short filenames should be preserved unchanged."""
2243
mock_response = MagicMock()
2254
2255
assert filename == "photo.jpg"
2256
2259
- @patch("composio.core.models._files.assert_safe_fetch_target")
2260
- @patch("composio.core.models._files.requests.get")
2261
- def test_fetch_truncates_after_adding_extension(
2262
- self, mock_get, _mock_safe_fetch_target
2263
- ):
2257
+ @patch("composio.core.models._files.safe_get")
2258
+ def test_fetch_truncates_after_adding_extension(self, mock_get):
2259
"""Truncation should happen after extension is appended."""
2260
mock_response = MagicMock()
2261
mock_response.ok = True
2275
assert len(filename) <= _MAX_FILENAME_LENGTH
2276
assert filename.endswith(".pdf")
2277
2283
- @patch("composio.core.models._files.requests.get")
2278
+ @patch("composio.core.models._files.safe_get")
2279
def test_fetch_generated_filename_not_truncated(self, mock_get):
2280
"""Generated timestamped filenames (when URL has no filename) should be short enough."""
2281
mock_response = MagicMock()
2294
assert filename.endswith(".png")
2295
assert len(filename) < 50 # Timestamped names are short
2296
2302
- @patch("composio.core.models._files.requests.get")
2297
+ @patch("composio.core.models._files.safe_get")
2298
def test_fetch_long_real_world_url(self, mock_get):
2299
"""Long real-world URLs should be handled correctly."""
2300
mock_response = MagicMock()
2321
class TestResponseSizeLimit:
2322
"""Test response size limiting."""
2323
2329
- @patch("composio.core.models._files.requests.get")
2324
+ @patch("composio.core.models._files.safe_get")
2325
def test_rejects_oversized_content_length(self, mock_get):
2326
"""Files with Content-Length > max_size should be rejected early."""
2327
mock_response = MagicMock()
2336
"https://example.com/large.zip", max_size=100 * 1024 * 1024
2337
)
2338
2344
- @patch("composio.core.models._files.requests.get")
2339
+ @patch("composio.core.models._files.safe_get")
2340
def test_rejects_oversized_during_streaming(self, mock_get):
2341
"""Files that exceed max_size during download should be rejected."""
2342
mock_response = MagicMock()
2355
"https://example.com/large.zip", max_size=10 * 1024 * 1024
2356
)
2357
2363
- @patch("composio.core.models._files.requests.get")
2358
+ @patch("composio.core.models._files.safe_get")
2359
def test_accepts_file_within_limit(self, mock_get):
2360
"""Files within size limit should be accepted."""
2361
mock_response = MagicMock()
2377
class TestRedirectHandling:
2378
"""Test redirect handling (redirects should be rejected)."""
2379
2385
- @patch("composio.core.models._files.requests.get")
2380
+ @patch("composio.core.models._files.safe_get")
2381
def test_rejects_redirect_302(self, mock_get):
2382
"""302 redirects should be rejected with clear error message."""
2383
mock_response = MagicMock()
2389
with pytest.raises(ErrorUploadingFile, match="redirect"):
2390
_fetch_file_from_url("https://example.com/redirect")
2391
2397
- @patch("composio.core.models._files.requests.get")
2392
+ @patch("composio.core.models._files.safe_get")
2393
def test_rejects_redirect_301(self, mock_get):
2394
"""301 redirects should be rejected."""
2395
mock_response = MagicMock()
2401
with pytest.raises(ErrorUploadingFile, match="redirect"):
2402
_fetch_file_from_url("https://example.com/test")
2403
2409
- @patch("composio.core.models._files.requests.get")
2404
+ @patch("composio.core.models._files.safe_get")
2405
def test_rejects_redirect_307(self, mock_get):
2406
"""307 redirects should be rejected."""
2407
mock_response = MagicMock()
2413
with pytest.raises(ErrorUploadingFile, match="redirect"):
2414
_fetch_file_from_url("https://example.com/test")
2415
2421
- @patch("composio.core.models._files.requests.get")
2416
+ @patch("composio.core.models._files.safe_get")
2417
def test_rejects_redirect_308(self, mock_get):
2418
"""308 redirects should be rejected."""
2419
mock_response = MagicMock()
2535
2536
@pytest.fixture(autouse=True)
2537
def _allow_fetch_target(self):
2543
- with patch("composio.core.models._files.assert_safe_fetch_target"):
2544
- yield
2538
+ yield
2539
2540
def _mock_response(self, content: bytes = b"data") -> MagicMock:
2541
response = MagicMock()
2553
s3url="https://example.com/file",
2554
)
2555
with patch(
2562
- "composio.core.models._files.requests.get",
2556
+ "composio.core.models._files.safe_get",
2557
return_value=self._mock_response(b"#!/bin/sh\n"),
2558
):
2559
written = f.download(outdir, root=outdir)
2572
s3url="https://example.com/file",
2573
)
2574
with patch(
2581
- "composio.core.models._files.requests.get",
2575
+ "composio.core.models._files.safe_get",
2576
return_value=self._mock_response(b"x"),
2577
):
2578
written = f.download(outdir, root=outdir)
2593
s3url="https://example.com/file",
2594
)
2595
with patch(
2602
- "composio.core.models._files.requests.get",
2596
+ "composio.core.models._files.safe_get",
2597
return_value=self._mock_response(),
2598
):
2599
with pytest.raises(ErrorDownloadingFile, match="Path traversal detected"):
2609
s3url="https://example.com/file",
2610
)
2611
with patch(
2618
- "composio.core.models._files.requests.get",
2612
+ "composio.core.models._files.safe_get",
2613
return_value=self._mock_response(b"%PDF-1.4"),
2614
):
2615
written = f.download(outdir, root=outdir)
2625
s3url="https://example.com/file",
2626
)
2627
with patch(
2634
- "composio.core.models._files.requests.get",
2628
+ "composio.core.models._files.safe_get",
2629
return_value=self._mock_response(b"%PDF-1.4"),
2630
) as mock_get:
2631
f.download(outdir, root=outdir)
2632
2633
mock_get.assert_called_once_with(
2640
- url="https://example.com/file",
2634
+ "https://example.com/file",
2635
stream=True,
2642
- allow_redirects=False,
2636
timeout=(5, 60),
2637
)
2638
2644
s3url="https://example.com/file?token=abc",
2645
)
2646
with patch(
2654
- "composio.core.models._files.requests.get",
2647
+ "composio.core.models._files.safe_get",
2648
side_effect=requests.exceptions.Timeout(
2649
"Max retries exceeded with url: /file?token=abc"
2650
),
2669
s3url="https://example.com/file?token=abc",
2670
)
2671
with patch(
2679
- "composio.core.models._files.requests.get",
2672
+ "composio.core.models._files.safe_get",
2673
return_value=response,
2674
):
2675
with pytest.raises(ErrorDownloadingFile) as exc_info:
2689
s3url="https://example.com/file?token=abc",
2690
)
2691
with patch(
2699
- "composio.core.models._files.requests.get",
2692
+ "composio.core.models._files.safe_get",
2693
return_value=response,
2694
):
2695
with pytest.raises(ErrorDownloadingFile) as exc_info:
2713
s3url="https://example.com/file",
2714
)
2715
with patch(
2723
- "composio.core.models._files.requests.get",
2716
+ "composio.core.models._files.safe_get",
2717
return_value=self._mock_response(),
2718
):
2719
with pytest.raises(ErrorDownloadingFile, match="Path traversal detected"):
2740
s3url="https://example.com/file",
2741
)
2742
with patch(
2750
- "composio.core.models._files.requests.get",
2743
+ "composio.core.models._files.safe_get",
2744
return_value=self._mock_response(b"x"),
2745
):
2746
with pytest.raises(ErrorDownloadingFile, match="no usable basename"):
2776
s3url="https://example.com/file",
2777
)
2778
with patch(
2786
- "composio.core.models._files.requests.get",
2779
+ "composio.core.models._files.safe_get",
2780
return_value=self._mock_response(b"x"),
2781
):
2782
with pytest.raises(ErrorDownloadingFile, match=reason):
2831
response.status_code = 200
2832
response.iter_content = lambda chunk_size: [content]
2833
response.close = MagicMock()
2841
- return patch("composio.core.models._files.requests.get", return_value=response)
2834
+ return patch("composio.core.models._files.safe_get", return_value=response)
2835
2836
@pytest.mark.parametrize("slug", TRAVERSALS)
2837
def test_hostile_tool_slug_is_rejected(self, slug, tmp_path):
3062
s3url="https://s3.example.com/file",
3063
)
3064
with patch(
3072
- "composio.core.models._files.assert_safe_fetch_target"
3073
- ) as mock_assert:
3074
- with patch(
3075
- "composio.core.models._files.requests.get",
3076
- return_value=self._download_response(),
3077
- ):
3078
- f.download(tmp_path / "out", root=tmp_path / "out")
3065
+ "composio.core.models._files.safe_get",
3066
+ return_value=self._download_response(),
3067
+ ) as mock_get:
3068
+ f.download(tmp_path / "out", root=tmp_path / "out")
3069
3080
- mock_assert.assert_called_once_with("https://s3.example.com/file")
3070
+ # `safe_get` is the guard: it validates the target and connects to the
3071
+ # address it validated, rather than re-resolving the hostname.
3072
+ assert mock_get.call_args.args == ("https://s3.example.com/file",)
3073
3074
def test_download_blocked_url_never_reaches_the_network(self, tmp_path):
3075
outdir = tmp_path / "out"
3079
s3url="http://169.254.169.254/latest/meta-data",
3080
)
3081
with patch(
3090
- "composio.core.models._files.assert_safe_fetch_target",
3082
+ "composio.utils.url_safety.assert_safe_fetch_target",
3083
side_effect=BlockedInternalUrlError("blocked"),
3084
):
3093
- with patch("composio.core.models._files.requests.get") as mock_get:
3085
+ with patch(
3086
+ "composio.utils.url_safety.requests.Session.request"
3087
+ ) as mock_send:
3088
with pytest.raises(BlockedInternalUrlError):
3089
f.download(outdir, root=outdir)
3090
3097
- mock_get.assert_not_called()
3091
+ mock_send.assert_not_called()
3092
assert not outdir.exists()
3093
3094
def test_download_refuses_to_follow_redirects(self, tmp_path):
3097
mimetype="application/pdf",
3098
s3url="https://s3.example.com/file",
3099
)
3106
- with patch("composio.core.models._files.assert_safe_fetch_target"):
3107
- with patch(
3108
- "composio.core.models._files.requests.get",
3109
- return_value=self._download_response(),
3110
- ) as mock_get:
3111
- f.download(tmp_path / "out", root=tmp_path / "out")
3100
+ with patch(
3101
+ "composio.core.models._files.safe_get",
3102
+ return_value=self._download_response(),
3103
+ ) as mock_get:
3104
+ f.download(tmp_path / "out", root=tmp_path / "out")
3105
3113
- assert mock_get.call_args.kwargs["allow_redirects"] is False
3106
+ # `safe_get` never follows redirects, and passing `allow_redirects`
3107
+ # through to it would be a way to turn that off.
3108
+ assert "allow_redirects" not in mock_get.call_args.kwargs
3109
3110
def test_upload_bytes_to_s3_goes_through_safe_request(self):
3111
client = self._s3_client("https://s3.example.com/upload")
3131
"composio.utils.url_safety.assert_safe_fetch_target",
3132
side_effect=BlockedInternalUrlError("blocked"),
3133
):
3139
- with patch("composio.utils.url_safety.requests.request") as mock_request:
3134
+ with patch(
3135
+ "composio.utils.url_safety.requests.Session.request"
3136
+ ) as mock_request:
3137
with pytest.raises(BlockedInternalUrlError):
3138
_upload_bytes_to_s3(
3139
client=client,
3167
"composio.utils.url_safety.assert_safe_fetch_target",
3168
side_effect=BlockedInternalUrlError("blocked"),
3169
):
3173
- with patch("composio.utils.url_safety.requests.request") as mock_request:
3170
+ with patch(
3171
+ "composio.utils.url_safety.requests.Session.request"
3172
+ ) as mock_request:
3173
with pytest.raises(BlockedInternalUrlError):
3174
upload(url="http://127.0.0.1:9000/upload", file=source)
3175