Remove backup file count cap
Let backup creation and restore cleanup use unlimited pattern scans so archives for agents with more than 50,000 files are complete. Keep UI preview and dry-run paths bounded for responsiveness while making the dry-run truncated flag safe for optional limits. Add regression coverage for unlimited scans, backup creation, clean-before-restore, and restoring an archive entry past the old 50,000-file boundary.
Alessandro committed
Jun 26, 2026 at 14:03 UTC
71607476071ee4ba4d9f0c7036dc62e0a26318b3
5 files changed
+182
-10
api/backup_test.py
+2
-1
@@ -47,12 +47,13 @@ class BackupTest(ApiHandler):
47
48
backup_service = BackupService()
49
matched_files = await backup_service.test_patterns(metadata, max_files=max_files)
50
+ truncated = max_files is not None and len(matched_files) >= max_files
51
52
return {
53
"success": True,
54
"files": matched_files,
55
"total_count": len(matched_files),
55
- "truncated": len(matched_files) >= max_files
56
+ "truncated": truncated
57
}
58
59
except Exception as e:
api/backup_test.py.dox.md
+1
@@ -25,6 +25,7 @@
25
- `BackupTest` defines `requires_auth(...)`.
26
- `BackupTest` defines `requires_loopback(...)`.
27
- Imported dependency areas include: `helpers.api`, `helpers.backup`.
28
+- The `truncated` response flag is true only when a finite `max_files` limit is supplied and the result reaches that limit.
29
30
## Key Concepts
31
helpers/backup.py
+15
-8
@@ -240,8 +240,12 @@ class BackupService:
240
241
return translated_patterns
242
243
- async def test_patterns(self, metadata: Dict[str, Any], max_files: int = 1000) -> List[Dict[str, Any]]:
244
- """Test backup patterns and return list of matched files"""
243
+ async def test_patterns(self, metadata: Dict[str, Any], max_files: Optional[int] = 1000) -> List[Dict[str, Any]]:
244
+ """Test backup patterns and return list of matched files.
245
+
246
+ Pass max_files=None for internal flows that must process the complete
247
+ match set, such as backup creation and restore cleanup.
248
+ """
249
include_patterns = metadata.get("include_patterns", [])
250
exclude_patterns = metadata.get("exclude_patterns", [])
251
include_hidden = metadata.get("include_hidden", True)
@@ -258,6 +262,7 @@ class BackupService:
262
# Get explicit patterns for hidden file handling
263
explicit_patterns = self._get_explicit_patterns(include_patterns)
264
265
+ has_limit = max_files is not None
266
matched_files = []
267
processed_count = 0
268
@@ -285,7 +290,7 @@ class BackupService:
290
dirs[:] = dirs_to_keep
291
292
for file in files_list:
288
- if processed_count >= max_files:
293
+ if has_limit and processed_count >= max_files:
294
break
295
296
file_path = os.path.join(root, file)
@@ -317,10 +322,10 @@ class BackupService:
322
# Skip files we can't access
323
continue
324
320
- if processed_count >= max_files:
325
+ if has_limit and processed_count >= max_files:
326
break
327
323
- if processed_count >= max_files:
328
+ if has_limit and processed_count >= max_files:
329
break
330
331
except Exception as e:
@@ -344,8 +349,10 @@ class BackupService:
349
"include_hidden": include_hidden
350
}
351
347
- # Get matched files
348
- matched_files = await self.test_patterns(metadata, max_files=50000)
352
+ # Get the complete matched file set. Preview and dry-run callers may
353
+ # cap their scans for UI responsiveness, but the archive itself must be
354
+ # complete.
355
+ matched_files = await self.test_patterns(metadata, max_files=None)
356
357
if not matched_files:
358
raise Exception("No files matched the backup patterns")
@@ -824,7 +831,7 @@ class BackupService:
831
832
# Find existing files that match the translated user-edited patterns
833
try:
827
- existing_files = await self.test_patterns(metadata, max_files=10000)
834
+ existing_files = await self.test_patterns(metadata, max_files=None)
835
836
# Convert to delete operations format
837
files_to_delete = []
helpers/backup.py.dox.md
+2
-1
@@ -13,7 +13,7 @@
13
- Classes:
14
- `BackupService` (no explicit base class)
15
- `get_default_backup_metadata(self) -> Dict[str, Any]`
16
- - `async test_patterns(self, metadata: Dict[str, Any], max_files: int=...) -> List[Dict[str, Any]]`
16
+ - `async test_patterns(self, metadata: Dict[str, Any], max_files: Optional[int]=...) -> List[Dict[str, Any]]`
17
- `async create_backup(self, include_patterns: List[str], exclude_patterns: List[str], include_hidden: bool=..., backup_name: str=...) -> str`
18
- `async inspect_backup(self, backup_file) -> Dict[str, Any]`
19
- `async preview_restore(self, backup_file, restore_include_patterns: Optional[List[str]]=..., restore_exclude_patterns: Optional[List[str]]=..., overwrite_policy: str=..., clean_before_restore: bool=..., user_edited_metadata: Optional[Dict[str, Any]]=...) -> Dict[str, Any]`
@@ -25,6 +25,7 @@
25
- Update this file whenever public functions, classes, persistence behavior, path/security assumptions, side effects, or cross-module contracts change.
26
- Observed side-effect areas: filesystem reads, filesystem writes, filesystem deletion, settings/state persistence, secret handling.
27
- Imported dependency areas include: `datetime`, `helpers`, `helpers.localization`, `helpers.print_style`, `json`, `os`, `pathspec`, `platform`, `tempfile`, `typing`, `zipfile`.
28
+- `test_patterns(..., max_files=None)` is the unlimited scan mode. UI preview and dry-run callers may pass bounded limits, but real backup creation and restore clean-before-restore must use unlimited matching so archives and cleanup are not silently truncated.
29
30
## Key Concepts
31
tests/test_backup_large_archives.py
new
+162
@@ -0,0 +1,162 @@
1
+import json
2
+import shutil
3
+import zipfile
4
+from pathlib import Path
5
+
6
+import pytest
7
+
8
+from helpers.backup import BackupService
9
+
10
+
11
+class UploadedBackup:
12
+ def __init__(self, path: Path):
13
+ self.path = path
14
+
15
+ def save(self, target: str) -> None:
16
+ shutil.copyfile(self.path, target)
17
+
18
+
19
+@pytest.mark.asyncio
20
+async def test_pattern_scan_can_run_without_file_limit(tmp_path):
21
+ root = tmp_path / "a0"
22
+ usr = root / "usr"
23
+ usr.mkdir(parents=True)
24
+ for index in range(3):
25
+ (usr / f"file-{index}.txt").write_text(f"{index}\n", encoding="utf-8")
26
+
27
+ service = BackupService()
28
+ service.agent_zero_root = str(root)
29
+ service.base_paths = {str(root): str(root)}
30
+ metadata = {
31
+ "include_patterns": [f"{root}/usr/**"],
32
+ "exclude_patterns": [],
33
+ "include_hidden": True,
34
+ }
35
+
36
+ capped_files = await service.test_patterns(metadata, max_files=2)
37
+ all_files = await service.test_patterns(metadata, max_files=None)
38
+
39
+ assert len(capped_files) == 2
40
+ assert len(all_files) == 3
41
+
42
+
43
+@pytest.mark.asyncio
44
+async def test_create_backup_uses_unlimited_pattern_scan(tmp_path, monkeypatch):
45
+ source_file = tmp_path / "source.txt"
46
+ source_file.write_text("payload\n", encoding="utf-8")
47
+ captured = {}
48
+
49
+ service = BackupService()
50
+
51
+ async def fake_test_patterns(metadata, max_files=1000):
52
+ captured["max_files"] = max_files
53
+ return [
54
+ {
55
+ "path": f"{service.agent_zero_root.rstrip('/')}/usr/file-{index}.txt",
56
+ "real_path": str(source_file),
57
+ "size": source_file.stat().st_size,
58
+ "modified": "2026-06-26T00:00:00+00:00",
59
+ "type": "file",
60
+ }
61
+ for index in range(3)
62
+ ]
63
+
64
+ async def fake_info():
65
+ return {}
66
+
67
+ async def fake_author():
68
+ return "test"
69
+
70
+ monkeypatch.setattr(service, "test_patterns", fake_test_patterns)
71
+ monkeypatch.setattr(service, "_get_system_info", fake_info)
72
+ monkeypatch.setattr(service, "_get_environment_info", fake_info)
73
+ monkeypatch.setattr(service, "_get_backup_author", fake_author)
74
+
75
+ zip_path = await service.create_backup(
76
+ include_patterns=[f"{service.agent_zero_root}/usr/**"],
77
+ exclude_patterns=[],
78
+ include_hidden=True,
79
+ backup_name="large-backup",
80
+ )
81
+
82
+ assert captured["max_files"] is None
83
+ with zipfile.ZipFile(zip_path) as archive:
84
+ metadata = json.loads(archive.read("metadata.json").decode("utf-8"))
85
+ assert metadata["total_files"] == 3
86
+ assert (
87
+ f"{service.agent_zero_root.rstrip('/').lstrip('/')}/usr/file-2.txt"
88
+ in archive.namelist()
89
+ )
90
+
91
+
92
+@pytest.mark.asyncio
93
+async def test_restore_can_reach_files_after_50000_archive_entries(tmp_path):
94
+ old_root = "/old-a0"
95
+ archive_root = old_root.lstrip("/")
96
+ file_count = 50_001
97
+ last_index = file_count - 1
98
+ zip_path = tmp_path / "large-backup.zip"
99
+
100
+ metadata = {
101
+ "environment_info": {"agent_zero_root": old_root},
102
+ "include_patterns": [f"{old_root}/usr/large/**"],
103
+ "exclude_patterns": [],
104
+ "include_hidden": True,
105
+ }
106
+ with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_STORED) as archive:
107
+ archive.writestr("metadata.json", json.dumps(metadata))
108
+ for index in range(file_count):
109
+ payload = "tail payload\n" if index == last_index else ""
110
+ archive.writestr(
111
+ f"{archive_root}/usr/large/file-{index:05d}.txt",
112
+ payload,
113
+ )
114
+
115
+ service = BackupService()
116
+ service.agent_zero_root = str(tmp_path / "restored-a0")
117
+
118
+ result = await service.restore_backup(
119
+ backup_file=UploadedBackup(zip_path),
120
+ restore_include_patterns=[
121
+ f"{old_root}/usr/large/file-{last_index:05d}.txt"
122
+ ],
123
+ restore_exclude_patterns=[],
124
+ overwrite_policy="overwrite",
125
+ )
126
+
127
+ restored_path = (
128
+ Path(service.agent_zero_root)
129
+ / "usr"
130
+ / "large"
131
+ / f"file-{last_index:05d}.txt"
132
+ )
133
+ assert len(result["restored_files"]) == 1
134
+ assert len(result["skipped_files"]) == file_count - 1
135
+ assert result["errors"] == []
136
+ assert restored_path.read_text(encoding="utf-8") == "tail payload\n"
137
+
138
+
139
+@pytest.mark.asyncio
140
+async def test_restore_clean_before_restore_uses_unlimited_pattern_scan(monkeypatch):
141
+ service = BackupService()
142
+ captured = {}
143
+
144
+ async def fake_test_patterns(metadata, max_files=1000):
145
+ captured["max_files"] = max_files
146
+ return []
147
+
148
+ monkeypatch.setattr(service, "test_patterns", fake_test_patterns)
149
+
150
+ result = await service._find_files_to_clean_with_user_metadata(
151
+ user_metadata={
152
+ "include_patterns": [f"{service.agent_zero_root}/usr/**"],
153
+ "exclude_patterns": [],
154
+ "include_hidden": True,
155
+ },
156
+ original_metadata={
157
+ "environment_info": {"agent_zero_root": service.agent_zero_root}
158
+ },
159
+ )
160
+
161
+ assert result == []
162
+ assert captured["max_files"] is None