refactor: migrate user data to usr/ + update frontend paths
linuztx committed
Dec 12, 2025 at 10:11 UTC
10bc874f05180a3b3655b74279615b1d80862907
19 files changed
+170
-45
initialize.py
+6
@@ -140,6 +140,12 @@ def initialize_preload():
140
import preload
141
return defer.DeferredTask().start_task(preload.preload)
142
143
+def initialize_migration():
144
+ from python.helpers import migration
145
+ # run migration
146
+ migration.migrate_user_data()
147
+ # reload settings to ensure new paths are picked up
148
+ settings.reload_settings()
149
150
def _args_override(config):
151
# update config with runtime args
python/api/api_files_get.py
+1
-1
@@ -50,7 +50,7 @@ class ApiFilesGet(ApiHandler):
50
if path.startswith("/a0/tmp/uploads/"):
51
# Internal path - convert to external
52
filename = path.replace("/a0/tmp/uploads/", "")
53
- external_path = files.get_abs_path("tmp/uploads", filename)
53
+ external_path = files.get_abs_path("usr/uploads", filename)
54
filename = os.path.basename(external_path)
55
elif path.startswith("/a0/"):
56
# Other internal Agent Zero paths
python/api/api_message.py
+2
-2
@@ -40,8 +40,8 @@ class ApiMessage(ApiHandler):
40
# Handle attachments (base64 encoded)
41
attachment_paths = []
42
if attachments:
43
- upload_folder_int = "/a0/tmp/uploads"
44
- upload_folder_ext = files.get_abs_path("tmp/uploads")
43
+ upload_folder_int = "/a0/usr/uploads"
44
+ upload_folder_ext = files.get_abs_path("usr/uploads")
45
os.makedirs(upload_folder_ext, exist_ok=True)
46
47
for attachment in attachments:
python/api/message.py
+2
-2
@@ -29,8 +29,8 @@ class Message(ApiHandler):
29
attachments = request.files.getlist("attachments")
30
attachment_paths = []
31
32
- upload_folder_int = "/a0/tmp/uploads"
33
- upload_folder_ext = files.get_abs_path("tmp/uploads") # for development environment
32
+ upload_folder_int = "/a0/usr/uploads"
33
+ upload_folder_ext = files.get_abs_path("usr/uploads") # for development environment
34
35
if attachments:
36
os.makedirs(upload_folder_ext, exist_ok=True)
python/api/upload.py
+1
-1
@@ -14,7 +14,7 @@ class UploadFile(ApiHandler):
14
for file in file_list:
15
if file and self.allowed_file(file.filename): # Check file type
16
filename = secure_filename(file.filename) # type: ignore
17
- file.save(files.get_abs_path("tmp/upload", filename))
17
+ file.save(files.get_abs_path("usr/upload", filename))
18
saved_filenames.append(filename)
19
20
return {"filenames": saved_filenames} # Return saved filenames
python/helpers/backup.py
+5
-21
@@ -60,28 +60,12 @@ class BackupService:
60
# Ensure paths don't have double slashes
61
agent_root = self.agent_zero_root.rstrip('/')
62
63
- return f"""# Agent Zero Knowledge (excluding defaults)
64
-{agent_root}/knowledge/**
65
-!{agent_root}/knowledge/default/**
66
-
67
-# Agent Zero Instruments (excluding defaults)
68
-{agent_root}/instruments/**
69
-!{agent_root}/instruments/default/**
70
-
71
-# Memory (excluding embeddings cache)
72
-{agent_root}/memory/**
73
-!{agent_root}/memory/**/embeddings/**
74
-
75
-# Configuration and Settings (CRITICAL)
76
-{agent_root}/.env
77
-{agent_root}/tmp/settings.json
78
-{agent_root}/tmp/secrets.env
79
-{agent_root}/tmp/chats/**
80
-{agent_root}/tmp/scheduler/**
81
-{agent_root}/tmp/uploads/**
82
-
83
-# User data
63
+ return f"""# User data
64
+# All persistent user data is now centralized in /usr for easier backup and restore
65
{agent_root}/usr/**
66
+
67
+# Explicitly include .env
68
+{agent_root}/usr/.env
69
"""
70
71
def _get_agent_zero_version(self) -> str:
python/helpers/dotenv.py
+1
-1
@@ -15,7 +15,7 @@ def load_dotenv():
15
16
17
def get_dotenv_file_path():
18
- return get_abs_path(".env")
18
+ return get_abs_path("usr/.env")
19
20
def get_dotenv_value(key: str, default: Any = None):
21
# load_dotenv()
python/helpers/email_client.py
+1
-1
@@ -537,7 +537,7 @@ async def read_messages(
537
port: int = 993,
538
username: str = "",
539
password: str = "",
540
- download_folder: str = "tmp/email",
540
+ download_folder: str = "usr/email",
541
options: Optional[Dict[str, Any]] = None,
542
filter: Optional[Dict[str, Any]] = None,
543
) -> List[Message]:
python/helpers/files.py
+4
@@ -410,6 +410,10 @@ def move_dir(old_path: str, new_path: str):
410
abs_new = get_abs_path(new_path)
411
if not os.path.isdir(abs_old):
412
return # nothing to rename
413
+
414
+ # ensure parent directory exists
415
+ os.makedirs(os.path.dirname(abs_new), exist_ok=True)
416
+
417
try:
418
os.rename(abs_old, abs_new)
419
except Exception:
python/helpers/memory.py
+17
-5
@@ -139,7 +139,7 @@ class Memory:
139
log_item.stream(progress="\nInitializing VectorDB")
140
141
em_dir = files.get_abs_path(
142
- "memory/embeddings"
142
+ "tmp/memory/embeddings"
143
) # just caching, no need to parameterize
144
db_dir = abs_db_dir(memory_subdir)
145
@@ -333,6 +333,16 @@ class Memory:
333
recursive=True,
334
)
335
336
+ # load custom instruments descriptions
337
+ index = knowledge_import.load_knowledge(
338
+ log_item,
339
+ files.get_abs_path("usr/instruments"),
340
+ index,
341
+ {"area": Memory.Area.INSTRUMENTS.value},
342
+ filename_pattern="**/*.md",
343
+ recursive=True,
344
+ )
345
+
346
return index
347
348
def get_document_by_id(self, id: str) -> Document | None:
@@ -483,7 +493,7 @@ class Memory:
493
def get_custom_knowledge_subdir_abs(agent: Agent) -> str:
494
for dir in agent.config.knowledge_subdirs:
495
if dir != "default":
486
- return files.get_abs_path("knowledge", dir)
496
+ return files.get_abs_path("usr/knowledge", dir)
497
raise Exception("No custom knowledge subdir set")
498
499
@@ -499,7 +509,7 @@ def abs_db_dir(memory_subdir: str) -> str:
509
510
return files.get_abs_path(get_project_meta_folder(memory_subdir[9:]), "memory")
511
# standard subdirs
502
- return files.get_abs_path("memory", memory_subdir)
512
+ return files.get_abs_path("usr/memory", memory_subdir)
513
514
515
def abs_knowledge_dir(knowledge_subdir: str, *sub_dirs: str) -> str:
@@ -511,7 +521,9 @@ def abs_knowledge_dir(knowledge_subdir: str, *sub_dirs: str) -> str:
521
get_project_meta_folder(knowledge_subdir[9:]), "knowledge", *sub_dirs
522
)
523
# standard subdirs
514
- return files.get_abs_path("knowledge", knowledge_subdir, *sub_dirs)
524
+ if knowledge_subdir == "default":
525
+ return files.get_abs_path("knowledge", *sub_dirs)
526
+ return files.get_abs_path("usr/knowledge", knowledge_subdir, *sub_dirs)
527
528
529
def get_memory_subdir_abs(agent: Agent) -> str:
@@ -546,7 +558,7 @@ def get_existing_memory_subdirs() -> list[str]:
558
)
559
560
# Get subdirectories from memory folder
549
- subdirs = files.get_subdirectories("memory", exclude="embeddings")
561
+ subdirs = files.get_subdirectories("usr/memory")
562
563
project_subdirs = files.get_subdirectories(get_projects_parent_folder())
564
for project_subdir in project_subdirs:
python/helpers/migration.py
new
+110
@@ -0,0 +1,110 @@
1
+import os
2
+from python.helpers import files
3
+from python.helpers.print_style import PrintStyle
4
+
5
+def migrate_user_data() -> None:
6
+ """
7
+ Migrate user data from /tmp and other locations to /usr.
8
+ """
9
+
10
+ PrintStyle().print("Checking for data migration...")
11
+
12
+ # --- Migrate Directories -------------------------------------------------------
13
+ # Move directories from tmp/ or other source locations to usr/
14
+
15
+ _move_dir("tmp/chats", "usr/chats")
16
+ _move_dir("tmp/scheduler", "usr/scheduler")
17
+ _move_dir("tmp/uploads", "usr/uploads")
18
+ _move_dir("tmp/upload", "usr/upload")
19
+ _move_dir("tmp/downloads", "usr/downloads")
20
+ _move_dir("tmp/email", "usr/email")
21
+ _move_dir("knowledge/custom", "usr/knowledge")
22
+ _move_dir("instruments/custom", "usr/instruments")
23
+
24
+ # --- Migrate Files -------------------------------------------------------------
25
+ # Move specific configuration files to usr/
26
+
27
+ _move_file("tmp/settings.json", "usr/settings.json")
28
+ _move_file("tmp/secrets.env", "usr/secrets.env")
29
+ _move_file(".env", "usr/.env")
30
+
31
+ # --- Special Migration Cases ---------------------------------------------------
32
+
33
+ # Migrate Memory
34
+ _migrate_memory()
35
+
36
+ # Flatten default directories (knowledge/default -> knowledge/, etc.)
37
+ # We use _merge_dir_contents because we want to move the *contents* of default/
38
+ # into the parent directory, not move the default directory itself.
39
+ _merge_dir_contents("knowledge/default", "knowledge")
40
+ _merge_dir_contents("instruments/default", "instruments")
41
+
42
+ # --- Cleanup -------------------------------------------------------------------
43
+
44
+ # Remove obsolete directories after migration
45
+ _cleanup_obsolete()
46
+
47
+ PrintStyle().print("Migration check complete.")
48
+
49
+# --- Helper Functions ----------------------------------------------------------
50
+
51
+def _move_dir(src: str, dst: str) -> None:
52
+ """
53
+ Move a directory from src to dst if src exists and dst does not.
54
+ """
55
+ if files.exists(src) and not files.exists(dst):
56
+ PrintStyle().print(f"Migrating {src} to {dst}...")
57
+ files.move_dir(src, dst)
58
+
59
+def _move_file(src: str, dst: str) -> None:
60
+ """
61
+ Move a file from src to dst if src exists and dst does not.
62
+ """
63
+ if files.exists(src) and not files.exists(dst):
64
+ PrintStyle().print(f"Migrating {src} to {dst}...")
65
+ files.move_file(src, dst)
66
+
67
+def _migrate_memory(base_path: str = "memory") -> None:
68
+ """
69
+ Migrate memory subdirectories.
70
+ """
71
+ subdirs = files.get_subdirectories(base_path)
72
+ for subdir in subdirs:
73
+ if subdir == "embeddings":
74
+ # Special case: Embeddings
75
+ _move_dir("memory/embeddings", "tmp/memory/embeddings")
76
+ else:
77
+ # Move other memory items to usr/memory
78
+ dst = f"usr/memory/{subdir}"
79
+ _move_dir(f"memory/{subdir}", dst)
80
+
81
+def _merge_dir_contents(src_parent: str, dst_parent: str) -> None:
82
+ """
83
+ Moves all subdirectories from src_parent to dst_parent.
84
+ Useful for flattening structures like 'knowledge/default/*' -> 'knowledge/*'.
85
+ """
86
+ if not files.exists(src_parent):
87
+ return
88
+
89
+ # Iterate over subdirectories in the source parent
90
+ subdirs = files.get_subdirectories(src_parent)
91
+ for subdir in subdirs:
92
+ src = f"{src_parent}/{subdir}"
93
+ dst = f"{dst_parent}/{subdir}"
94
+
95
+ # Move the subdirectory if it doesn't exist in destination
96
+ _move_dir(src, dst)
97
+
98
+def _cleanup_obsolete() -> None:
99
+ """
100
+ Remove directories that are no longer needed.
101
+ """
102
+ to_remove = [
103
+ "knowledge/default",
104
+ "instruments/default",
105
+ "memory"
106
+ ]
107
+ for path in to_remove:
108
+ if files.exists(path):
109
+ PrintStyle().print(f"Removing {path}...")
110
+ files.delete_dir(path)
python/helpers/persist_chat.py
+1
-1
@@ -9,7 +9,7 @@ from initialize import initialize_agent
9
10
from python.helpers.log import Log, LogItem
11
12
-CHATS_FOLDER = "tmp/chats"
12
+CHATS_FOLDER = "usr/chats"
13
LOG_SIZE = 1000
14
CHAT_FILE_NAME = "chat.json"
15
python/helpers/secrets.py
+1
-1
@@ -15,7 +15,7 @@ if TYPE_CHECKING:
15
16
# New alias-based placeholder format §§secret(KEY)
17
ALIAS_PATTERN = r"§§secret\(([A-Za-z_][A-Za-z0-9_]*)\)"
18
-DEFAULT_SECRETS_FILE = "tmp/secrets.env"
18
+DEFAULT_SECRETS_FILE = "usr/secrets.env"
19
20
21
def alias_for_key(key: str, placeholder: str = "§§secret({key})") -> str:
python/helpers/settings.py
+7
-1
@@ -161,7 +161,7 @@ class SettingsOutput(TypedDict):
161
PASSWORD_PLACEHOLDER = "****PSWD****"
162
API_KEY_PLACEHOLDER = "************"
163
164
-SETTINGS_FILE = files.get_abs_path("tmp/settings.json")
164
+SETTINGS_FILE = files.get_abs_path("usr/settings.json")
165
_settings: Settings | None = None
166
167
@@ -1343,6 +1343,12 @@ def get_settings() -> Settings:
1343
return norm
1344
1345
1346
+def reload_settings() -> Settings:
1347
+ global _settings
1348
+ _settings = None
1349
+ return get_settings()
1350
+
1351
+
1352
def set_settings(settings: Settings, apply: bool = True):
1353
global _settings
1354
previous = _settings
python/helpers/task_scheduler.py
+1
-1
@@ -26,7 +26,7 @@ from python.helpers import projects
26
import pytz
27
from typing import Annotated
28
29
-SCHEDULER_FOLDER = "tmp/scheduler"
29
+SCHEDULER_FOLDER = "usr/scheduler"
30
31
# ----------------------
32
# Task Models
python/tools/browser_agent.py
+1
-1
@@ -56,7 +56,7 @@ class State:
56
disable_security=True,
57
chromium_sandbox=False,
58
accept_downloads=True,
59
- downloads_path=files.get_abs_path("tmp/downloads"),
59
+ downloads_path=files.get_abs_path("usr/downloads"),
60
allowed_domains=["*", "http://*", "https://*"],
61
executable_path=pw_binary,
62
keep_alive=True,
run_ui.py
+3
@@ -190,6 +190,9 @@ async def serve_index():
190
def run():
191
PrintStyle().print("Initializing framework...")
192
193
+ # migrate data before anything else
194
+ initialize.initialize_migration()
195
+
196
# Suppress only request logs but keep the startup messages
197
from werkzeug.serving import WSGIRequestHandler
198
from werkzeug.serving import make_server
webui/components/chat/attachments/attachmentsStore.js
+2
-2
@@ -259,11 +259,11 @@ const model = {
259
260
// Generate server-side API URL for file (for device sync)
261
getServerImgUrl(filename) {
262
- return `/image_get?path=/a0/tmp/uploads/${encodeURIComponent(filename)}`;
262
+ return `/image_get?path=/a0/usr/uploads/${encodeURIComponent(filename)}`;
263
},
264
265
getServerFileUrl(filename) {
266
- return `/a0/tmp/uploads/${encodeURIComponent(filename)}`;
266
+ return `/a0/usr/uploads/${encodeURIComponent(filename)}`;
267
},
268
269
// Check if file is an image based on extension
webui/components/settings/external/api-examples.html
+4
-4
@@ -146,7 +146,7 @@
146
<div style="background-color: var(--color-bg-secondary); border: 1px solid var(--color-border); border-radius: 6px; padding: 12px; margin: 0 0 20px 0;">
147
<p style="margin: 0 0 8px 0; color: var(--color-text-secondary); font-size: 14px;">
148
<strong>Parameters:</strong><br>
149
- • <code>paths</code> (array, required): Array of file paths to retrieve (e.g., ["/a0/tmp/uploads/file.txt"])
149
+ • <code>paths</code> (array, required): Array of file paths to retrieve (e.g., ["/a0/usr/uploads/file.txt"])
150
</p>
151
<p style="margin: 0; color: var(--color-text-secondary); font-size: 14px;">
152
<strong>Headers:</strong> <code>X-API-KEY</code> (required), <code>Content-Type: application/json</code>
@@ -562,8 +562,8 @@ async function getFiles(filePaths) {
562
563
// Example 1: Get specific files
564
const filePaths = [
565
- "/a0/tmp/uploads/document.txt",
566
- "/a0/tmp/uploads/data.json"
565
+ "/a0/usr/uploads/document.txt",
566
+ "/a0/usr/uploads/data.json"
567
];
568
getFiles(filePaths);
569
@@ -590,7 +590,7 @@ async function attachmentWorkflow() {
590
console.log('Message sent with attachment');
591
592
// Step 2: Retrieve the uploaded file
593
- const retrievedFiles = await getFiles(["/a0/tmp/uploads/test.txt"]);
593
+ const retrievedFiles = await getFiles(["/a0/usr/uploads/test.txt"]);
594
595
if (retrievedFiles && retrievedFiles["test.txt"]) {
596
const originalContent = atob(retrievedFiles["test.txt"]);