knowledge import/reload
frdel committed
Dec 8, 2024 at 19:42 UTC
f7b3e2540c0ab798658cc6fed783942423d74df8
7 files changed
+55
-24
python/api/import_knowledge.py
+16
-3
@@ -2,7 +2,7 @@ from python.helpers.api import ApiHandler
2
from flask import Request, Response
3
4
from python.helpers.file_browser import FileBrowser
5
-from python.helpers import files
5
+from python.helpers import files, memory
6
import os
7
from werkzeug.utils import secure_filename
8
@@ -12,8 +12,14 @@ class ImportKnowledge(ApiHandler):
12
if "files[]" not in request.files:
13
raise Exception("No files part")
14
15
+ ctxid = request.form.get("ctxid", "")
16
+ if not ctxid:
17
+ raise Exception("No context id provided")
18
+
19
+ context = self.get_context(ctxid)
20
+
21
file_list = request.files.getlist("files[]")
16
- KNOWLEDGE_FOLDER = files.get_abs_path("knowledge/custom/main")
22
+ KNOWLEDGE_FOLDER = files.get_abs_path(memory.get_custom_knowledge_subdir_abs(context.agent0),"main")
23
24
saved_filenames = []
25
@@ -23,4 +29,11 @@ class ImportKnowledge(ApiHandler):
29
file.save(os.path.join(KNOWLEDGE_FOLDER, filename))
30
saved_filenames.append(filename)
31
26
- return {"message": "Knowledge Imported", "filenames": saved_filenames}
32
+ #reload memory to re-import knowledge
33
+ await memory.Memory.reload(context.agent0)
34
+ context.log.set_initial_progress()
35
+
36
+ return {
37
+ "message": "Knowledge Imported",
38
+ "filenames": saved_filenames[:5]
39
+ }
\ No newline at end of file
python/api/poll.py
+1
@@ -35,5 +35,6 @@ class Poll(ApiHandler):
35
"log_guid": context.log.guid,
36
"log_version": len(context.log.updates),
37
"log_progress": context.log.progress,
38
+ "log_progress_active": context.log.progress_active,
39
"paused": context.paused,
40
}
\ No newline at end of file
python/extensions/monologue_end/_90_waiting_for_input_msg.py
+1
-3
@@ -6,7 +6,5 @@ class WaitingForInputMsg(Extension):
6
async def execute(self, loop_data: LoopData = LoopData(), **kwargs):
7
# show temp info message
8
if self.agent.number == 0:
9
- self.agent.context.log.log(
10
- type="util", heading="Waiting for input", temp=True
11
- )
9
+ self.agent.context.log.set_initial_progress()
10
python/helpers/log.py
+12
-10
@@ -83,8 +83,7 @@ class Log:
83
self.guid: str = str(uuid.uuid4())
84
self.updates: list[int] = []
85
self.logs: list[LogItem] = []
86
- self.progress = ""
87
- self.progress_no = 0
86
+ self.set_initial_progress()
87
88
def log(
89
self,
@@ -111,8 +110,7 @@ class Log:
110
self.logs.append(item)
111
self.updates += [item.no]
112
if heading and item.no >= self.progress_no:
114
- self.progress = heading
115
- self.progress_no = item.no
113
+ self.set_progress(heading, item.no)
114
return item
115
116
def update_item(
@@ -131,8 +129,7 @@ class Log:
129
if heading is not None:
130
item.heading = heading
131
if no >= self.progress_no:
134
- self.progress = heading
135
- self.progress_no = no
132
+ self.set_progress(heading, no)
133
if content is not None:
134
item.content = content
135
if kvps is not None:
@@ -149,9 +146,15 @@ class Log:
146
147
self.updates += [item.no]
148
152
- def set_progress(self, progress: str):
149
+ def set_progress(self, progress: str, no: int = 0, active: bool = True):
150
self.progress = progress
154
- self.progress_no = len(self.logs)
151
+ if not no:
152
+ no = len(self.logs)
153
+ self.progress_no = no
154
+ self.progress_active = active
155
+
156
+ def set_initial_progress(self):
157
+ self.set_progress("Waiting for input", 0, False)
158
159
def output(self, start=None, end=None):
160
if start is None:
@@ -172,5 +175,4 @@ class Log:
175
self.guid = str(uuid.uuid4())
176
self.updates = []
177
self.logs = []
175
- self.progress = ""
176
- self.progress_no = 0
178
+ self.set_initial_progress()
python/helpers/memory.py
+18
-1
@@ -72,6 +72,13 @@ class Memory:
72
memory_subdir=memory_subdir,
73
)
74
75
+ @staticmethod
76
+ async def reload(agent: Agent):
77
+ memory_subdir = agent.config.memory_subdir or "default"
78
+ if Memory.index.get(memory_subdir):
79
+ del Memory.index[memory_subdir]
80
+ return await Memory.get(agent)
81
+
82
@staticmethod
83
def initialize(
84
log_item: LogItem | None,
@@ -151,6 +158,9 @@ class Memory:
158
async def preload_knowledge(
159
self, log_item: LogItem | None, kn_dirs: list[str], memory_subdir: str
160
):
161
+ if log_item:
162
+ log_item.update(heading="Preloading knowledge...")
163
+
164
# db abs path
165
db_dir = Memory._abs_db_dir(memory_subdir)
166
@@ -351,5 +361,12 @@ class Memory:
361
def get_timestamp():
362
return datetime.now().strftime("%Y-%m-%d %H:%M:%S")
363
364
+
365
def get_memory_subdir_abs(agent: Agent) -> str:
355
- return files.get_abs_path("memory", agent.config.memory_subdir or "default")
\ No newline at end of file
366
+ return files.get_abs_path("memory", agent.config.memory_subdir or "default")
367
+
368
+def get_custom_knowledge_subdir_abs(agent: Agent) -> str:
369
+ for dir in agent.config.knowledge_subdirs:
370
+ if dir != "default":
371
+ return files.get_abs_path("knowledge", dir)
372
+ raise Exception("No custom knowledge subdir set")
python/helpers/persist_chat.py
+1
-2
@@ -166,8 +166,7 @@ def _deserialize_agents(
166
def _deserialize_log(data: dict[str, Any]) -> "Log":
167
log = Log()
168
log.guid = data.get("guid", str(uuid.uuid4()))
169
- log.progress = "" # data.get("progress", "")
170
- log.progress_no = data.get("progress_no", 0)
169
+ log.set_initial_progress()
170
171
# Deserialize the list of LogItem objects
172
i = 0
webui/index.js
+6
-5
@@ -269,6 +269,8 @@ window.loadKnowledge = async function () {
269
formData.append('files[]', file);
270
}
271
272
+ formData.append('ctxid', getContext());
273
+
274
const response = await fetch('/import_knowledge', {
275
method: 'POST',
276
body: formData,
@@ -357,7 +359,7 @@ async function poll() {
359
afterMessagesUpdate(response.logs)
360
}
361
360
- updateProgress(response.log_progress)
362
+ updateProgress(response.log_progress, response.log_progress_active)
363
364
//set ui model vars from backend
365
const inputAD = Alpine.$data(inputSection);
@@ -400,11 +402,10 @@ function speakMessages(logs) {
402
}
403
}
404
403
-function updateProgress(progress) {
404
- const defaultText = "Waiting for input"
405
- if (!progress) progress = defaultText
405
+function updateProgress(progress, active) {
406
+ if (!progress) progress = ""
407
407
- if (progress == defaultText) {
408
+ if (!active) {
409
removeClassFromElement(progressBar, "shiny-text")
410
} else {
411
addClassToElement(progressBar, "shiny-text")