@samitouri / QOSami_Agents_Zero / commits / 1cbecc24

memory: harden FAISS integrity and consolidation scoring

- Add FAISS index integrity checks using a SHA-256 sidecar (`index.faiss.sha256`) and write hash on save. - Harden `memory_load` filter evaluation with input validation (allowlist + length cap) and `simple_eval(..., functions={})`. - Add score-preserving similarity search and use real relevance scores in consolidation (including best-score dedupe by memory id). - Prevent utility-model context overflows by truncating memorize input history for fragments and solutions.

Alessandro committed Apr 8, 2026 at 13:51 UTC 1cbecc241e89a4ac4f51579e75a20537974440e1

5 files changed +112 -64

plugins/_memory/extensions/python/monologue_end/_50_memorize_fragments.py

@@ -50,6 +50,10 @@ class MemorizeMemories(Extension):

50 # get system message and chat history for util llm

51 system = self.agent.read_prompt("memory.memories_sum.sys.md")

52 msgs_text = self.agent.concat_messages(self.agent.history)

53 + # Keep only recent context to avoid utility-model context-window overflow.

54 + MAX_MSGS_CHARS = 80000

55 + if len(msgs_text) > MAX_MSGS_CHARS:

56 + msgs_text = msgs_text[-MAX_MSGS_CHARS:]

58 # # log query streamed by LLM

59 # async def log_callback(content):

plugins/_memory/extensions/python/monologue_end/_51_memorize_solutions.py

@@ -51,6 +51,10 @@ class MemorizeSolutions(Extension):

51 # get system message and chat history for util llm

52 system = self.agent.read_prompt("memory.solutions_sum.sys.md")

53 msgs_text = self.agent.concat_messages(self.agent.history)

54 + # Keep only recent context to avoid utility-model context-window overflow.

55 + MAX_MSGS_CHARS = 80000

56 + if len(msgs_text) > MAX_MSGS_CHARS:

57 + msgs_text = msgs_text[-MAX_MSGS_CHARS:]

59 # log query streamed by LLM

60 # async def log_callback(content):

plugins/_memory/helpers/memory.py

+69 -10

@@ -18,7 +18,7 @@ from langchain_community.vectorstores.utils import (

18 )

19 from langchain_core.embeddings import Embeddings

21 -import os, json

21 +import os, json, hashlib, re

23 import numpy as np

@@ -178,14 +178,19 @@ class Memory:

178

179 # if db folder exists and is not empty:

180 if os.path.exists(db_dir) and files.exists(db_dir, "index.faiss"):

181 - db = MyFaiss.load_local(

182 - folder_path=db_dir,

183 - embeddings=embedder,

184 - allow_dangerous_deserialization=True,

185 - distance_strategy=DistanceStrategy.COSINE,

186 - # normalize_L2=True,

187 - relevance_score_fn=Memory._cosine_normalizer,

188 - ) # type: ignore

181 + if not Memory._verify_index_hash(db_dir):

182 + PrintStyle(font_color="yellow").print(

183 + f"FAISS index hash mismatch in '{db_dir}' — index will be rebuilt."

184 + )

185 + else:

186 + db = MyFaiss.load_local(

187 + folder_path=db_dir,

188 + embeddings=embedder,

189 + allow_dangerous_deserialization=True,

190 + distance_strategy=DistanceStrategy.COSINE,

191 + # normalize_L2=True,

192 + relevance_score_fn=Memory._cosine_normalizer,

193 + ) # type: ignore

194

195 # if there is a mismatch in embeddings used, re-index the whole DB

196 emb_ok = False

@@ -345,6 +350,18 @@ class Memory:

350 filter=comparator,

351 )

352

353 + async def search_similarity_threshold_with_scores(

354 + self, query: str, limit: int, threshold: float, filter: str = ""

355 + ) -> list[tuple[Document, float]]:

356 + comparator = Memory._get_comparator(filter) if filter else None

357 +

358 + return await self.db.asimilarity_search_with_relevance_scores(

359 + query,

360 + k=limit,

361 + score_threshold=threshold,

362 + filter=comparator,

363 + )

364 +

365 async def delete_documents_by_query(

366 self, query: str, threshold: float, filter: str = ""

367 ):

@@ -432,12 +449,54 @@ class Memory:

449 def _save_db_file(db: MyFaiss, memory_subdir: str):

450 abs_dir = abs_db_dir(memory_subdir)

451 db.save_local(folder_path=abs_dir)

452 + Memory._write_index_hash(abs_dir)

453 +

454 + @staticmethod

455 + def _write_index_hash(abs_dir: str) -> None:

456 + faiss_path = os.path.join(abs_dir, "index.faiss")

457 + hash_path = os.path.join(abs_dir, "index.faiss.sha256")

458 + try:

459 + h = hashlib.sha256()

460 + with open(faiss_path, "rb") as f:

461 + for chunk in iter(lambda: f.read(65536), b""):

462 + h.update(chunk)

463 + with open(hash_path, "w") as f:

464 + f.write(h.hexdigest())

465 + except Exception as e:

466 + PrintStyle(font_color="yellow").print(f"Warning: could not write FAISS hash: {e}")

467 +

468 + @staticmethod

469 + def _verify_index_hash(abs_dir: str) -> bool:

470 + faiss_path = os.path.join(abs_dir, "index.faiss")

471 + hash_path = os.path.join(abs_dir, "index.faiss.sha256")

472 + if not os.path.exists(hash_path):

473 + return True

474 + try:

475 + with open(hash_path, "r") as f:

476 + stored = f.read().strip()

477 + h = hashlib.sha256()

478 + with open(faiss_path, "rb") as f:

479 + for chunk in iter(lambda: f.read(65536), b""):

480 + h.update(chunk)

481 + return h.hexdigest() == stored

482 + except Exception as e:

483 + PrintStyle(font_color="yellow").print(f"Warning: FAISS hash check failed: {e}")

484 + return True

485

486 @staticmethod

487 def _get_comparator(condition: str):

488 + _FILTER_SAFE = re.compile(

489 + r"^[a-zA-Z0-9_\-\.\ \t'\"=<>!()\[\],:\+]+$"

490 + )

491 + if len(condition) > 512 or not _FILTER_SAFE.match(condition):

492 + PrintStyle.error(

493 + f"Memory filter rejected (unsafe characters or too long): {condition!r}"

494 + )

495 + return lambda _data: False

496 +

497 def comparator(data: dict[str, Any]):

498 try:

440 - result = simple_eval(condition, names=data)

499 + result = simple_eval(condition, names=data, functions={})

500 return result

501 except Exception as e:

502 PrintStyle.error(f"Error evaluating condition: {e}")

plugins/_memory/helpers/memory_consolidation.py

+30 -51

@@ -36,7 +36,7 @@ class ConsolidationConfig:

36 keyword_extraction_msg_prompt: str = "memory.keyword_extraction.msg.md"

37 processing_timeout_seconds: int = 60

38 # Add safety threshold for REPLACE actions

39 - replace_similarity_threshold: float = 0.9 # Higher threshold for replacement safety

39 + replace_similarity_threshold: float = 0.75 # Threshold tuned for real cosine similarity scores

42 @dataclass

@@ -336,73 +336,52 @@ class MemoryConsolidator:

336

337 all_similar = []

338

339 - # Step 2: Semantic similarity search with scores

340 - semantic_similar = await db.search_similarity_threshold(

339 + # Step 2: Semantic similarity search with real scores

340 + semantic_results = await db.search_similarity_threshold_with_scores(

341 query=new_memory,

342 limit=self.config.max_similar_memories,

343 threshold=self.config.similarity_threshold,

344 filter=f"area == '{area}'"

345 )

346 - all_similar.extend(semantic_similar)

346 + for doc, score in semantic_results:

347 + doc.metadata['_consolidation_similarity'] = score

348 + all_similar.append(doc)

349

348 - # Step 3: Keyword-based searches

350 + # Step 3: Keyword-based searches with real scores

351 for query in search_queries:

352 if query.strip():

351 - # Fix division by zero: ensure len(search_queries) > 0

352 - queries_count = max(1, len(search_queries)) # Prevent division by zero

353 - keyword_similar = await db.search_similarity_threshold(

353 + queries_count = max(1, len(search_queries))

354 + keyword_results = await db.search_similarity_threshold_with_scores(

355 query=query.strip(),

356 limit=max(3, self.config.max_similar_memories // queries_count),

357 threshold=self.config.similarity_threshold,

358 filter=f"area == '{area}'"

359 )

359 - all_similar.extend(keyword_similar)

360 + for doc, score in keyword_results:

361 + doc.metadata['_consolidation_similarity'] = score

362 + all_similar.append(doc)

363

361 - # Step 4: Deduplicate by document ID and store similarity info

362 - seen_ids = set()

363 - unique_similar = []

364 + # Step 4: Deduplicate by document ID, keep highest score per memory ID

365 + best_by_id: Dict[str, Document] = {}

366 for doc in all_similar:

365 - doc_id = doc.metadata.get('id')

366 - if doc_id and doc_id not in seen_ids:

367 - seen_ids.add(doc_id)

368 - unique_similar.append(doc)

369 -

370 - # Step 5: Calculate similarity scores for replacement validation

371 - # Since FAISS doesn't directly expose similarity scores, use ranking-based estimation

372 - # CRITICAL: All documents must have similarity >= search_threshold since FAISS returned them

373 - # FIXED: Use conservative scoring that keeps all scores in safe consolidation range

374 - similarity_scores = {}

375 - total_docs = len(unique_similar)

376 - search_threshold = self.config.similarity_threshold

377 - safety_threshold = self.config.replace_similarity_threshold

378 -

379 - for i, doc in enumerate(unique_similar):

367 doc_id = doc.metadata.get('id')

368 if doc_id:

382 - # Convert ranking to similarity score with conservative distribution

383 - if total_docs == 1:

384 - ranking_similarity = 1.0 # Single document gets perfect score

385 - else:

386 - # Use conservative scoring: distribute between safety_threshold and 1.0

387 - # This ensures all scores are suitable for consolidation

388 - # First document gets 1.0, last gets safety_threshold (0.9 by default)

389 - ranking_factor = 1.0 - (i / (total_docs - 1))

390 - score_range = 1.0 - safety_threshold # e.g., 1.0 - 0.9 = 0.1

391 - ranking_similarity = safety_threshold + (score_range * ranking_factor)

392 -

393 - # Ensure minimum score is search_threshold for logical consistency

394 - ranking_similarity = max(ranking_similarity, search_threshold)

395 -

396 - similarity_scores[doc_id] = ranking_similarity

397 -

398 - # Step 6: Add similarity score to document metadata for LLM analysis

399 - for doc in unique_similar:

400 - doc_id = doc.metadata.get('id')

401 - estimated_similarity = similarity_scores.get(doc_id, 0.7)

402 - # Store for later validation

403 - doc.metadata['_consolidation_similarity'] = estimated_similarity

369 + existing = best_by_id.get(doc_id)

370 + if (

371 + existing is None

372 + or doc.metadata.get('_consolidation_similarity', 0)

373 + > existing.metadata.get('_consolidation_similarity', 0)

374 + ):

375 + best_by_id[doc_id] = doc

376 + unique_similar = list(best_by_id.values())

377 +

378 + # Step 5: Sort by similarity score descending

379 + unique_similar.sort(

380 + key=lambda d: d.metadata.get('_consolidation_similarity', 0),

381 + reverse=True

382 + )

383

405 - # Step 7: Limit to max context for LLM

384 + # Step 6: Limit to max context for LLM

385 limited_similar = unique_similar[:self.config.max_llm_context_memories]

386

387 return limited_similar

@@ -782,7 +761,7 @@ def create_memory_consolidator(agent: Agent, **config_overrides) -> MemoryConsol

761

762 Available configuration options:

763 - similarity_threshold: Discovery threshold for finding related memories (default 0.7)

785 - - replace_similarity_threshold: Safety threshold for REPLACE actions (default 0.9)

764 + - replace_similarity_threshold: Safety threshold for REPLACE actions (default 0.75)

765 - max_similar_memories: Maximum memories to discover (default 10)

766 - max_llm_context_memories: Maximum memories to send to LLM (default 5)

767 - processing_timeout_seconds: Timeout for consolidation processing (default 30)

plugins/_memory/prompts/agent.system.tool.memory.md

+5 -3

@@ -4,14 +4,16 @@ use when durable recall or storage is useful

4 - `memory_save`: args `text`, optional `area` and metadata kwargs

5 - `memory_delete`: arg `ids` comma-separated ids

6 - `memory_forget`: args `query`, optional `threshold`, `filter`

7 +

8 notes:

9 - `threshold` is similarity from `0` to `1`

9 -- `filter` is a python expression over metadata

10 -- verify destructive memory changes if accuracy matters

10 +- `filter` is a metadata expression (e.g. `area=='main'`)

11 +- confirm destructive changes when accuracy matters

12 +

13 example:

14 ~~~json

15 {

14 - "thoughts": ["I should search memory for the relevant prior guidance."],

16 + "thoughts": ["I should search memory for relevant prior guidance."],

17 "headline": "Loading related memories",

18 "tool_name": "memory_load",

19 "tool_args": {