Fix compaction backups for malformed Unicode
Use the shared sanitized file writer for chat compaction backup artifacts so JSON and transcript backups remain writable when a long chat contains malformed Unicode such as lone surrogates. Add a regression that exercises surrogate-containing backup content and document the backup artifact contract in the chat compaction DOX file. Verified with: conda run -n a0 pytest tests/test_chat_compaction.py -q; python -m py_compile plugins/_chat_compaction/helpers/compactor.py tests/test_chat_compaction.py; live localhost:32080 surrogate-long-chat repro after syncing into /a0 and restarting the container.
Alessandro committed
Jun 19, 2026 at 14:33 UTC
98e51fbb2a3f90257ce0b14552187610869a5f13
3 files changed
+31
-6
plugins/_chat_compaction/AGENTS.md
+1
@@ -16,6 +16,7 @@
16
## Local Contracts
17
18
- Preserve chat history integrity and persistence after compaction.
19
+- Backup JSON and transcript artifacts must remain UTF-8 writable when chat content contains malformed Unicode such as lone surrogates.
20
- Keep generated summaries bounded by configured model and token limits.
21
- Do not discard original context data unless the compaction flow explicitly owns that behavior.
22
plugins/_chat_compaction/helpers/compactor.py
+3
-6
@@ -4,7 +4,7 @@ from collections import deque
4
5
import models as models_module
6
from agent import Agent
7
-from helpers import tokens
7
+from helpers import files, tokens
8
from helpers.history import History, output_text
9
from helpers.persist_chat import (
10
export_json_chat,
@@ -42,11 +42,8 @@ def _save_pre_compaction_backup(context, full_text: str) -> dict[str, str]:
42
txt_path = os.path.join(backup_dir, f"pre-compact-{timestamp}.txt")
43
44
json_content = export_json_chat(context)
45
- with open(json_path, "w", encoding="utf-8") as f:
46
- f.write(json_content)
47
-
48
- with open(txt_path, "w", encoding="utf-8") as f:
49
- f.write(full_text)
45
+ files.write_file(json_path, json_content)
46
+ files.write_file(txt_path, full_text)
47
48
return {"json": json_path, "txt": txt_path}
49
tests/test_chat_compaction.py
+27
@@ -1,4 +1,5 @@
1
import sys
2
+from types import SimpleNamespace
3
from pathlib import Path
4
5
import pytest
@@ -42,6 +43,32 @@ class _RecordingModel:
43
return f"summary-{len(self.user_messages)}", None
44
45
46
+def test_pre_compaction_backup_sanitizes_surrogate_text(tmp_path, monkeypatch):
47
+ monkeypatch.setattr(
48
+ compactor,
49
+ "get_chat_folder_path",
50
+ lambda _ctxid: str(tmp_path),
51
+ )
52
+ monkeypatch.setattr(
53
+ compactor,
54
+ "export_json_chat",
55
+ lambda _context: '{"content":"before\ud83dafter"}',
56
+ )
57
+
58
+ paths = compactor._save_pre_compaction_backup(
59
+ SimpleNamespace(id="surrogate-chat"),
60
+ "transcript before\ud83dafter",
61
+ )
62
+
63
+ json_backup = Path(paths["json"]).read_text(encoding="utf-8")
64
+ text_backup = Path(paths["txt"]).read_text(encoding="utf-8")
65
+
66
+ assert "\ud83d" not in json_backup
67
+ assert "\ud83d" not in text_backup
68
+ assert "before?after" in json_backup
69
+ assert "before?after" in text_backup
70
+
71
+
72
def test_compaction_splitter_wraps_single_line_85k_payload(monkeypatch):
73
monkeypatch.setattr(
74
compactor.tokens, "approximate_tokens", lambda text: len(text or "")