The best way to control Claude Code, Codex and more from any computer or phone

main
ts 193 lines 8.05 KB
Raw
1 import { assistantCitationsToPlainText } from "@t3tools/shared/assistantCitations";
2 import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "@t3tools/shared/composerContextReferences";
3 import * as Effect from "effect/Effect";
4 import type * as SqlClient from "effect/sql/SqlClient";
5
6 // SQLite's ltrim needs the characters matched by isAttachmentOnlyBootstrapPrompt's \s explicitly.
7 const BOOTSTRAP_PREFIX_WHITESPACE =
8 "\t\n\v\f\r \u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000\ufeff";
9
10 export const nonPlaceholderMessageCondition = (sql: SqlClient.SqlClient, row = "messages") =>
11 sql.literal(`
12 CASE
13 WHEN substr(json_extract(${row}.payload_json, '$.text'), 1, 11) = 'Ultrathink:' COLLATE NOCASE
14 THEN ltrim(substr(json_extract(${row}.payload_json, '$.text'), 12), '${BOOTSTRAP_PREFIX_WHITESPACE}')
15 ELSE json_extract(${row}.payload_json, '$.text')
16 END COLLATE BINARY <> '${ATTACHMENT_ONLY_BOOTSTRAP_PROMPT.replaceAll("'", "''")}'
17 `);
18
19 /**
20 * Exclude attachment placeholders, generated notices and subagent threads. ACP subagents
21 * rewrite their finished answer while it streams.
22 * Migration triggers, backfill, rebuild and scans share this predicate; `row` may be `new`.
23 */
24 export const indexedMessageCondition = (sql: SqlClient.SqlClient, row = "messages") =>
25 sql`
26 ${sql.literal(`
27 ${row}.streaming = 0
28 AND (
29 ${row}.role = 'assistant'
30 OR (
31 ${row}.role = 'user'
32 AND COALESCE(json_extract(${row}.payload_json, '$.createdBy'), 'user') = 'user'
33 )
34 )
35 AND NOT EXISTS (
36 SELECT 1 FROM orchestration_v2_projection_threads AS subagent_threads
37 WHERE subagent_threads.thread_id = ${row}.thread_id
38 AND json_extract(subagent_threads.payload_json, '$.lineage.relationshipToParent') = 'subagent'
39 )
40 `)}
41 AND ${nonPlaceholderMessageCondition(sql, row)}
42 `;
43
44 // Curly apostrophes straight, as search folds a query, since iOS keyboards
45 // type "’" and others type "'".
46 const foldApostrophes = (text: string) => `replace(replace(${text}, '’', ''''), '‘', '''')`;
47
48 const readableText = (row: string) =>
49 `COALESCE(${row}.search_text, json_extract(${row}.payload_json, '$.text'))`;
50
51 // A file mention stores `__init__.py` as `[\_\_init\_\_.py](...)`; the index
52 // holds the label as it reads, so search can find the name as typed.
53 const LINK_LABEL = /(!?\[)([^\]\n]*)(\]\([^)\s]*\))/g;
54
55 /**
56 * What search_text holds for a message: its text as it reads, a quote's
57 * percent-encoded link replaced by its words and comment. NULL when that is
58 * the stored text, so most messages store nothing extra.
59 */
60 export const readableSearchText = (text: string) => {
61 const readable = assistantCitationsToPlainText(text).replace(
62 LINK_LABEL,
63 (_link, open: string, label: string, close: string) =>
64 open + label.replace(/\\([*_])/g, "$1") + close,
65 );
66 return readable === text ? null : readable;
67 };
68
69 /** A message's text as it reads, for thread search. */
70 export const messageSearchText = (sql: SqlClient.SqlClient) =>
71 sql.literal(readableText("messages"));
72
73 /** What the index stores for a message, so a scan matches the folded query as the index does. */
74 export const foldedMessageSearchText = (sql: SqlClient.SqlClient, row = "messages") =>
75 sql.literal(foldApostrophes(readableText(row)));
76
77 /**
78 * A message's stored text with apostrophes folded, what the index held before
79 * migration 81. Migration 69's backfill uses it. `payload` is the message's
80 * JSON column.
81 */
82 export const indexedMessageText = (sql: SqlClient.SqlClient, payload: string) =>
83 sql.literal(foldApostrophes(`json_extract(${payload}, '$.text')`));
84
85 // What the index stores: a message's readable text with apostrophes folded.
86 // Migrations 69 and 74 create the triggers before migration 81 adds
87 // search_text, and `t3 doctor` can rebuild a database that has not migrated,
88 // so those fall back to the stored text.
89 const indexedText = Effect.fn("MessageSearchIndex.indexedText")(function* (
90 sql: SqlClient.SqlClient,
91 ) {
92 const column = yield* sql`
93 SELECT 1 FROM pragma_table_info('orchestration_v2_projection_messages')
94 WHERE name = 'search_text'
95 `;
96 return column.length > 0
97 ? (row: string) => foldedMessageSearchText(sql, row)
98 : (row: string) => indexedMessageText(sql, `${row}.payload_json`);
99 });
100
101 export const createMessageSearchTriggers = Effect.fn("MessageSearchIndex.createTriggers")(
102 function* (sql: SqlClient.SqlClient) {
103 const text = yield* indexedText(sql);
104 yield* sql`
105 CREATE TRIGGER orchestration_v2_message_search_insert
106 AFTER INSERT ON orchestration_v2_projection_messages
107 WHEN ${indexedMessageCondition(sql, "new")}
108 BEGIN
109 INSERT INTO orchestration_v2_message_search (rowid, text)
110 SELECT new.rowid, ${text("new")}
111 WHERE json_extract(new.payload_json, '$.text') IS NOT NULL;
112 END
113 `;
114 // Streaming deltas rewrite the row many times a second; skip them until the
115 // message settles. Upserts rewrite every column, so re-index only when what
116 // the index stores changed; migration 81's backfill writes only search_text.
117 // Deleting a rowid the index never held is a no-op.
118 yield* sql`
119 CREATE TRIGGER orchestration_v2_message_search_update
120 AFTER UPDATE ON orchestration_v2_projection_messages
121 WHEN (old.streaming = 0 OR new.streaming = 0)
122 AND (
123 old.streaming IS NOT new.streaming
124 OR old.role IS NOT new.role
125 OR ${text("old")} IS NOT ${text("new")}
126 OR json_extract(old.payload_json, '$.createdBy')
127 IS NOT json_extract(new.payload_json, '$.createdBy')
128 )
129 BEGIN
130 DELETE FROM orchestration_v2_message_search WHERE rowid = old.rowid AND old.streaming = 0;
131 INSERT INTO orchestration_v2_message_search (rowid, text)
132 SELECT new.rowid, ${text("new")}
133 WHERE ${indexedMessageCondition(sql, "new")}
134 AND json_extract(new.payload_json, '$.text') IS NOT NULL;
135 END
136 `;
137 yield* sql`
138 CREATE TRIGGER orchestration_v2_message_search_delete
139 AFTER DELETE ON orchestration_v2_projection_messages
140 WHEN old.streaming = 0
141 BEGIN
142 DELETE FROM orchestration_v2_message_search WHERE rowid = old.rowid;
143 END
144 `;
145 },
146 );
147
148 /** Replaces the search triggers with their current definition, where the index exists. */
149 export const recreateMessageSearchTriggers = Effect.fn("MessageSearchIndex.recreateTriggers")(
150 function* (sql: SqlClient.SqlClient) {
151 const index = yield* sql`
152 SELECT 1 FROM sqlite_master WHERE name = 'orchestration_v2_message_search'
153 `;
154 if (index.length === 0) return;
155 yield* sql`DROP TRIGGER orchestration_v2_message_search_insert`;
156 yield* sql`DROP TRIGGER orchestration_v2_message_search_update`;
157 yield* sql`DROP TRIGGER orchestration_v2_message_search_delete`;
158 yield* createMessageSearchTriggers(sql);
159 },
160 );
161
162 /**
163 * Refills the search index from the messages table. The index is keyed by
164 * message rowid, so anything that may renumber or copy rows (VACUUM, a dev
165 * database copy) rebuilds it instead of trusting it. Databases from before
166 * migration 69, or on an SQLite without FTS5 trigrams, have no index.
167 */
168 export const rebuildMessageSearch = Effect.fn("MessageSearchIndex.rebuild")(function* (
169 sql: SqlClient.SqlClient,
170 ) {
171 const index = yield* sql`
172 SELECT 1 FROM sqlite_master WHERE name = 'orchestration_v2_message_search'
173 `;
174 if (index.length === 0) return;
175 const text = yield* indexedText(sql);
176 // One transaction: an interrupt between clear and refill would leave an
177 // empty index that nothing repairs, since search only checks the table exists.
178 yield* sql.withTransaction(
179 Effect.gen(function* () {
180 yield* sql`
181 INSERT INTO orchestration_v2_message_search (orchestration_v2_message_search)
182 VALUES ('delete-all')
183 `;
184 yield* sql`
185 INSERT INTO orchestration_v2_message_search (rowid, text)
186 SELECT messages.rowid, ${text("messages")}
187 FROM orchestration_v2_projection_messages AS messages
188 WHERE ${indexedMessageCondition(sql)}
189 AND json_extract(messages.payload_json, '$.text') IS NOT NULL
190 `;
191 }),
192 );
193 });