The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 03:37 UTC
| 1 | import { assistantCitationsToPlainText } from "@t3tools/shared/assistantCitations"; |
| 2 | import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "@t3tools/shared/composerContextReferences"; |
| 3 | import * as Effect from "effect/Effect"; |
| 4 | import type * as SqlClient from "effect/sql/SqlClient"; |
| 5 | |
| 6 | // SQLite's ltrim needs the characters matched by isAttachmentOnlyBootstrapPrompt's \s explicitly. |
| 7 | const BOOTSTRAP_PREFIX_WHITESPACE = |
| 8 | "\t\n\v\f\r \u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000\ufeff"; |
| 9 | |
| 10 | export const nonPlaceholderMessageCondition = (sql: SqlClient.SqlClient, row = "messages") => |
| 11 | sql.literal(` |
| 12 | CASE |
| 13 | WHEN substr(json_extract(${row}.payload_json, '$.text'), 1, 11) = 'Ultrathink:' COLLATE NOCASE |
| 14 | THEN ltrim(substr(json_extract(${row}.payload_json, '$.text'), 12), '${BOOTSTRAP_PREFIX_WHITESPACE}') |
| 15 | ELSE json_extract(${row}.payload_json, '$.text') |
| 16 | END COLLATE BINARY <> '${ATTACHMENT_ONLY_BOOTSTRAP_PROMPT.replaceAll("'", "''")}' |
| 17 | `); |
| 18 | |
| 19 | /** |
| 20 | * Exclude attachment placeholders, generated notices and subagent threads. ACP subagents |
| 21 | * rewrite their finished answer while it streams. |
| 22 | * Migration triggers, backfill, rebuild and scans share this predicate; `row` may be `new`. |
| 23 | */ |
| 24 | export const indexedMessageCondition = (sql: SqlClient.SqlClient, row = "messages") => |
| 25 | sql` |
| 26 | ${sql.literal(` |
| 27 | ${row}.streaming = 0 |
| 28 | AND ( |
| 29 | ${row}.role = 'assistant' |
| 30 | OR ( |
| 31 | ${row}.role = 'user' |
| 32 | AND COALESCE(json_extract(${row}.payload_json, '$.createdBy'), 'user') = 'user' |
| 33 | ) |
| 34 | ) |
| 35 | AND NOT EXISTS ( |
| 36 | SELECT 1 FROM orchestration_v2_projection_threads AS subagent_threads |
| 37 | WHERE subagent_threads.thread_id = ${row}.thread_id |
| 38 | AND json_extract(subagent_threads.payload_json, '$.lineage.relationshipToParent') = 'subagent' |
| 39 | ) |
| 40 | `)} |
| 41 | AND ${nonPlaceholderMessageCondition(sql, row)} |
| 42 | `; |
| 43 | |
| 44 | // Curly apostrophes straight, as search folds a query, since iOS keyboards |
| 45 | // type "’" and others type "'". |
| 46 | const foldApostrophes = (text: string) => `replace(replace(${text}, '’', ''''), '‘', '''')`; |
| 47 | |
| 48 | const readableText = (row: string) => |
| 49 | `COALESCE(${row}.search_text, json_extract(${row}.payload_json, '$.text'))`; |
| 50 | |
| 51 | // A file mention stores `__init__.py` as `[\_\_init\_\_.py](...)`; the index |
| 52 | // holds the label as it reads, so search can find the name as typed. |
| 53 | const LINK_LABEL = /(!?\[)([^\]\n]*)(\]\([^)\s]*\))/g; |
| 54 | |
| 55 | /** |
| 56 | * What search_text holds for a message: its text as it reads, a quote's |
| 57 | * percent-encoded link replaced by its words and comment. NULL when that is |
| 58 | * the stored text, so most messages store nothing extra. |
| 59 | */ |
| 60 | export const readableSearchText = (text: string) => { |
| 61 | const readable = assistantCitationsToPlainText(text).replace( |
| 62 | LINK_LABEL, |
| 63 | (_link, open: string, label: string, close: string) => |
| 64 | open + label.replace(/\\([*_])/g, "$1") + close, |
| 65 | ); |
| 66 | return readable === text ? null : readable; |
| 67 | }; |
| 68 | |
| 69 | /** A message's text as it reads, for thread search. */ |
| 70 | export const messageSearchText = (sql: SqlClient.SqlClient) => |
| 71 | sql.literal(readableText("messages")); |
| 72 | |
| 73 | /** What the index stores for a message, so a scan matches the folded query as the index does. */ |
| 74 | export const foldedMessageSearchText = (sql: SqlClient.SqlClient, row = "messages") => |
| 75 | sql.literal(foldApostrophes(readableText(row))); |
| 76 | |
| 77 | /** |
| 78 | * A message's stored text with apostrophes folded, what the index held before |
| 79 | * migration 81. Migration 69's backfill uses it. `payload` is the message's |
| 80 | * JSON column. |
| 81 | */ |
| 82 | export const indexedMessageText = (sql: SqlClient.SqlClient, payload: string) => |
| 83 | sql.literal(foldApostrophes(`json_extract(${payload}, '$.text')`)); |
| 84 | |
| 85 | // What the index stores: a message's readable text with apostrophes folded. |
| 86 | // Migrations 69 and 74 create the triggers before migration 81 adds |
| 87 | // search_text, and `t3 doctor` can rebuild a database that has not migrated, |
| 88 | // so those fall back to the stored text. |
| 89 | const indexedText = Effect.fn("MessageSearchIndex.indexedText")(function* ( |
| 90 | sql: SqlClient.SqlClient, |
| 91 | ) { |
| 92 | const column = yield* sql` |
| 93 | SELECT 1 FROM pragma_table_info('orchestration_v2_projection_messages') |
| 94 | WHERE name = 'search_text' |
| 95 | `; |
| 96 | return column.length > 0 |
| 97 | ? (row: string) => foldedMessageSearchText(sql, row) |
| 98 | : (row: string) => indexedMessageText(sql, `${row}.payload_json`); |
| 99 | }); |
| 100 | |
| 101 | export const createMessageSearchTriggers = Effect.fn("MessageSearchIndex.createTriggers")( |
| 102 | function* (sql: SqlClient.SqlClient) { |
| 103 | const text = yield* indexedText(sql); |
| 104 | yield* sql` |
| 105 | CREATE TRIGGER orchestration_v2_message_search_insert |
| 106 | AFTER INSERT ON orchestration_v2_projection_messages |
| 107 | WHEN ${indexedMessageCondition(sql, "new")} |
| 108 | BEGIN |
| 109 | INSERT INTO orchestration_v2_message_search (rowid, text) |
| 110 | SELECT new.rowid, ${text("new")} |
| 111 | WHERE json_extract(new.payload_json, '$.text') IS NOT NULL; |
| 112 | END |
| 113 | `; |
| 114 | // Streaming deltas rewrite the row many times a second; skip them until the |
| 115 | // message settles. Upserts rewrite every column, so re-index only when what |
| 116 | // the index stores changed; migration 81's backfill writes only search_text. |
| 117 | // Deleting a rowid the index never held is a no-op. |
| 118 | yield* sql` |
| 119 | CREATE TRIGGER orchestration_v2_message_search_update |
| 120 | AFTER UPDATE ON orchestration_v2_projection_messages |
| 121 | WHEN (old.streaming = 0 OR new.streaming = 0) |
| 122 | AND ( |
| 123 | old.streaming IS NOT new.streaming |
| 124 | OR old.role IS NOT new.role |
| 125 | OR ${text("old")} IS NOT ${text("new")} |
| 126 | OR json_extract(old.payload_json, '$.createdBy') |
| 127 | IS NOT json_extract(new.payload_json, '$.createdBy') |
| 128 | ) |
| 129 | BEGIN |
| 130 | DELETE FROM orchestration_v2_message_search WHERE rowid = old.rowid AND old.streaming = 0; |
| 131 | INSERT INTO orchestration_v2_message_search (rowid, text) |
| 132 | SELECT new.rowid, ${text("new")} |
| 133 | WHERE ${indexedMessageCondition(sql, "new")} |
| 134 | AND json_extract(new.payload_json, '$.text') IS NOT NULL; |
| 135 | END |
| 136 | `; |
| 137 | yield* sql` |
| 138 | CREATE TRIGGER orchestration_v2_message_search_delete |
| 139 | AFTER DELETE ON orchestration_v2_projection_messages |
| 140 | WHEN old.streaming = 0 |
| 141 | BEGIN |
| 142 | DELETE FROM orchestration_v2_message_search WHERE rowid = old.rowid; |
| 143 | END |
| 144 | `; |
| 145 | }, |
| 146 | ); |
| 147 | |
| 148 | /** Replaces the search triggers with their current definition, where the index exists. */ |
| 149 | export const recreateMessageSearchTriggers = Effect.fn("MessageSearchIndex.recreateTriggers")( |
| 150 | function* (sql: SqlClient.SqlClient) { |
| 151 | const index = yield* sql` |
| 152 | SELECT 1 FROM sqlite_master WHERE name = 'orchestration_v2_message_search' |
| 153 | `; |
| 154 | if (index.length === 0) return; |
| 155 | yield* sql`DROP TRIGGER orchestration_v2_message_search_insert`; |
| 156 | yield* sql`DROP TRIGGER orchestration_v2_message_search_update`; |
| 157 | yield* sql`DROP TRIGGER orchestration_v2_message_search_delete`; |
| 158 | yield* createMessageSearchTriggers(sql); |
| 159 | }, |
| 160 | ); |
| 161 | |
| 162 | /** |
| 163 | * Refills the search index from the messages table. The index is keyed by |
| 164 | * message rowid, so anything that may renumber or copy rows (VACUUM, a dev |
| 165 | * database copy) rebuilds it instead of trusting it. Databases from before |
| 166 | * migration 69, or on an SQLite without FTS5 trigrams, have no index. |
| 167 | */ |
| 168 | export const rebuildMessageSearch = Effect.fn("MessageSearchIndex.rebuild")(function* ( |
| 169 | sql: SqlClient.SqlClient, |
| 170 | ) { |
| 171 | const index = yield* sql` |
| 172 | SELECT 1 FROM sqlite_master WHERE name = 'orchestration_v2_message_search' |
| 173 | `; |
| 174 | if (index.length === 0) return; |
| 175 | const text = yield* indexedText(sql); |
| 176 | // One transaction: an interrupt between clear and refill would leave an |
| 177 | // empty index that nothing repairs, since search only checks the table exists. |
| 178 | yield* sql.withTransaction( |
| 179 | Effect.gen(function* () { |
| 180 | yield* sql` |
| 181 | INSERT INTO orchestration_v2_message_search (orchestration_v2_message_search) |
| 182 | VALUES ('delete-all') |
| 183 | `; |
| 184 | yield* sql` |
| 185 | INSERT INTO orchestration_v2_message_search (rowid, text) |
| 186 | SELECT messages.rowid, ${text("messages")} |
| 187 | FROM orchestration_v2_projection_messages AS messages |
| 188 | WHERE ${indexedMessageCondition(sql)} |
| 189 | AND json_extract(messages.payload_json, '$.text') IS NOT NULL |
| 190 | `; |
| 191 | }), |
| 192 | ); |
| 193 | }); |