The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 23:30 UTC
fix(server): thread search finds phrases inside quotes and their comments (#22553)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Julius Marminge <julius0216@outlook.com>
Victor committed
Nov 11, 2026 at 02:08 UTC
a536ff7ac0c080190f098965d0a0d244caf5b211
8 files changed
+332
-20
apps/server/src/orchestration-v2/ProjectionStore.ts
+7
-3
@@ -96,6 +96,7 @@ import type * as Statement from "effect/sql/Statement";
96
97
import { MCP_APP_OUTPUT_KEY } from "@t3tools/shared/mcpApp";
98
import { threadHtmlRenderAttachmentIds } from "../attachmentStore.ts";
99
+import { readableSearchText } from "./messageSearchIndex.ts";
100
import {
101
isThreadHistoryUserTurn,
102
isConversationHistoryItem,
@@ -2598,7 +2599,8 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
2599
streaming,
2600
created_at,
2601
updated_at,
2601
- payload_json
2602
+ payload_json,
2603
+ search_text
2604
)
2605
VALUES (
2606
${event.payload.id},
@@ -2609,7 +2611,8 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
2611
${booleanInt(event.payload.streaming)},
2612
${createdAt},
2613
${updatedAt},
2612
- ${payloadJson}
2614
+ ${payloadJson},
2615
+ ${readableSearchText(event.payload.text)}
2616
)
2617
ON CONFLICT(message_id)
2618
DO UPDATE SET
@@ -2620,7 +2623,8 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
2623
streaming = excluded.streaming,
2624
created_at = excluded.created_at,
2625
updated_at = excluded.updated_at,
2623
- payload_json = excluded.payload_json
2626
+ payload_json = excluded.payload_json,
2627
+ search_text = excluded.search_text
2628
`;
2629
break;
2630
}
apps/server/src/orchestration-v2/ThreadSearch.test.ts
+107
@@ -1,5 +1,6 @@
1
import { assert, it } from "@effect/vitest";
2
import {
3
+ EnvironmentId,
4
EventId,
5
MessageId,
6
ProjectId,
@@ -18,9 +19,11 @@ import * as Fiber from "effect/Fiber";
19
import * as Layer from "effect/Layer";
20
import * as Path from "effect/Path";
21
import * as SqlClient from "effect/sql/SqlClient";
22
+import { serializeAssistantCitation } from "@t3tools/shared/assistantCitations";
23
import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "@t3tools/shared/composerContextReferences";
24
25
import * as SqlitePersistence from "../persistence/Sqlite.ts";
26
+import { rebuildMessageSearch } from "./messageSearchIndex.ts";
27
import * as ProjectionStore from "./ProjectionStore.ts";
28
import * as ProjectStore from "./ProjectStore.ts";
29
import * as ThreadSearch from "./ThreadSearch.ts";
@@ -150,6 +153,20 @@ const message = (
153
},
154
});
155
156
+// The prefix and suffix are hidden selection context, not text the reader sees.
157
+const quote = serializeAssistantCitation({
158
+ version: 1,
159
+ environmentId: EnvironmentId.make("environment:local"),
160
+ threadId: ThreadId.make("thread:quoted"),
161
+ messageId: MessageId.make("quoted-answer"),
162
+ text: "A lighthouse guides ships safely through the darkness.",
163
+ comment: "what are you talking about",
164
+ start: 0,
165
+ end: 54,
166
+ prefix: "beacon01",
167
+ suffix: "harbor",
168
+});
169
+
170
it.effect.each([true, false])(
171
"ThreadSearch excludes bootstrap text before the hit and candidate limits (indexed: %s)",
172
(indexed) =>
@@ -349,6 +366,96 @@ it.layer(layerTest)("ThreadSearch", (it) => {
366
}),
367
);
368
369
+ it.effect("matches the readable text of quotes and their comments", () =>
370
+ Effect.gen(function* () {
371
+ const projections = yield* ProjectionStore.ProjectionStoreV2;
372
+ const search = yield* ThreadSearch.ThreadSearch;
373
+ const project = ProjectId.make("project:search-citation");
374
+ const threadId = ThreadId.make("thread:citation");
375
+ yield* createProject(project);
376
+ yield* Effect.forEach(
377
+ [
378
+ thread(threadId, project),
379
+ message(threadId, "citation", "user", `${quote} quiet silver lantern`),
380
+ ],
381
+ projections.apply,
382
+ { discard: true },
383
+ );
384
+ const snippets = (query: string) =>
385
+ Effect.map(search.search({ query }), (result) =>
386
+ result.matches
387
+ .filter((match) => match.threadId === threadId)
388
+ .map((match) => match.snippet),
389
+ );
390
+
391
+ const readable =
392
+ "A lighthouse guides ships safely through the darkness. Comment: what are you talking about quiet silver lantern";
393
+ assert.deepEqual(yield* snippets("what are you talking about"), [
394
+ `…${readable.slice(readable.indexOf("the darkness"))}`,
395
+ ]);
396
+ assert.deepEqual(yield* snippets("ships safely through"), [readable]);
397
+ assert.deepEqual(yield* snippets("beacon01"), []);
398
+ assert.deepEqual(yield* snippets("harbor"), []);
399
+
400
+ yield* projections.apply(message(threadId, "citation", "user", "quiet silver lantern"));
401
+ assert.deepEqual(yield* snippets("what are you talking about"), []);
402
+ assert.deepEqual(yield* snippets("lantern"), ["quiet silver lantern"]);
403
+ }),
404
+ );
405
+
406
+ it.effect("finds accented, CJK, and apostrophe words of a quote through the index", () =>
407
+ Effect.gen(function* () {
408
+ const sql = yield* SqlClient.SqlClient;
409
+ const projections = yield* ProjectionStore.ProjectionStoreV2;
410
+ const search = yield* ThreadSearch.ThreadSearch;
411
+ const project = ProjectId.make("project:search-citation-index");
412
+ const threadId = ThreadId.make("thread:citation-index");
413
+ // The stored link percent-encodes these words; the index must hold them as they read.
414
+ const encodedQuote = serializeAssistantCitation({
415
+ version: 1,
416
+ environmentId: EnvironmentId.make("environment:local"),
417
+ threadId: ThreadId.make("thread:quoted"),
418
+ messageId: MessageId.make("quoted-answer"),
419
+ text: "The kittiwake's résumé mentions 灯台の光 twice.",
420
+ comment: "naïve question в логах",
421
+ start: 0,
422
+ end: 44,
423
+ prefix: "",
424
+ suffix: "",
425
+ });
426
+ yield* createProject(project);
427
+ yield* Effect.forEach(
428
+ [thread(threadId, project), message(threadId, "citation-index", "user", encodedQuote)],
429
+ projections.apply,
430
+ { discard: true },
431
+ );
432
+ const index = yield* sql`
433
+ SELECT 1 FROM sqlite_master WHERE name = 'orchestration_v2_message_search'
434
+ `;
435
+ assert.lengthOf(index, 1);
436
+ const threadIds = (query: string) =>
437
+ search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId)));
438
+
439
+ for (const query of [
440
+ "kittiwake's",
441
+ "kittiwake’s",
442
+ "résumé",
443
+ "灯台の光",
444
+ "naïve",
445
+ "логах в",
446
+ ]) {
447
+ assert.deepEqual(yield* threadIds(query), [threadId], query);
448
+ }
449
+ // `t3 doctor` rebuilds the index after VACUUM; quotes must still read.
450
+ yield* rebuildMessageSearch(sql);
451
+ assert.deepEqual(yield* threadIds("résumé"), [threadId]);
452
+
453
+ yield* projections.apply(message(threadId, "citation-index", "user", "plain words"));
454
+ assert.deepEqual(yield* threadIds("résumé"), []);
455
+ assert.deepEqual(yield* threadIds("plain"), [threadId]);
456
+ }),
457
+ );
458
+
459
it.effect("indexes a message when it finishes streaming and follows later edits", () =>
460
Effect.gen(function* () {
461
const projections = yield* ProjectionStore.ProjectionStoreV2;
apps/server/src/orchestration-v2/ThreadSearch.ts
+9
-5
@@ -33,7 +33,11 @@ import {
33
type ThreadSearchSubject,
34
} from "@t3tools/shared/paletteSearch";
35
import { threadSearchPullRequests } from "@t3tools/shared/threadPullRequests";
36
-import { indexedMessageCondition, nonPlaceholderMessageCondition } from "./messageSearchIndex.ts";
36
+import {
37
+ indexedMessageCondition,
38
+ messageSearchText,
39
+ nonPlaceholderMessageCondition,
40
+} from "./messageSearchIndex.ts";
41
import { loadOlderSettledThreadsPage } from "./ShellStream.ts";
42
43
// When the user last wrote in the thread aliased `threads`, as the clients
@@ -203,7 +207,7 @@ export const messageHitsQuery = (
207
messages.thread_id AS "threadId",
208
eligible.project_id AS "projectId",
209
messages.role AS source,
206
- json_extract(messages.payload_json, '$.text') AS text,
210
+ ${messageSearchText(sql)} AS text,
211
messages.created_at AS "messageCreatedAt"
212
FROM orchestration_v2_projection_messages AS messages
213
INNER JOIN eligible ON eligible.thread_id = messages.thread_id
@@ -220,7 +224,7 @@ export const messageHitsQuery = (
224
messages.thread_id AS "threadId",
225
eligible.project_id AS "projectId",
226
messages.role AS source,
223
- json_extract(messages.payload_json, '$.text') AS text,
227
+ ${messageSearchText(sql)} AS text,
228
messages.created_at AS "messageCreatedAt"
229
FROM orchestration_v2_message_search
230
INNER JOIN orchestration_v2_projection_messages AS messages
@@ -241,13 +245,13 @@ export const messageHitsQuery = (
245
AND messages.message_id NOT LIKE 'async-answer:%'
246
AND NOT EXISTS (
247
SELECT 1 FROM json_each(${substrings}) AS token
244
- WHERE instr(lower(json_extract(messages.payload_json, '$.text')), token.value) = 0
248
+ WHERE instr(lower(${messageSearchText(sql)}), token.value) = 0
249
)
250
AND NOT EXISTS (
251
SELECT 1 FROM json_each(${shortSpellings}) AS token
252
WHERE NOT EXISTS (
253
SELECT 1 FROM json_each(token.value) AS spelling
250
- WHERE instr(json_extract(messages.payload_json, '$.text'), spelling.value) > 0
254
+ WHERE instr(${messageSearchText(sql)}, spelling.value) > 0
255
)
256
)
257
-- The index's own rowid, so FTS5 returns hits newest first and stops at
apps/server/src/orchestration-v2/messageSearchIndex.ts
+65
-12
@@ -1,3 +1,4 @@
1
+import { assistantCitationsToPlainText } from "@t3tools/shared/assistantCitations";
2
import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "@t3tools/shared/composerContextReferences";
3
import * as Effect from "effect/Effect";
4
import type * as SqlClient from "effect/sql/SqlClient";
@@ -40,46 +41,83 @@ export const indexedMessageCondition = (sql: SqlClient.SqlClient, row = "message
41
AND ${nonPlaceholderMessageCondition(sql, row)}
42
`;
43
44
+// Curly apostrophes straight, as search folds a query, since iOS keyboards
45
+// type "’" and others type "'".
46
+const foldApostrophes = (text: string) => `replace(replace(${text}, '’', ''''), '‘', '''')`;
47
+
48
+const readableText = (row: string) =>
49
+ `COALESCE(${row}.search_text, json_extract(${row}.payload_json, '$.text'))`;
50
+
51
+/**
52
+ * What search_text holds for a message: its text as it reads, a quote's
53
+ * percent-encoded link replaced by its words and comment. NULL when that is
54
+ * the stored text, so most messages store nothing extra.
55
+ */
56
+export const readableSearchText = (text: string) => {
57
+ const readable = assistantCitationsToPlainText(text);
58
+ return readable === text ? null : readable;
59
+};
60
+
61
+/** A message's text as it reads, for thread search. */
62
+export const messageSearchText = (sql: SqlClient.SqlClient) =>
63
+ sql.literal(readableText("messages"));
64
+
65
/**
44
- * What the index stores for a message: its text with curly apostrophes
45
- * straight, as search folds a query, since iOS keyboards type "’" and others
46
- * type "'". Migration 69's triggers use it too. `payload` is the message's
66
+ * A message's stored text with apostrophes folded, what the index held before
67
+ * migration 81. Migration 69's backfill uses it. `payload` is the message's
68
* JSON column.
69
*/
70
export const indexedMessageText = (sql: SqlClient.SqlClient, payload: string) =>
50
- sql.literal(`replace(replace(json_extract(${payload}, '$.text'), '’', ''''), '‘', '''')`);
71
+ sql.literal(foldApostrophes(`json_extract(${payload}, '$.text')`));
72
+
73
+// What the index stores: a message's readable text with apostrophes folded.
74
+// Migrations 69 and 74 create the triggers before migration 81 adds
75
+// search_text, and `t3 doctor` can rebuild a database that has not migrated,
76
+// so those fall back to the stored text.
77
+const indexedText = Effect.fn("MessageSearchIndex.indexedText")(function* (
78
+ sql: SqlClient.SqlClient,
79
+) {
80
+ const column = yield* sql`
81
+ SELECT 1 FROM pragma_table_info('orchestration_v2_projection_messages')
82
+ WHERE name = 'search_text'
83
+ `;
84
+ return column.length > 0
85
+ ? (row: string) => sql.literal(foldApostrophes(readableText(row)))
86
+ : (row: string) => indexedMessageText(sql, `${row}.payload_json`);
87
+});
88
89
export const createMessageSearchTriggers = Effect.fn("MessageSearchIndex.createTriggers")(
90
function* (sql: SqlClient.SqlClient) {
91
+ const text = yield* indexedText(sql);
92
yield* sql`
93
CREATE TRIGGER orchestration_v2_message_search_insert
94
AFTER INSERT ON orchestration_v2_projection_messages
95
WHEN ${indexedMessageCondition(sql, "new")}
96
BEGIN
97
INSERT INTO orchestration_v2_message_search (rowid, text)
60
- SELECT new.rowid, ${indexedMessageText(sql, "new.payload_json")}
98
+ SELECT new.rowid, ${text("new")}
99
WHERE json_extract(new.payload_json, '$.text') IS NOT NULL;
100
END
101
`;
102
// Streaming deltas rewrite the row many times a second; skip them until the
65
- // message settles. Upserts rewrite every column, so a finished message
66
- // re-indexes only when what the index reads changed. Deleting a rowid the
67
- // index never held is a no-op.
103
+ // message settles. Upserts rewrite every column, so re-index only when what
104
+ // the index stores changed; migration 81's backfill writes only search_text.
105
+ // Deleting a rowid the index never held is a no-op.
106
yield* sql`
107
CREATE TRIGGER orchestration_v2_message_search_update
70
- AFTER UPDATE OF payload_json, streaming, role ON orchestration_v2_projection_messages
108
+ AFTER UPDATE ON orchestration_v2_projection_messages
109
WHEN (old.streaming = 0 OR new.streaming = 0)
110
AND (
111
old.streaming IS NOT new.streaming
112
OR old.role IS NOT new.role
75
- OR json_extract(old.payload_json, '$.text') IS NOT json_extract(new.payload_json, '$.text')
113
+ OR ${text("old")} IS NOT ${text("new")}
114
OR json_extract(old.payload_json, '$.createdBy')
115
IS NOT json_extract(new.payload_json, '$.createdBy')
116
)
117
BEGIN
118
DELETE FROM orchestration_v2_message_search WHERE rowid = old.rowid AND old.streaming = 0;
119
INSERT INTO orchestration_v2_message_search (rowid, text)
82
- SELECT new.rowid, ${indexedMessageText(sql, "new.payload_json")}
120
+ SELECT new.rowid, ${text("new")}
121
WHERE ${indexedMessageCondition(sql, "new")}
122
AND json_extract(new.payload_json, '$.text') IS NOT NULL;
123
END
@@ -95,6 +133,20 @@ export const createMessageSearchTriggers = Effect.fn("MessageSearchIndex.createT
133
},
134
);
135
136
+/** Replaces the search triggers with their current definition, where the index exists. */
137
+export const recreateMessageSearchTriggers = Effect.fn("MessageSearchIndex.recreateTriggers")(
138
+ function* (sql: SqlClient.SqlClient) {
139
+ const index = yield* sql`
140
+ SELECT 1 FROM sqlite_master WHERE name = 'orchestration_v2_message_search'
141
+ `;
142
+ if (index.length === 0) return;
143
+ yield* sql`DROP TRIGGER orchestration_v2_message_search_insert`;
144
+ yield* sql`DROP TRIGGER orchestration_v2_message_search_update`;
145
+ yield* sql`DROP TRIGGER orchestration_v2_message_search_delete`;
146
+ yield* createMessageSearchTriggers(sql);
147
+ },
148
+);
149
+
150
/**
151
* Refills the search index from the messages table. The index is keyed by
152
* message rowid, so anything that may renumber or copy rows (VACUUM, a dev
@@ -108,6 +160,7 @@ export const rebuildMessageSearch = Effect.fn("MessageSearchIndex.rebuild")(func
160
SELECT 1 FROM sqlite_master WHERE name = 'orchestration_v2_message_search'
161
`;
162
if (index.length === 0) return;
163
+ const text = yield* indexedText(sql);
164
// One transaction: an interrupt between clear and refill would leave an
165
// empty index that nothing repairs, since search only checks the table exists.
166
yield* sql.withTransaction(
@@ -118,7 +171,7 @@ export const rebuildMessageSearch = Effect.fn("MessageSearchIndex.rebuild")(func
171
`;
172
yield* sql`
173
INSERT INTO orchestration_v2_message_search (rowid, text)
121
- SELECT messages.rowid, ${indexedMessageText(sql, "messages.payload_json")}
174
+ SELECT messages.rowid, ${text("messages")}
175
FROM orchestration_v2_projection_messages AS messages
176
WHERE ${indexedMessageCondition(sql)}
177
AND json_extract(messages.payload_json, '$.text') IS NOT NULL
apps/server/src/persistence/Migrations.ts
+2
@@ -83,6 +83,7 @@ import Migration0077 from "./Migrations/077_RepairBooleanSettledOverride.ts";
83
import Migration0078 from "./Migrations/078_DropOrchestrationEventCorrelationIndex.ts";
84
import Migration0079 from "./Migrations/079_DelegatedRecoveryIndexes.ts";
85
import Migration0080 from "./Migrations/080_ScheduledTaskThreadMode.ts";
86
+import Migration0081 from "./Migrations/081_ProjectionMessageSearchText.ts";
87
88
const migrationEntries = [
89
[1, "OrchestrationEvents", Migration0001],
@@ -166,6 +167,7 @@ const migrationEntries = [
167
[78, "DropOrchestrationEventCorrelationIndex", Migration0078],
168
[79, "DelegatedRecoveryIndexes", Migration0079],
169
[80, "ScheduledTaskThreadMode", Migration0080],
170
+ [81, "ProjectionMessageSearchText", Migration0081],
171
] as const;
172
173
export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const);
apps/server/src/persistence/Migrations/081_ProjectionMessageSearchText.test.ts
new
+96
@@ -0,0 +1,96 @@
1
+import { assert, it } from "@effect/vitest";
2
+import { EnvironmentId, MessageId, ThreadId } from "@t3tools/contracts";
3
+import { serializeAssistantCitation } from "@t3tools/shared/assistantCitations";
4
+import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient";
5
+import * as Effect from "effect/Effect";
6
+import * as SqlClient from "effect/sql/SqlClient";
7
+
8
+import { runMigrations } from "../Migrations.ts";
9
+
10
+const createdAt = "2026-10-01T00:00:00.000Z";
11
+const quote = serializeAssistantCitation({
12
+ version: 1,
13
+ environmentId: EnvironmentId.make("environment:local"),
14
+ threadId: ThreadId.make("thread:quoted"),
15
+ messageId: MessageId.make("quoted-answer"),
16
+ text: "A lighthouse guides ships safely through the darkness.",
17
+ comment: "what are you talking about",
18
+ start: 0,
19
+ end: 54,
20
+ prefix: "beacon01",
21
+ suffix: "harbor",
22
+});
23
+const readable =
24
+ "A lighthouse guides ships safely through the darkness.\nComment: what are you talking about";
25
+const indexed = (sql: SqlClient.SqlClient, phrase: string) =>
26
+ sql<{ readonly message_id: string }>`
27
+ SELECT messages.message_id
28
+ FROM orchestration_v2_message_search AS search
29
+ INNER JOIN orchestration_v2_projection_messages AS messages
30
+ ON messages.rowid = search.rowid
31
+ WHERE orchestration_v2_message_search MATCH ${phrase}
32
+ `.pipe(Effect.map((rows) => rows.map((row) => row.message_id)));
33
+
34
+it.layer(NodeSqliteClient.layer({ filename: ":memory:" }))(
35
+ "081_ProjectionMessageSearchText",
36
+ (it) => {
37
+ it.effect("backfills readable text only for messages whose citations read differently", () =>
38
+ Effect.gen(function* () {
39
+ const sql = yield* SqlClient.SqlClient;
40
+ yield* runMigrations({ toMigrationInclusive: 80 });
41
+ const insertMessage = (messageId: string, text: unknown) => sql`
42
+ INSERT INTO orchestration_v2_projection_messages (
43
+ message_id, thread_id, role, streaming, created_at, updated_at, payload_json
44
+ ) VALUES (
45
+ ${messageId}, 'thread:quote', 'user', 0, ${createdAt}, ${createdAt},
46
+ ${JSON.stringify({ text })}
47
+ )
48
+ `;
49
+ yield* insertMessage("quoted", `${quote} quiet silver lantern`);
50
+ yield* insertMessage("plain", "quiet silver lantern");
51
+ yield* insertMessage("invalid", "[Assistant quote](t3-citation://v1/not-a-citation)");
52
+ yield* insertMessage("unreadable", 20261001);
53
+ yield* insertMessage(
54
+ "accented",
55
+ serializeAssistantCitation({
56
+ version: 1,
57
+ environmentId: EnvironmentId.make("environment:local"),
58
+ threadId: ThreadId.make("thread:quoted"),
59
+ messageId: MessageId.make("quoted-answer"),
60
+ text: "The kittiwake's résumé",
61
+ start: 0,
62
+ end: 22,
63
+ prefix: "",
64
+ suffix: "",
65
+ }),
66
+ );
67
+ yield* Effect.forEach(
68
+ Array.from({ length: 501 }, (_, index) => `page-${String(index).padStart(3, "0")}`),
69
+ (messageId) => insertMessage(messageId, quote),
70
+ { discard: true },
71
+ );
72
+
73
+ assert.deepEqual(yield* runMigrations({ toMigrationInclusive: 81 }), [
74
+ [81, "ProjectionMessageSearchText"],
75
+ ]);
76
+ const rows = yield* sql<{
77
+ readonly message_id: string;
78
+ readonly search_text: string | null;
79
+ }>`SELECT message_id, search_text FROM orchestration_v2_projection_messages`;
80
+ const searchText = new Map(rows.map((row) => [row.message_id, row.search_text]));
81
+ assert.equal(searchText.get("quoted"), `${readable} quiet silver lantern`);
82
+ assert.isNull(searchText.get("plain"));
83
+ assert.isNull(searchText.get("invalid"));
84
+ assert.isNull(searchText.get("unreadable"));
85
+ assert.equal(searchText.get("page-000"), readable);
86
+ assert.equal(searchText.get("page-500"), readable);
87
+
88
+ // Messages indexed as stored are indexed again as they read.
89
+ assert.deepEqual(yield* indexed(sql, '"résumé"'), ["accented"]);
90
+ assert.deepEqual(yield* indexed(sql, `"kittiwake's"`), ["accented"]);
91
+ assert.deepEqual(yield* indexed(sql, '"beacon01"'), []);
92
+ assert.lengthOf(yield* indexed(sql, '"what are you talking about"'), 502);
93
+ }),
94
+ );
95
+ },
96
+);
apps/server/src/persistence/Migrations/081_ProjectionMessageSearchText.ts
new
+44
@@ -0,0 +1,44 @@
1
+import * as Effect from "effect/Effect";
2
+import * as SqlClient from "effect/sql/SqlClient";
3
+
4
+import {
5
+ readableSearchText,
6
+ recreateMessageSearchTriggers,
7
+} from "../../orchestration-v2/messageSearchIndex.ts";
8
+
9
+export default Effect.gen(function* () {
10
+ const sql = yield* SqlClient.SqlClient;
11
+ yield* sql`ALTER TABLE orchestration_v2_projection_messages ADD COLUMN search_text TEXT`;
12
+
13
+ // Migration 74's triggers index the stored text, where a quote's words are
14
+ // percent-encoded. Recreated now that search_text exists, they index it, and
15
+ // the backfill below re-indexes every message that reads differently.
16
+ yield* recreateMessageSearchTriggers(sql);
17
+
18
+ // Only messages holding a citation link read differently. The raw check
19
+ // first skips parsing the JSON of every other message.
20
+ const pageSize = 500;
21
+ let afterMessageId = "";
22
+ while (true) {
23
+ const rows = yield* sql<{ readonly message_id: string; readonly text: string }>`
24
+ SELECT message_id, json_extract(payload_json, '$.text') AS text
25
+ FROM orchestration_v2_projection_messages
26
+ WHERE message_id > ${afterMessageId}
27
+ AND payload_json LIKE '%t3-citation://%'
28
+ AND json_type(payload_json, '$.text') = 'text'
29
+ ORDER BY message_id
30
+ LIMIT ${pageSize}
31
+ `;
32
+ for (const row of rows) {
33
+ const searchText = readableSearchText(row.text);
34
+ if (searchText === null) continue;
35
+ yield* sql`
36
+ UPDATE orchestration_v2_projection_messages
37
+ SET search_text = ${searchText}
38
+ WHERE message_id = ${row.message_id}
39
+ `;
40
+ }
41
+ if (rows.length < pageSize) break;
42
+ afterMessageId = rows.at(-1)!.message_id;
43
+ }
44
+});
apps/server/src/persistence/reconcileV2PreviewMigration.test.ts
+2
@@ -60,6 +60,7 @@ describe("V2 preview upgrade", () => {
60
[78, "DropOrchestrationEventCorrelationIndex"],
61
[79, "DelegatedRecoveryIndexes"],
62
[80, "ScheduledTaskThreadMode"],
63
+ [81, "ProjectionMessageSearchText"],
64
]);
65
assert.deepStrictEqual(yield* runMigrations(), []);
66
assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports);
@@ -163,6 +164,7 @@ describe("V2 preview upgrade", () => {
164
[78, "DropOrchestrationEventCorrelationIndex"],
165
[79, "DelegatedRecoveryIndexes"],
166
[80, "ScheduledTaskThreadMode"],
167
+ [81, "ProjectionMessageSearchText"],
168
]);
169
}).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))),
170
);