@devries-cameron20 / t3code Mirror · read-only / commits / a536ff7ac0
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 23:30 UTC

fix(server): thread search finds phrases inside quotes and their comments (#22553)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Julius Marminge <julius0216@outlook.com>

Victor committed Nov 11, 2026 at 02:08 UTC a536ff7ac0c080190f098965d0a0d244caf5b211
8 files changed +332 -20
apps/server/src/orchestration-v2/ProjectionStore.ts
+7 -3
@@ -96,6 +96,7 @@ import type * as Statement from "effect/sql/Statement";
96
97 import { MCP_APP_OUTPUT_KEY } from "@t3tools/shared/mcpApp";
98 import { threadHtmlRenderAttachmentIds } from "../attachmentStore.ts";
99 +import { readableSearchText } from "./messageSearchIndex.ts";
100 import {
101 isThreadHistoryUserTurn,
102 isConversationHistoryItem,
@@ -2598,7 +2599,8 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
2599 streaming,
2600 created_at,
2601 updated_at,
2601 - payload_json
2602 + payload_json,
2603 + search_text
2604 )
2605 VALUES (
2606 ${event.payload.id},
@@ -2609,7 +2611,8 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
2611 ${booleanInt(event.payload.streaming)},
2612 ${createdAt},
2613 ${updatedAt},
2612 - ${payloadJson}
2614 + ${payloadJson},
2615 + ${readableSearchText(event.payload.text)}
2616 )
2617 ON CONFLICT(message_id)
2618 DO UPDATE SET
@@ -2620,7 +2623,8 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
2623 streaming = excluded.streaming,
2624 created_at = excluded.created_at,
2625 updated_at = excluded.updated_at,
2623 - payload_json = excluded.payload_json
2626 + payload_json = excluded.payload_json,
2627 + search_text = excluded.search_text
2628 `;
2629 break;
2630 }
apps/server/src/orchestration-v2/ThreadSearch.test.ts
+107
@@ -1,5 +1,6 @@
1 import { assert, it } from "@effect/vitest";
2 import {
3 + EnvironmentId,
4 EventId,
5 MessageId,
6 ProjectId,
@@ -18,9 +19,11 @@ import * as Fiber from "effect/Fiber";
19 import * as Layer from "effect/Layer";
20 import * as Path from "effect/Path";
21 import * as SqlClient from "effect/sql/SqlClient";
22 +import { serializeAssistantCitation } from "@t3tools/shared/assistantCitations";
23 import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "@t3tools/shared/composerContextReferences";
24
25 import * as SqlitePersistence from "../persistence/Sqlite.ts";
26 +import { rebuildMessageSearch } from "./messageSearchIndex.ts";
27 import * as ProjectionStore from "./ProjectionStore.ts";
28 import * as ProjectStore from "./ProjectStore.ts";
29 import * as ThreadSearch from "./ThreadSearch.ts";
@@ -150,6 +153,20 @@ const message = (
153 },
154 });
155
156 +// The prefix and suffix are hidden selection context, not text the reader sees.
157 +const quote = serializeAssistantCitation({
158 + version: 1,
159 + environmentId: EnvironmentId.make("environment:local"),
160 + threadId: ThreadId.make("thread:quoted"),
161 + messageId: MessageId.make("quoted-answer"),
162 + text: "A lighthouse guides ships safely through the darkness.",
163 + comment: "what are you talking about",
164 + start: 0,
165 + end: 54,
166 + prefix: "beacon01",
167 + suffix: "harbor",
168 +});
169 +
170 it.effect.each([true, false])(
171 "ThreadSearch excludes bootstrap text before the hit and candidate limits (indexed: %s)",
172 (indexed) =>
@@ -349,6 +366,96 @@ it.layer(layerTest)("ThreadSearch", (it) => {
366 }),
367 );
368
369 + it.effect("matches the readable text of quotes and their comments", () =>
370 + Effect.gen(function* () {
371 + const projections = yield* ProjectionStore.ProjectionStoreV2;
372 + const search = yield* ThreadSearch.ThreadSearch;
373 + const project = ProjectId.make("project:search-citation");
374 + const threadId = ThreadId.make("thread:citation");
375 + yield* createProject(project);
376 + yield* Effect.forEach(
377 + [
378 + thread(threadId, project),
379 + message(threadId, "citation", "user", `${quote} quiet silver lantern`),
380 + ],
381 + projections.apply,
382 + { discard: true },
383 + );
384 + const snippets = (query: string) =>
385 + Effect.map(search.search({ query }), (result) =>
386 + result.matches
387 + .filter((match) => match.threadId === threadId)
388 + .map((match) => match.snippet),
389 + );
390 +
391 + const readable =
392 + "A lighthouse guides ships safely through the darkness. Comment: what are you talking about quiet silver lantern";
393 + assert.deepEqual(yield* snippets("what are you talking about"), [
394 + `…${readable.slice(readable.indexOf("the darkness"))}`,
395 + ]);
396 + assert.deepEqual(yield* snippets("ships safely through"), [readable]);
397 + assert.deepEqual(yield* snippets("beacon01"), []);
398 + assert.deepEqual(yield* snippets("harbor"), []);
399 +
400 + yield* projections.apply(message(threadId, "citation", "user", "quiet silver lantern"));
401 + assert.deepEqual(yield* snippets("what are you talking about"), []);
402 + assert.deepEqual(yield* snippets("lantern"), ["quiet silver lantern"]);
403 + }),
404 + );
405 +
406 + it.effect("finds accented, CJK, and apostrophe words of a quote through the index", () =>
407 + Effect.gen(function* () {
408 + const sql = yield* SqlClient.SqlClient;
409 + const projections = yield* ProjectionStore.ProjectionStoreV2;
410 + const search = yield* ThreadSearch.ThreadSearch;
411 + const project = ProjectId.make("project:search-citation-index");
412 + const threadId = ThreadId.make("thread:citation-index");
413 + // The stored link percent-encodes these words; the index must hold them as they read.
414 + const encodedQuote = serializeAssistantCitation({
415 + version: 1,
416 + environmentId: EnvironmentId.make("environment:local"),
417 + threadId: ThreadId.make("thread:quoted"),
418 + messageId: MessageId.make("quoted-answer"),
419 + text: "The kittiwake's résumé mentions 灯台の光 twice.",
420 + comment: "naïve question в логах",
421 + start: 0,
422 + end: 44,
423 + prefix: "",
424 + suffix: "",
425 + });
426 + yield* createProject(project);
427 + yield* Effect.forEach(
428 + [thread(threadId, project), message(threadId, "citation-index", "user", encodedQuote)],
429 + projections.apply,
430 + { discard: true },
431 + );
432 + const index = yield* sql`
433 + SELECT 1 FROM sqlite_master WHERE name = 'orchestration_v2_message_search'
434 + `;
435 + assert.lengthOf(index, 1);
436 + const threadIds = (query: string) =>
437 + search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId)));
438 +
439 + for (const query of [
440 + "kittiwake's",
441 + "kittiwake’s",
442 + "résumé",
443 + "灯台の光",
444 + "naïve",
445 + "логах в",
446 + ]) {
447 + assert.deepEqual(yield* threadIds(query), [threadId], query);
448 + }
449 + // `t3 doctor` rebuilds the index after VACUUM; quotes must still read.
450 + yield* rebuildMessageSearch(sql);
451 + assert.deepEqual(yield* threadIds("résumé"), [threadId]);
452 +
453 + yield* projections.apply(message(threadId, "citation-index", "user", "plain words"));
454 + assert.deepEqual(yield* threadIds("résumé"), []);
455 + assert.deepEqual(yield* threadIds("plain"), [threadId]);
456 + }),
457 + );
458 +
459 it.effect("indexes a message when it finishes streaming and follows later edits", () =>
460 Effect.gen(function* () {
461 const projections = yield* ProjectionStore.ProjectionStoreV2;
apps/server/src/orchestration-v2/ThreadSearch.ts
+9 -5
@@ -33,7 +33,11 @@ import {
33 type ThreadSearchSubject,
34 } from "@t3tools/shared/paletteSearch";
35 import { threadSearchPullRequests } from "@t3tools/shared/threadPullRequests";
36 -import { indexedMessageCondition, nonPlaceholderMessageCondition } from "./messageSearchIndex.ts";
36 +import {
37 + indexedMessageCondition,
38 + messageSearchText,
39 + nonPlaceholderMessageCondition,
40 +} from "./messageSearchIndex.ts";
41 import { loadOlderSettledThreadsPage } from "./ShellStream.ts";
42
43 // When the user last wrote in the thread aliased `threads`, as the clients
@@ -203,7 +207,7 @@ export const messageHitsQuery = (
207 messages.thread_id AS "threadId",
208 eligible.project_id AS "projectId",
209 messages.role AS source,
206 - json_extract(messages.payload_json, '$.text') AS text,
210 + ${messageSearchText(sql)} AS text,
211 messages.created_at AS "messageCreatedAt"
212 FROM orchestration_v2_projection_messages AS messages
213 INNER JOIN eligible ON eligible.thread_id = messages.thread_id
@@ -220,7 +224,7 @@ export const messageHitsQuery = (
224 messages.thread_id AS "threadId",
225 eligible.project_id AS "projectId",
226 messages.role AS source,
223 - json_extract(messages.payload_json, '$.text') AS text,
227 + ${messageSearchText(sql)} AS text,
228 messages.created_at AS "messageCreatedAt"
229 FROM orchestration_v2_message_search
230 INNER JOIN orchestration_v2_projection_messages AS messages
@@ -241,13 +245,13 @@ export const messageHitsQuery = (
245 AND messages.message_id NOT LIKE 'async-answer:%'
246 AND NOT EXISTS (
247 SELECT 1 FROM json_each(${substrings}) AS token
244 - WHERE instr(lower(json_extract(messages.payload_json, '$.text')), token.value) = 0
248 + WHERE instr(lower(${messageSearchText(sql)}), token.value) = 0
249 )
250 AND NOT EXISTS (
251 SELECT 1 FROM json_each(${shortSpellings}) AS token
252 WHERE NOT EXISTS (
253 SELECT 1 FROM json_each(token.value) AS spelling
250 - WHERE instr(json_extract(messages.payload_json, '$.text'), spelling.value) > 0
254 + WHERE instr(${messageSearchText(sql)}, spelling.value) > 0
255 )
256 )
257 -- The index's own rowid, so FTS5 returns hits newest first and stops at
apps/server/src/orchestration-v2/messageSearchIndex.ts
+65 -12
@@ -1,3 +1,4 @@
1 +import { assistantCitationsToPlainText } from "@t3tools/shared/assistantCitations";
2 import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "@t3tools/shared/composerContextReferences";
3 import * as Effect from "effect/Effect";
4 import type * as SqlClient from "effect/sql/SqlClient";
@@ -40,46 +41,83 @@ export const indexedMessageCondition = (sql: SqlClient.SqlClient, row = "message
41 AND ${nonPlaceholderMessageCondition(sql, row)}
42 `;
43
44 +// Curly apostrophes straight, as search folds a query, since iOS keyboards
45 +// type "’" and others type "'".
46 +const foldApostrophes = (text: string) => `replace(replace(${text}, '’', ''''), '‘', '''')`;
47 +
48 +const readableText = (row: string) =>
49 + `COALESCE(${row}.search_text, json_extract(${row}.payload_json, '$.text'))`;
50 +
51 +/**
52 + * What search_text holds for a message: its text as it reads, a quote's
53 + * percent-encoded link replaced by its words and comment. NULL when that is
54 + * the stored text, so most messages store nothing extra.
55 + */
56 +export const readableSearchText = (text: string) => {
57 + const readable = assistantCitationsToPlainText(text);
58 + return readable === text ? null : readable;
59 +};
60 +
61 +/** A message's text as it reads, for thread search. */
62 +export const messageSearchText = (sql: SqlClient.SqlClient) =>
63 + sql.literal(readableText("messages"));
64 +
65 /**
44 - * What the index stores for a message: its text with curly apostrophes
45 - * straight, as search folds a query, since iOS keyboards type "’" and others
46 - * type "'". Migration 69's triggers use it too. `payload` is the message's
66 + * A message's stored text with apostrophes folded, what the index held before
67 + * migration 81. Migration 69's backfill uses it. `payload` is the message's
68 * JSON column.
69 */
70 export const indexedMessageText = (sql: SqlClient.SqlClient, payload: string) =>
50 - sql.literal(`replace(replace(json_extract(${payload}, '$.text'), '’', ''''), '‘', '''')`);
71 + sql.literal(foldApostrophes(`json_extract(${payload}, '$.text')`));
72 +
73 +// What the index stores: a message's readable text with apostrophes folded.
74 +// Migrations 69 and 74 create the triggers before migration 81 adds
75 +// search_text, and `t3 doctor` can rebuild a database that has not migrated,
76 +// so those fall back to the stored text.
77 +const indexedText = Effect.fn("MessageSearchIndex.indexedText")(function* (
78 + sql: SqlClient.SqlClient,
79 +) {
80 + const column = yield* sql`
81 + SELECT 1 FROM pragma_table_info('orchestration_v2_projection_messages')
82 + WHERE name = 'search_text'
83 + `;
84 + return column.length > 0
85 + ? (row: string) => sql.literal(foldApostrophes(readableText(row)))
86 + : (row: string) => indexedMessageText(sql, `${row}.payload_json`);
87 +});
88
89 export const createMessageSearchTriggers = Effect.fn("MessageSearchIndex.createTriggers")(
90 function* (sql: SqlClient.SqlClient) {
91 + const text = yield* indexedText(sql);
92 yield* sql`
93 CREATE TRIGGER orchestration_v2_message_search_insert
94 AFTER INSERT ON orchestration_v2_projection_messages
95 WHEN ${indexedMessageCondition(sql, "new")}
96 BEGIN
97 INSERT INTO orchestration_v2_message_search (rowid, text)
60 - SELECT new.rowid, ${indexedMessageText(sql, "new.payload_json")}
98 + SELECT new.rowid, ${text("new")}
99 WHERE json_extract(new.payload_json, '$.text') IS NOT NULL;
100 END
101 `;
102 // Streaming deltas rewrite the row many times a second; skip them until the
65 - // message settles. Upserts rewrite every column, so a finished message
66 - // re-indexes only when what the index reads changed. Deleting a rowid the
67 - // index never held is a no-op.
103 + // message settles. Upserts rewrite every column, so re-index only when what
104 + // the index stores changed; migration 81's backfill writes only search_text.
105 + // Deleting a rowid the index never held is a no-op.
106 yield* sql`
107 CREATE TRIGGER orchestration_v2_message_search_update
70 - AFTER UPDATE OF payload_json, streaming, role ON orchestration_v2_projection_messages
108 + AFTER UPDATE ON orchestration_v2_projection_messages
109 WHEN (old.streaming = 0 OR new.streaming = 0)
110 AND (
111 old.streaming IS NOT new.streaming
112 OR old.role IS NOT new.role
75 - OR json_extract(old.payload_json, '$.text') IS NOT json_extract(new.payload_json, '$.text')
113 + OR ${text("old")} IS NOT ${text("new")}
114 OR json_extract(old.payload_json, '$.createdBy')
115 IS NOT json_extract(new.payload_json, '$.createdBy')
116 )
117 BEGIN
118 DELETE FROM orchestration_v2_message_search WHERE rowid = old.rowid AND old.streaming = 0;
119 INSERT INTO orchestration_v2_message_search (rowid, text)
82 - SELECT new.rowid, ${indexedMessageText(sql, "new.payload_json")}
120 + SELECT new.rowid, ${text("new")}
121 WHERE ${indexedMessageCondition(sql, "new")}
122 AND json_extract(new.payload_json, '$.text') IS NOT NULL;
123 END
@@ -95,6 +133,20 @@ export const createMessageSearchTriggers = Effect.fn("MessageSearchIndex.createT
133 },
134 );
135
136 +/** Replaces the search triggers with their current definition, where the index exists. */
137 +export const recreateMessageSearchTriggers = Effect.fn("MessageSearchIndex.recreateTriggers")(
138 + function* (sql: SqlClient.SqlClient) {
139 + const index = yield* sql`
140 + SELECT 1 FROM sqlite_master WHERE name = 'orchestration_v2_message_search'
141 + `;
142 + if (index.length === 0) return;
143 + yield* sql`DROP TRIGGER orchestration_v2_message_search_insert`;
144 + yield* sql`DROP TRIGGER orchestration_v2_message_search_update`;
145 + yield* sql`DROP TRIGGER orchestration_v2_message_search_delete`;
146 + yield* createMessageSearchTriggers(sql);
147 + },
148 +);
149 +
150 /**
151 * Refills the search index from the messages table. The index is keyed by
152 * message rowid, so anything that may renumber or copy rows (VACUUM, a dev
@@ -108,6 +160,7 @@ export const rebuildMessageSearch = Effect.fn("MessageSearchIndex.rebuild")(func
160 SELECT 1 FROM sqlite_master WHERE name = 'orchestration_v2_message_search'
161 `;
162 if (index.length === 0) return;
163 + const text = yield* indexedText(sql);
164 // One transaction: an interrupt between clear and refill would leave an
165 // empty index that nothing repairs, since search only checks the table exists.
166 yield* sql.withTransaction(
@@ -118,7 +171,7 @@ export const rebuildMessageSearch = Effect.fn("MessageSearchIndex.rebuild")(func
171 `;
172 yield* sql`
173 INSERT INTO orchestration_v2_message_search (rowid, text)
121 - SELECT messages.rowid, ${indexedMessageText(sql, "messages.payload_json")}
174 + SELECT messages.rowid, ${text("messages")}
175 FROM orchestration_v2_projection_messages AS messages
176 WHERE ${indexedMessageCondition(sql)}
177 AND json_extract(messages.payload_json, '$.text') IS NOT NULL
apps/server/src/persistence/Migrations.ts
+2
@@ -83,6 +83,7 @@ import Migration0077 from "./Migrations/077_RepairBooleanSettledOverride.ts";
83 import Migration0078 from "./Migrations/078_DropOrchestrationEventCorrelationIndex.ts";
84 import Migration0079 from "./Migrations/079_DelegatedRecoveryIndexes.ts";
85 import Migration0080 from "./Migrations/080_ScheduledTaskThreadMode.ts";
86 +import Migration0081 from "./Migrations/081_ProjectionMessageSearchText.ts";
87
88 const migrationEntries = [
89 [1, "OrchestrationEvents", Migration0001],
@@ -166,6 +167,7 @@ const migrationEntries = [
167 [78, "DropOrchestrationEventCorrelationIndex", Migration0078],
168 [79, "DelegatedRecoveryIndexes", Migration0079],
169 [80, "ScheduledTaskThreadMode", Migration0080],
170 + [81, "ProjectionMessageSearchText", Migration0081],
171 ] as const;
172
173 export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const);
apps/server/src/persistence/Migrations/081_ProjectionMessageSearchText.test.ts new
+96
@@ -0,0 +1,96 @@
1 +import { assert, it } from "@effect/vitest";
2 +import { EnvironmentId, MessageId, ThreadId } from "@t3tools/contracts";
3 +import { serializeAssistantCitation } from "@t3tools/shared/assistantCitations";
4 +import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient";
5 +import * as Effect from "effect/Effect";
6 +import * as SqlClient from "effect/sql/SqlClient";
7 +
8 +import { runMigrations } from "../Migrations.ts";
9 +
10 +const createdAt = "2026-10-01T00:00:00.000Z";
11 +const quote = serializeAssistantCitation({
12 + version: 1,
13 + environmentId: EnvironmentId.make("environment:local"),
14 + threadId: ThreadId.make("thread:quoted"),
15 + messageId: MessageId.make("quoted-answer"),
16 + text: "A lighthouse guides ships safely through the darkness.",
17 + comment: "what are you talking about",
18 + start: 0,
19 + end: 54,
20 + prefix: "beacon01",
21 + suffix: "harbor",
22 +});
23 +const readable =
24 + "A lighthouse guides ships safely through the darkness.\nComment: what are you talking about";
25 +const indexed = (sql: SqlClient.SqlClient, phrase: string) =>
26 + sql<{ readonly message_id: string }>`
27 + SELECT messages.message_id
28 + FROM orchestration_v2_message_search AS search
29 + INNER JOIN orchestration_v2_projection_messages AS messages
30 + ON messages.rowid = search.rowid
31 + WHERE orchestration_v2_message_search MATCH ${phrase}
32 + `.pipe(Effect.map((rows) => rows.map((row) => row.message_id)));
33 +
34 +it.layer(NodeSqliteClient.layer({ filename: ":memory:" }))(
35 + "081_ProjectionMessageSearchText",
36 + (it) => {
37 + it.effect("backfills readable text only for messages whose citations read differently", () =>
38 + Effect.gen(function* () {
39 + const sql = yield* SqlClient.SqlClient;
40 + yield* runMigrations({ toMigrationInclusive: 80 });
41 + const insertMessage = (messageId: string, text: unknown) => sql`
42 + INSERT INTO orchestration_v2_projection_messages (
43 + message_id, thread_id, role, streaming, created_at, updated_at, payload_json
44 + ) VALUES (
45 + ${messageId}, 'thread:quote', 'user', 0, ${createdAt}, ${createdAt},
46 + ${JSON.stringify({ text })}
47 + )
48 + `;
49 + yield* insertMessage("quoted", `${quote} quiet silver lantern`);
50 + yield* insertMessage("plain", "quiet silver lantern");
51 + yield* insertMessage("invalid", "[Assistant quote](t3-citation://v1/not-a-citation)");
52 + yield* insertMessage("unreadable", 20261001);
53 + yield* insertMessage(
54 + "accented",
55 + serializeAssistantCitation({
56 + version: 1,
57 + environmentId: EnvironmentId.make("environment:local"),
58 + threadId: ThreadId.make("thread:quoted"),
59 + messageId: MessageId.make("quoted-answer"),
60 + text: "The kittiwake's résumé",
61 + start: 0,
62 + end: 22,
63 + prefix: "",
64 + suffix: "",
65 + }),
66 + );
67 + yield* Effect.forEach(
68 + Array.from({ length: 501 }, (_, index) => `page-${String(index).padStart(3, "0")}`),
69 + (messageId) => insertMessage(messageId, quote),
70 + { discard: true },
71 + );
72 +
73 + assert.deepEqual(yield* runMigrations({ toMigrationInclusive: 81 }), [
74 + [81, "ProjectionMessageSearchText"],
75 + ]);
76 + const rows = yield* sql<{
77 + readonly message_id: string;
78 + readonly search_text: string | null;
79 + }>`SELECT message_id, search_text FROM orchestration_v2_projection_messages`;
80 + const searchText = new Map(rows.map((row) => [row.message_id, row.search_text]));
81 + assert.equal(searchText.get("quoted"), `${readable} quiet silver lantern`);
82 + assert.isNull(searchText.get("plain"));
83 + assert.isNull(searchText.get("invalid"));
84 + assert.isNull(searchText.get("unreadable"));
85 + assert.equal(searchText.get("page-000"), readable);
86 + assert.equal(searchText.get("page-500"), readable);
87 +
88 + // Messages indexed as stored are indexed again as they read.
89 + assert.deepEqual(yield* indexed(sql, '"résumé"'), ["accented"]);
90 + assert.deepEqual(yield* indexed(sql, `"kittiwake's"`), ["accented"]);
91 + assert.deepEqual(yield* indexed(sql, '"beacon01"'), []);
92 + assert.lengthOf(yield* indexed(sql, '"what are you talking about"'), 502);
93 + }),
94 + );
95 + },
96 +);
apps/server/src/persistence/Migrations/081_ProjectionMessageSearchText.ts new
+44
@@ -0,0 +1,44 @@
1 +import * as Effect from "effect/Effect";
2 +import * as SqlClient from "effect/sql/SqlClient";
3 +
4 +import {
5 + readableSearchText,
6 + recreateMessageSearchTriggers,
7 +} from "../../orchestration-v2/messageSearchIndex.ts";
8 +
9 +export default Effect.gen(function* () {
10 + const sql = yield* SqlClient.SqlClient;
11 + yield* sql`ALTER TABLE orchestration_v2_projection_messages ADD COLUMN search_text TEXT`;
12 +
13 + // Migration 74's triggers index the stored text, where a quote's words are
14 + // percent-encoded. Recreated now that search_text exists, they index it, and
15 + // the backfill below re-indexes every message that reads differently.
16 + yield* recreateMessageSearchTriggers(sql);
17 +
18 + // Only messages holding a citation link read differently. The raw check
19 + // first skips parsing the JSON of every other message.
20 + const pageSize = 500;
21 + let afterMessageId = "";
22 + while (true) {
23 + const rows = yield* sql<{ readonly message_id: string; readonly text: string }>`
24 + SELECT message_id, json_extract(payload_json, '$.text') AS text
25 + FROM orchestration_v2_projection_messages
26 + WHERE message_id > ${afterMessageId}
27 + AND payload_json LIKE '%t3-citation://%'
28 + AND json_type(payload_json, '$.text') = 'text'
29 + ORDER BY message_id
30 + LIMIT ${pageSize}
31 + `;
32 + for (const row of rows) {
33 + const searchText = readableSearchText(row.text);
34 + if (searchText === null) continue;
35 + yield* sql`
36 + UPDATE orchestration_v2_projection_messages
37 + SET search_text = ${searchText}
38 + WHERE message_id = ${row.message_id}
39 + `;
40 + }
41 + if (rows.length < pageSize) break;
42 + afterMessageId = rows.at(-1)!.message_id;
43 + }
44 +});
apps/server/src/persistence/reconcileV2PreviewMigration.test.ts
+2
@@ -60,6 +60,7 @@ describe("V2 preview upgrade", () => {
60 [78, "DropOrchestrationEventCorrelationIndex"],
61 [79, "DelegatedRecoveryIndexes"],
62 [80, "ScheduledTaskThreadMode"],
63 + [81, "ProjectionMessageSearchText"],
64 ]);
65 assert.deepStrictEqual(yield* runMigrations(), []);
66 assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports);
@@ -163,6 +164,7 @@ describe("V2 preview upgrade", () => {
164 [78, "DropOrchestrationEventCorrelationIndex"],
165 [79, "DelegatedRecoveryIndexes"],
166 [80, "ScheduledTaskThreadMode"],
167 + [81, "ProjectionMessageSearchText"],
168 ]);
169 }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))),
170 );