@devries-cameron20 / t3code-1 / commits / b800f9919b
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): message search keeps results when a short non-Latin word differs in case (#30332)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

MarcoDotIO committed Nov 5, 2026 at 23:29 UTC b800f9919bcb151ef441f056ff49c301c34018d2
2 files changed +107 -3
apps/server/src/orchestration-v2/ThreadSearch.test.ts
+80 -1
@@ -270,7 +270,7 @@ it.layer(layerTest)("ThreadSearch", (it) => {
270 for (const match of ['"needle"', null]) {
271 const plan = yield* sql<{
272 readonly detail: string;
273 - }>`EXPLAIN QUERY PLAN ${ThreadSearch.messageHitsQuery(sql, { match, substrings: "[]", projectId: null, excludedThreadIds: "[]" })}`;
273 + }>`EXPLAIN QUERY PLAN ${ThreadSearch.messageHitsQuery(sql, { match, substrings: "[]", shortSpellings: "[]", projectId: null, excludedThreadIds: "[]" })}`;
274 assert.isFalse(
275 plan.some((row) => row.detail.includes("TEMP B-TREE")),
276 plan.map((row) => row.detail).join("\n"),
@@ -732,6 +732,85 @@ it.layer(layerTest)("ThreadSearch filters before it ranks", (it) => {
732 );
733 });
734
735 +it.effect.each([true, false])(
736 + "ThreadSearch narrows short words outside ASCII before the hit limit (indexed: %s)",
737 + (indexed) =>
738 + Effect.gen(function* () {
739 + const projections = yield* ProjectionStore.ProjectionStoreV2;
740 + const search = yield* ThreadSearch.ThreadSearch;
741 + const sql = yield* SqlClient.SqlClient;
742 + const project = ProjectId.make("project:short-limit");
743 + const older = ThreadId.make("thread:short-older");
744 + const newer = ThreadId.make("thread:short-newer");
745 + yield* createProject(project);
746 + yield* Effect.forEach(
747 + [
748 + thread(older, project),
749 + message(older, "older", "user", "В логах ошибка"),
750 + thread(newer, project),
751 + ],
752 + projections.apply,
753 + { discard: true },
754 + );
755 + if (!indexed) {
756 + yield* sql`DROP TRIGGER orchestration_v2_message_search_insert`;
757 + yield* sql`DROP TRIGGER orchestration_v2_message_search_update`;
758 + yield* sql`DROP TRIGGER orchestration_v2_message_search_delete`;
759 + yield* sql`DROP TABLE orchestration_v2_message_search`;
760 + }
761 + // More newer hits for the long word than one read holds, none with the short one.
762 + yield* sql`
763 + WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 2000)
764 + INSERT INTO orchestration_v2_projection_messages
765 + (message_id, thread_id, role, streaming, created_at, updated_at, payload_json)
766 + SELECT 'short-newer-' || i, ${newer}, 'assistant', 0, ${DateTime.formatIso(at(5))},
767 + ${DateTime.formatIso(at(5))}, json_object('text', 'ошибка здесь')
768 + FROM n
769 + `;
770 + const { matches } = yield* search.search({ query: "ошибка в" });
771 + assert.deepEqual(
772 + matches.map((match) => match.threadId),
773 + [older],
774 + );
775 + }).pipe(Effect.provide(layerTest)),
776 +);
777 +
778 +it.layer(layerTest)("ThreadSearch short words outside ASCII", (it) => {
779 + it.effect("finds them in any case or accent, with or without the index", () =>
780 + Effect.gen(function* () {
781 + const projections = yield* ProjectionStore.ProjectionStoreV2;
782 + const search = yield* ThreadSearch.ThreadSearch;
783 + const sql = yield* SqlClient.SqlClient;
784 + const project = ProjectId.make("project:short-non-ascii");
785 + const russian = ThreadId.make("thread:short-russian");
786 + const french = ThreadId.make("thread:short-french");
787 + yield* createProject(project);
788 + yield* Effect.forEach(
789 + [
790 + thread(russian, project),
791 + message(russian, "russian", "user", "В логах ошибка"),
792 + thread(french, project),
793 + message(french, "french", "user", "École fermée à Lyon"),
794 + ],
795 + projections.apply,
796 + { discard: true },
797 + );
798 + const threadIds = (query: string) =>
799 + search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId)));
800 +
801 + const assertMatches = Effect.gen(function* () {
802 + assert.deepEqual(yield* threadIds("ошибка в"), [russian]);
803 + assert.deepEqual(yield* threadIds("fermée éc"), [french]);
804 + // They still narrow: a word the message lacks rules it out.
805 + assert.deepEqual(yield* threadIds("ошибка я"), []);
806 + });
807 + yield* assertMatches;
808 + yield* sql`DROP TABLE orchestration_v2_message_search`;
809 + yield* assertMatches;
810 + }),
811 + );
812 +});
813 +
814 it.layer(layerTest)("ThreadSearch across scripts and without the index", (it) => {
815 it.effect(
816 "matches Korean, Japanese, and two-character Chinese words, and whole short words",
apps/server/src/orchestration-v2/ThreadSearch.ts
+27 -2
@@ -67,6 +67,8 @@ const SearchRequest = Schema.Struct({
67 match: Schema.NullOr(Schema.String),
68 /** Tokens every hit must contain, checked on the text. */
69 substrings: TokenList,
70 + /** For each short word outside ASCII, the spellings a hit must contain one of. */
71 + shortSpellings: Schema.fromJsonString(Schema.Array(Schema.Array(Schema.String))),
72 projectId: Schema.NullOr(ProjectId),
73 /** Threads the caller already lists, left out before the hit limit. */
74 excludedThreadIds: TokenList,
@@ -99,6 +101,20 @@ const MESSAGE_HIT_LIMIT = 2000;
101 const MESSAGE_CANDIDATE_LIMIT = 10_000;
102 // Without the index, how many of the newest messages a search scans.
103 const MESSAGE_SCAN_LIMIT = 20_000;
104 +const NON_ASCII = /[\u0080-\uffff]/;
105 +
106 +/** SQLite's lower() folds ASCII only, so a short word is checked as each spelling it can match. */
107 +function shortTokenSpellings(token: string): ReadonlyArray<string> {
108 + const forms = (char: string) => {
109 + const plain = char.normalize("NFD").replace(/\p{M}/gu, "");
110 + return [char, char.toUpperCase(), plain, plain.toUpperCase()];
111 + };
112 + const spellings = Array.from(token).reduce<ReadonlyArray<string>>(
113 + (prefixes, char) => prefixes.flatMap((prefix) => forms(char).map((form) => prefix + form)),
114 + [""],
115 + );
116 + return [...new Set(spellings)];
117 +}
118
119 /**
120 * The SQL inputs for a message search, or null when the query has no token
@@ -114,9 +130,11 @@ function buildMessageSearch(
130 if (long.length === 0) return null;
131 // Two-character CJK words are too short for trigrams.
132 const indexed = hasIndex && long.every((token) => Array.from(token).length >= 3);
133 + const narrowing = short.filter((token) => !NON_ASCII.test(token));
134 return {
135 match: indexed ? long.map((token) => `"${token.replaceAll('"', '""')}"`).join(" ") : null,
119 - substrings: indexed ? short : [...long, ...short],
136 + substrings: indexed ? narrowing : [...long, ...narrowing],
137 + shortSpellings: short.filter((token) => NON_ASCII.test(token)).map(shortTokenSpellings),
138 };
139 }
140
@@ -160,7 +178,7 @@ export function buildSearchSnippet(text: string, matchStart: number): string {
178 // The scan folds ASCII case only, so accent variants and non-ASCII case variants may not match.
179 export const messageHitsQuery = (
180 sql: SqlClient.SqlClient,
163 - { match, substrings, projectId, excludedThreadIds }: typeof SearchRequest.Encoded,
181 + { match, substrings, shortSpellings, projectId, excludedThreadIds }: typeof SearchRequest.Encoded,
182 ) => sql`
183 WITH eligible AS MATERIALIZED (
184 SELECT threads.thread_id, threads.project_id
@@ -225,6 +243,13 @@ export const messageHitsQuery = (
243 SELECT 1 FROM json_each(${substrings}) AS token
244 WHERE instr(lower(json_extract(messages.payload_json, '$.text')), token.value) = 0
245 )
246 + AND NOT EXISTS (
247 + SELECT 1 FROM json_each(${shortSpellings}) AS token
248 + WHERE NOT EXISTS (
249 + SELECT 1 FROM json_each(token.value) AS spelling
250 + WHERE instr(json_extract(messages.payload_json, '$.text'), spelling.value) > 0
251 + )
252 + )
253 -- The index's own rowid, so FTS5 returns hits newest first and stops at
254 -- the limit; ordering by the messages table's rowid sorts every hit.
255 ORDER BY "messageRowid" DESC