@devries-cameron20 / t3code-1 / commits / 6aa14d3cd5
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): short search prefixes find accented messages (#36632)

Co-authored-by: Julius Marminge <julius0216@outlook.com>

Wout Stiens committed Nov 14, 2026 at 11:37 UTC 6aa14d3cd5300e9fbc59b0c5be816a33d00ca4fd
3 files changed +95 -18
apps/server/src/orchestration-v2/ThreadSearch.test.ts
+19 -9
@@ -840,7 +840,7 @@ it.layer(layerTest)("ThreadSearch filters before it ranks", (it) => {
840 });
841
842 it.effect.each([true, false])(
843 - "ThreadSearch narrows short words outside ASCII before the hit limit (indexed: %s)",
843 + "ThreadSearch narrows Cyrillic and accent-folded Latin short words before the hit limit (indexed: %s)",
844 (indexed) =>
845 Effect.gen(function* () {
846 const projections = yield* ProjectionStore.ProjectionStoreV2;
@@ -848,12 +848,15 @@ it.effect.each([true, false])(
848 const sql = yield* SqlClient.SqlClient;
849 const project = ProjectId.make("project:short-limit");
850 const older = ThreadId.make("thread:short-older");
851 + const olderFrench = ThreadId.make("thread:short-older-french");
852 const newer = ThreadId.make("thread:short-newer");
853 yield* createProject(project);
854 yield* Effect.forEach(
855 [
856 thread(older, project),
857 message(older, "older", "user", "В логах ошибка"),
858 + thread(olderFrench, project),
859 + message(olderFrench, "older-french", "user", "École fermée à Lyon"),
860 thread(newer, project),
861 ],
862 projections.apply,
@@ -870,15 +873,14 @@ it.effect.each([true, false])(
873 WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 2000)
874 INSERT INTO orchestration_v2_projection_messages
875 (message_id, thread_id, role, streaming, created_at, updated_at, payload_json)
873 - SELECT 'short-newer-' || i, ${newer}, 'assistant', 0, ${DateTime.formatIso(at(5))},
874 - ${DateTime.formatIso(at(5))}, json_object('text', 'ошибка здесь')
875 - FROM n
876 + SELECT 'short-newer-' || i || '-' || word, ${newer}, 'assistant', 0,
877 + ${DateTime.formatIso(at(5))}, ${DateTime.formatIso(at(5))}, json_object('text', text)
878 + FROM n, (SELECT 'ru' AS word, 'ошибка здесь' AS text UNION ALL SELECT 'fr', 'fermée ici')
879 `;
877 - const { matches } = yield* search.search({ query: "ошибка в" });
878 - assert.deepEqual(
879 - matches.map((match) => match.threadId),
880 - [older],
881 - );
880 + const threadIds = (query: string) =>
881 + search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId)));
882 + assert.deepEqual(yield* threadIds("ошибка в"), [older]);
883 + assert.deepEqual(yield* threadIds("fermée ec"), [olderFrench]);
884 }).pipe(Effect.provide(layerTest)),
885 );
886
@@ -898,6 +900,9 @@ it.layer(layerTest)("ThreadSearch short words outside ASCII", (it) => {
900 message(russian, "russian", "user", "В логах ошибка"),
901 thread(french, project),
902 message(french, "french", "user", "École fermée à Lyon"),
903 + message(french, "french-shout", "user", "Météo école"),
904 + message(french, "romanian", "user", "Școala închisă"),
905 + message(french, "vietnamese", "user", "Hồ Chí Minh"),
906 ],
907 projections.apply,
908 { discard: true },
@@ -908,8 +913,13 @@ it.layer(layerTest)("ThreadSearch short words outside ASCII", (it) => {
913 const assertMatches = Effect.gen(function* () {
914 assert.deepEqual(yield* threadIds("ошибка в"), [russian]);
915 assert.deepEqual(yield* threadIds("fermée éc"), [french]);
916 + assert.deepEqual(yield* threadIds("ecole fermée"), [french]);
917 + assert.deepEqual(yield* threadIds("école me"), [french]);
918 + assert.deepEqual(yield* threadIds("închisă sc"), [french]);
919 + assert.deepEqual(yield* threadIds("chí ho"), [french]);
920 // They still narrow: a word the message lacks rules it out.
921 assert.deepEqual(yield* threadIds("ошибка я"), []);
922 + assert.deepEqual(yield* threadIds("fermée ce"), []);
923 });
924 yield* assertMatches;
925 yield* sql`DROP TABLE orchestration_v2_message_search`;
apps/server/src/orchestration-v2/ThreadSearch.ts
+51 -9
@@ -69,8 +69,10 @@ const TokenList = Schema.fromJsonString(Schema.Array(Schema.String));
69 const SearchRequest = Schema.Struct({
70 /** The FTS5 query, or null to scan messages instead. */
71 match: Schema.NullOr(Schema.String),
72 - /** Tokens every hit must contain, checked on the text. */
73 - substrings: TokenList,
72 + /** Tokens every hit must contain, as typed or as their GLOB folding case and Latin accents. */
73 + substrings: Schema.fromJsonString(
74 + Schema.Array(Schema.Struct({ token: Schema.String, accentInsensitive: Schema.String })),
75 + ),
76 /** For each short word outside ASCII, the spellings a hit must contain one of. */
77 shortSpellings: Schema.fromJsonString(Schema.Array(Schema.Array(Schema.String))),
78 projectId: Schema.NullOr(ProjectId),
@@ -120,6 +122,37 @@ function shortTokenSpellings(token: string): ReadonlyArray<string> {
122 return [...new Set(spellings)];
123 }
124
125 +// Latin-1 Supplement and Extended-A/B/Additional letters that fold to ASCII,
126 +// including Romanian and Vietnamese letters SQLite's lower() leaves alone.
127 +const accentedForms = Map.groupBy(
128 + [
129 + ...Array.from({ length: 0x250 - 0xc0 }, (_, offset) => String.fromCharCode(0xc0 + offset)),
130 + ...Array.from({ length: 0x1f00 - 0x1e00 }, (_, offset) => String.fromCharCode(0x1e00 + offset)),
131 + ].filter((char) => /^[a-z]$/.test(foldSearchQuery(char))),
132 + (char) => foldSearchQuery(char),
133 +);
134 +// Only text that may hold one of those letters (by UTF-8 lead byte) needs the GLOB:
135 +// lower() already folds ASCII case.
136 +const MESSAGE_HOLDS_ACCENTED_LATIN = [
137 + ...new Set(
138 + [...accentedForms.values()].flat().map((char) => Buffer.from(char).toString("hex", 0, 1)),
139 + ),
140 +]
141 + .map((lead) => `instr(CAST(message.text AS BLOB), x'${lead}') > 0`)
142 + .join(" OR ");
143 +
144 +/** A GLOB pattern for `token` anywhere, in any case and with or without Latin accents. */
145 +function accentInsensitiveGlob(token: string): string {
146 + const pattern = Array.from(token, (char) => {
147 + const letter = foldSearchQuery(char);
148 + if (/^[a-z]$/.test(letter)) {
149 + return `[${letter}${letter.toUpperCase()}${accentedForms.get(letter)?.join("") ?? ""}]`;
150 + }
151 + return /[*?[]/.test(char) ? `[${char}]` : char;
152 + });
153 + return `*${pattern.join("")}*`;
154 +}
155 +
156 /**
157 * The SQL inputs for a message search, or null when the query has no token
158 * long enough to search. Tokens of three or more characters go to the
@@ -137,7 +170,10 @@ function buildMessageSearch(
170 const narrowing = short.filter((token) => !NON_ASCII.test(token));
171 return {
172 match: indexed ? long.map((token) => `"${token.replaceAll('"', '""')}"`).join(" ") : null,
140 - substrings: indexed ? narrowing : [...long, ...narrowing],
173 + substrings: (indexed ? narrowing : [...long, ...narrowing]).map((token) => ({
174 + token,
175 + accentInsensitive: accentInsensitiveGlob(token),
176 + })),
177 shortSpellings: short.filter((token) => NON_ASCII.test(token)).map(shortTokenSpellings),
178 };
179 }
@@ -176,10 +212,10 @@ export function buildSearchSnippet(text: string, matchStart: number): string {
212 return `${start > 0 ? "…" : ""}${shown.slice(0, end)}${cut ? "…" : ""}`;
213 }
214
179 -// Filter hidden subagents and side chats before the hit limit so visible matches aren't lost.
180 -// Word starts and ranking follow the title rules in JS.
181 -// Without FTS5 trigrams, or for two-character CJK words, scan the same eligible messages.
182 -// The scan folds ASCII case only, so accent variants and non-ASCII case variants may not match.
215 +// Filter hidden subagents and side chats before the hit limit; JS checks word starts and ranking.
216 +// Both paths narrow ASCII tokens with Latin-1 and Extended-A/B/Additional accent variants.
217 +// Without FTS5 trigrams, or for two-character CJK words, scan the same eligible messages;
218 +// accented queries and non-Latin case can still miss: "fermée"/"fermee", "ОШИБКА"/"ошибка".
219 export const messageHitsQuery = (
220 sql: SqlClient.SqlClient,
221 { match, substrings, shortSpellings, projectId, excludedThreadIds }: typeof SearchRequest.Encoded,
@@ -244,8 +280,14 @@ export const messageHitsQuery = (
280 -- neither find nor a jump can reach them.
281 AND messages.message_id NOT LIKE 'async-answer:%'
282 AND NOT EXISTS (
247 - SELECT 1 FROM json_each(${substrings}) AS token
248 - WHERE instr(lower(${messageSearchText(sql)}), token.value) = 0
283 + -- The text read once, not once per check.
284 + SELECT 1 FROM (SELECT ${messageSearchText(sql)} AS text) AS message,
285 + json_each(${substrings}) AS substring
286 + WHERE instr(lower(message.text), json_extract(substring.value, '$.token')) = 0
287 + AND NOT (
288 + (${sql.literal(MESSAGE_HOLDS_ACCENTED_LATIN)})
289 + AND message.text GLOB json_extract(substring.value, '$.accentInsensitive')
290 + )
291 )
292 AND NOT EXISTS (
293 SELECT 1 FROM json_each(${shortSpellings}) AS token
scripts/scenarios/thread-search-finds-accented-message.e2e.ts new
+25
@@ -0,0 +1,25 @@
1 +import { expect } from "e2e";
2 +
3 +import { test, WEB } from "./t3.ts";
4 +
5 +test(
6 + "Thread search finds an accented message by a short unaccented word",
7 + WEB,
8 + async ({ t3, app, screen }) => {
9 + const project = await t3.project("demo");
10 + const thread = await t3.thread({ project, title: "Weekly notes", messages: ["Start"] });
11 + await t3.open(thread);
12 +
13 + // Search reads what the user wrote; messages the CLI sends count as an agent's.
14 + const composer = screen.getByRole("textbox");
15 + await composer.pressSequentially("École fermée à Lyon");
16 + await composer.press("Enter");
17 + await expect(screen.getByText("second fixture turn complete", { exact: false })).toBeVisible();
18 +
19 + await screen.getByRole("combobox", "Search threads").fill("fermée ec");
20 + await expect(screen.getByRole("listbox", "Thread search results")).toContainText(
21 + "École fermée",
22 + );
23 + await app.screenshot("search");
24 + },
25 +);