The best way to control Claude Code, Codex and more from any computer or phone
fix(server): short search prefixes find accented messages (#36632)
Co-authored-by: Julius Marminge <julius0216@outlook.com>
Wout Stiens committed
Nov 14, 2026 at 11:37 UTC
6aa14d3cd5300e9fbc59b0c5be816a33d00ca4fd
3 files changed
+95
-18
apps/server/src/orchestration-v2/ThreadSearch.test.ts
+19
-9
@@ -840,7 +840,7 @@ it.layer(layerTest)("ThreadSearch filters before it ranks", (it) => {
840
});
841
842
it.effect.each([true, false])(
843
- "ThreadSearch narrows short words outside ASCII before the hit limit (indexed: %s)",
843
+ "ThreadSearch narrows Cyrillic and accent-folded Latin short words before the hit limit (indexed: %s)",
844
(indexed) =>
845
Effect.gen(function* () {
846
const projections = yield* ProjectionStore.ProjectionStoreV2;
@@ -848,12 +848,15 @@ it.effect.each([true, false])(
848
const sql = yield* SqlClient.SqlClient;
849
const project = ProjectId.make("project:short-limit");
850
const older = ThreadId.make("thread:short-older");
851
+ const olderFrench = ThreadId.make("thread:short-older-french");
852
const newer = ThreadId.make("thread:short-newer");
853
yield* createProject(project);
854
yield* Effect.forEach(
855
[
856
thread(older, project),
857
message(older, "older", "user", "В логах ошибка"),
858
+ thread(olderFrench, project),
859
+ message(olderFrench, "older-french", "user", "École fermée à Lyon"),
860
thread(newer, project),
861
],
862
projections.apply,
@@ -870,15 +873,14 @@ it.effect.each([true, false])(
873
WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 2000)
874
INSERT INTO orchestration_v2_projection_messages
875
(message_id, thread_id, role, streaming, created_at, updated_at, payload_json)
873
- SELECT 'short-newer-' || i, ${newer}, 'assistant', 0, ${DateTime.formatIso(at(5))},
874
- ${DateTime.formatIso(at(5))}, json_object('text', 'ошибка здесь')
875
- FROM n
876
+ SELECT 'short-newer-' || i || '-' || word, ${newer}, 'assistant', 0,
877
+ ${DateTime.formatIso(at(5))}, ${DateTime.formatIso(at(5))}, json_object('text', text)
878
+ FROM n, (SELECT 'ru' AS word, 'ошибка здесь' AS text UNION ALL SELECT 'fr', 'fermée ici')
879
`;
877
- const { matches } = yield* search.search({ query: "ошибка в" });
878
- assert.deepEqual(
879
- matches.map((match) => match.threadId),
880
- [older],
881
- );
880
+ const threadIds = (query: string) =>
881
+ search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId)));
882
+ assert.deepEqual(yield* threadIds("ошибка в"), [older]);
883
+ assert.deepEqual(yield* threadIds("fermée ec"), [olderFrench]);
884
}).pipe(Effect.provide(layerTest)),
885
);
886
@@ -898,6 +900,9 @@ it.layer(layerTest)("ThreadSearch short words outside ASCII", (it) => {
900
message(russian, "russian", "user", "В логах ошибка"),
901
thread(french, project),
902
message(french, "french", "user", "École fermée à Lyon"),
903
+ message(french, "french-shout", "user", "Météo école"),
904
+ message(french, "romanian", "user", "Școala închisă"),
905
+ message(french, "vietnamese", "user", "Hồ Chí Minh"),
906
],
907
projections.apply,
908
{ discard: true },
@@ -908,8 +913,13 @@ it.layer(layerTest)("ThreadSearch short words outside ASCII", (it) => {
913
const assertMatches = Effect.gen(function* () {
914
assert.deepEqual(yield* threadIds("ошибка в"), [russian]);
915
assert.deepEqual(yield* threadIds("fermée éc"), [french]);
916
+ assert.deepEqual(yield* threadIds("ecole fermée"), [french]);
917
+ assert.deepEqual(yield* threadIds("école me"), [french]);
918
+ assert.deepEqual(yield* threadIds("închisă sc"), [french]);
919
+ assert.deepEqual(yield* threadIds("chí ho"), [french]);
920
// They still narrow: a word the message lacks rules it out.
921
assert.deepEqual(yield* threadIds("ошибка я"), []);
922
+ assert.deepEqual(yield* threadIds("fermée ce"), []);
923
});
924
yield* assertMatches;
925
yield* sql`DROP TABLE orchestration_v2_message_search`;
apps/server/src/orchestration-v2/ThreadSearch.ts
+51
-9
@@ -69,8 +69,10 @@ const TokenList = Schema.fromJsonString(Schema.Array(Schema.String));
69
const SearchRequest = Schema.Struct({
70
/** The FTS5 query, or null to scan messages instead. */
71
match: Schema.NullOr(Schema.String),
72
- /** Tokens every hit must contain, checked on the text. */
73
- substrings: TokenList,
72
+ /** Tokens every hit must contain, as typed or as their GLOB folding case and Latin accents. */
73
+ substrings: Schema.fromJsonString(
74
+ Schema.Array(Schema.Struct({ token: Schema.String, accentInsensitive: Schema.String })),
75
+ ),
76
/** For each short word outside ASCII, the spellings a hit must contain one of. */
77
shortSpellings: Schema.fromJsonString(Schema.Array(Schema.Array(Schema.String))),
78
projectId: Schema.NullOr(ProjectId),
@@ -120,6 +122,37 @@ function shortTokenSpellings(token: string): ReadonlyArray<string> {
122
return [...new Set(spellings)];
123
}
124
125
+// Latin-1 Supplement and Extended-A/B/Additional letters that fold to ASCII,
126
+// including Romanian and Vietnamese letters SQLite's lower() leaves alone.
127
+const accentedForms = Map.groupBy(
128
+ [
129
+ ...Array.from({ length: 0x250 - 0xc0 }, (_, offset) => String.fromCharCode(0xc0 + offset)),
130
+ ...Array.from({ length: 0x1f00 - 0x1e00 }, (_, offset) => String.fromCharCode(0x1e00 + offset)),
131
+ ].filter((char) => /^[a-z]$/.test(foldSearchQuery(char))),
132
+ (char) => foldSearchQuery(char),
133
+);
134
+// Only text that may hold one of those letters (by UTF-8 lead byte) needs the GLOB:
135
+// lower() already folds ASCII case.
136
+const MESSAGE_HOLDS_ACCENTED_LATIN = [
137
+ ...new Set(
138
+ [...accentedForms.values()].flat().map((char) => Buffer.from(char).toString("hex", 0, 1)),
139
+ ),
140
+]
141
+ .map((lead) => `instr(CAST(message.text AS BLOB), x'${lead}') > 0`)
142
+ .join(" OR ");
143
+
144
+/** A GLOB pattern for `token` anywhere, in any case and with or without Latin accents. */
145
+function accentInsensitiveGlob(token: string): string {
146
+ const pattern = Array.from(token, (char) => {
147
+ const letter = foldSearchQuery(char);
148
+ if (/^[a-z]$/.test(letter)) {
149
+ return `[${letter}${letter.toUpperCase()}${accentedForms.get(letter)?.join("") ?? ""}]`;
150
+ }
151
+ return /[*?[]/.test(char) ? `[${char}]` : char;
152
+ });
153
+ return `*${pattern.join("")}*`;
154
+}
155
+
156
/**
157
* The SQL inputs for a message search, or null when the query has no token
158
* long enough to search. Tokens of three or more characters go to the
@@ -137,7 +170,10 @@ function buildMessageSearch(
170
const narrowing = short.filter((token) => !NON_ASCII.test(token));
171
return {
172
match: indexed ? long.map((token) => `"${token.replaceAll('"', '""')}"`).join(" ") : null,
140
- substrings: indexed ? narrowing : [...long, ...narrowing],
173
+ substrings: (indexed ? narrowing : [...long, ...narrowing]).map((token) => ({
174
+ token,
175
+ accentInsensitive: accentInsensitiveGlob(token),
176
+ })),
177
shortSpellings: short.filter((token) => NON_ASCII.test(token)).map(shortTokenSpellings),
178
};
179
}
@@ -176,10 +212,10 @@ export function buildSearchSnippet(text: string, matchStart: number): string {
212
return `${start > 0 ? "…" : ""}${shown.slice(0, end)}${cut ? "…" : ""}`;
213
}
214
179
-// Filter hidden subagents and side chats before the hit limit so visible matches aren't lost.
180
-// Word starts and ranking follow the title rules in JS.
181
-// Without FTS5 trigrams, or for two-character CJK words, scan the same eligible messages.
182
-// The scan folds ASCII case only, so accent variants and non-ASCII case variants may not match.
215
+// Filter hidden subagents and side chats before the hit limit; JS checks word starts and ranking.
216
+// Both paths narrow ASCII tokens with Latin-1 and Extended-A/B/Additional accent variants.
217
+// Without FTS5 trigrams, or for two-character CJK words, scan the same eligible messages;
218
+// accented queries and non-Latin case can still miss: "fermée"/"fermee", "ОШИБКА"/"ошибка".
219
export const messageHitsQuery = (
220
sql: SqlClient.SqlClient,
221
{ match, substrings, shortSpellings, projectId, excludedThreadIds }: typeof SearchRequest.Encoded,
@@ -244,8 +280,14 @@ export const messageHitsQuery = (
280
-- neither find nor a jump can reach them.
281
AND messages.message_id NOT LIKE 'async-answer:%'
282
AND NOT EXISTS (
247
- SELECT 1 FROM json_each(${substrings}) AS token
248
- WHERE instr(lower(${messageSearchText(sql)}), token.value) = 0
283
+ -- The text read once, not once per check.
284
+ SELECT 1 FROM (SELECT ${messageSearchText(sql)} AS text) AS message,
285
+ json_each(${substrings}) AS substring
286
+ WHERE instr(lower(message.text), json_extract(substring.value, '$.token')) = 0
287
+ AND NOT (
288
+ (${sql.literal(MESSAGE_HOLDS_ACCENTED_LATIN)})
289
+ AND message.text GLOB json_extract(substring.value, '$.accentInsensitive')
290
+ )
291
)
292
AND NOT EXISTS (
293
SELECT 1 FROM json_each(${shortSpellings}) AS token
scripts/scenarios/thread-search-finds-accented-message.e2e.ts
new
+25
@@ -0,0 +1,25 @@
1
+import { expect } from "e2e";
2
+
3
+import { test, WEB } from "./t3.ts";
4
+
5
+test(
6
+ "Thread search finds an accented message by a short unaccented word",
7
+ WEB,
8
+ async ({ t3, app, screen }) => {
9
+ const project = await t3.project("demo");
10
+ const thread = await t3.thread({ project, title: "Weekly notes", messages: ["Start"] });
11
+ await t3.open(thread);
12
+
13
+ // Search reads what the user wrote; messages the CLI sends count as an agent's.
14
+ const composer = screen.getByRole("textbox");
15
+ await composer.pressSequentially("École fermée à Lyon");
16
+ await composer.press("Enter");
17
+ await expect(screen.getByText("second fixture turn complete", { exact: false })).toBeVisible();
18
+
19
+ await screen.getByRole("combobox", "Search threads").fill("fermée ec");
20
+ await expect(screen.getByRole("listbox", "Thread search results")).toContainText(
21
+ "École fermée",
22
+ );
23
+ await app.screenshot("search");
24
+ },
25
+);