The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 03:15 UTC
fix(server): message search keeps results when a short non-Latin word differs in case (#30332)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
MarcoDotIO committed
Nov 5, 2026 at 23:29 UTC
b800f9919bcb151ef441f056ff49c301c34018d2
2 files changed
+107
-3
apps/server/src/orchestration-v2/ThreadSearch.test.ts
+80
-1
@@ -270,7 +270,7 @@ it.layer(layerTest)("ThreadSearch", (it) => {
270
for (const match of ['"needle"', null]) {
271
const plan = yield* sql<{
272
readonly detail: string;
273
- }>`EXPLAIN QUERY PLAN ${ThreadSearch.messageHitsQuery(sql, { match, substrings: "[]", projectId: null, excludedThreadIds: "[]" })}`;
273
+ }>`EXPLAIN QUERY PLAN ${ThreadSearch.messageHitsQuery(sql, { match, substrings: "[]", shortSpellings: "[]", projectId: null, excludedThreadIds: "[]" })}`;
274
assert.isFalse(
275
plan.some((row) => row.detail.includes("TEMP B-TREE")),
276
plan.map((row) => row.detail).join("\n"),
@@ -732,6 +732,85 @@ it.layer(layerTest)("ThreadSearch filters before it ranks", (it) => {
732
);
733
});
734
735
+it.effect.each([true, false])(
736
+ "ThreadSearch narrows short words outside ASCII before the hit limit (indexed: %s)",
737
+ (indexed) =>
738
+ Effect.gen(function* () {
739
+ const projections = yield* ProjectionStore.ProjectionStoreV2;
740
+ const search = yield* ThreadSearch.ThreadSearch;
741
+ const sql = yield* SqlClient.SqlClient;
742
+ const project = ProjectId.make("project:short-limit");
743
+ const older = ThreadId.make("thread:short-older");
744
+ const newer = ThreadId.make("thread:short-newer");
745
+ yield* createProject(project);
746
+ yield* Effect.forEach(
747
+ [
748
+ thread(older, project),
749
+ message(older, "older", "user", "В логах ошибка"),
750
+ thread(newer, project),
751
+ ],
752
+ projections.apply,
753
+ { discard: true },
754
+ );
755
+ if (!indexed) {
756
+ yield* sql`DROP TRIGGER orchestration_v2_message_search_insert`;
757
+ yield* sql`DROP TRIGGER orchestration_v2_message_search_update`;
758
+ yield* sql`DROP TRIGGER orchestration_v2_message_search_delete`;
759
+ yield* sql`DROP TABLE orchestration_v2_message_search`;
760
+ }
761
+ // More newer hits for the long word than one read holds, none with the short one.
762
+ yield* sql`
763
+ WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 2000)
764
+ INSERT INTO orchestration_v2_projection_messages
765
+ (message_id, thread_id, role, streaming, created_at, updated_at, payload_json)
766
+ SELECT 'short-newer-' || i, ${newer}, 'assistant', 0, ${DateTime.formatIso(at(5))},
767
+ ${DateTime.formatIso(at(5))}, json_object('text', 'ошибка здесь')
768
+ FROM n
769
+ `;
770
+ const { matches } = yield* search.search({ query: "ошибка в" });
771
+ assert.deepEqual(
772
+ matches.map((match) => match.threadId),
773
+ [older],
774
+ );
775
+ }).pipe(Effect.provide(layerTest)),
776
+);
777
+
778
+it.layer(layerTest)("ThreadSearch short words outside ASCII", (it) => {
779
+ it.effect("finds them in any case or accent, with or without the index", () =>
780
+ Effect.gen(function* () {
781
+ const projections = yield* ProjectionStore.ProjectionStoreV2;
782
+ const search = yield* ThreadSearch.ThreadSearch;
783
+ const sql = yield* SqlClient.SqlClient;
784
+ const project = ProjectId.make("project:short-non-ascii");
785
+ const russian = ThreadId.make("thread:short-russian");
786
+ const french = ThreadId.make("thread:short-french");
787
+ yield* createProject(project);
788
+ yield* Effect.forEach(
789
+ [
790
+ thread(russian, project),
791
+ message(russian, "russian", "user", "В логах ошибка"),
792
+ thread(french, project),
793
+ message(french, "french", "user", "École fermée à Lyon"),
794
+ ],
795
+ projections.apply,
796
+ { discard: true },
797
+ );
798
+ const threadIds = (query: string) =>
799
+ search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId)));
800
+
801
+ const assertMatches = Effect.gen(function* () {
802
+ assert.deepEqual(yield* threadIds("ошибка в"), [russian]);
803
+ assert.deepEqual(yield* threadIds("fermée éc"), [french]);
804
+ // They still narrow: a word the message lacks rules it out.
805
+ assert.deepEqual(yield* threadIds("ошибка я"), []);
806
+ });
807
+ yield* assertMatches;
808
+ yield* sql`DROP TABLE orchestration_v2_message_search`;
809
+ yield* assertMatches;
810
+ }),
811
+ );
812
+});
813
+
814
it.layer(layerTest)("ThreadSearch across scripts and without the index", (it) => {
815
it.effect(
816
"matches Korean, Japanese, and two-character Chinese words, and whole short words",
apps/server/src/orchestration-v2/ThreadSearch.ts
+27
-2
@@ -67,6 +67,8 @@ const SearchRequest = Schema.Struct({
67
match: Schema.NullOr(Schema.String),
68
/** Tokens every hit must contain, checked on the text. */
69
substrings: TokenList,
70
+ /** For each short word outside ASCII, the spellings a hit must contain one of. */
71
+ shortSpellings: Schema.fromJsonString(Schema.Array(Schema.Array(Schema.String))),
72
projectId: Schema.NullOr(ProjectId),
73
/** Threads the caller already lists, left out before the hit limit. */
74
excludedThreadIds: TokenList,
@@ -99,6 +101,20 @@ const MESSAGE_HIT_LIMIT = 2000;
101
const MESSAGE_CANDIDATE_LIMIT = 10_000;
102
// Without the index, how many of the newest messages a search scans.
103
const MESSAGE_SCAN_LIMIT = 20_000;
104
+const NON_ASCII = /[\u0080-\uffff]/;
105
+
106
+/** SQLite's lower() folds ASCII only, so a short word is checked as each spelling it can match. */
107
+function shortTokenSpellings(token: string): ReadonlyArray<string> {
108
+ const forms = (char: string) => {
109
+ const plain = char.normalize("NFD").replace(/\p{M}/gu, "");
110
+ return [char, char.toUpperCase(), plain, plain.toUpperCase()];
111
+ };
112
+ const spellings = Array.from(token).reduce<ReadonlyArray<string>>(
113
+ (prefixes, char) => prefixes.flatMap((prefix) => forms(char).map((form) => prefix + form)),
114
+ [""],
115
+ );
116
+ return [...new Set(spellings)];
117
+}
118
119
/**
120
* The SQL inputs for a message search, or null when the query has no token
@@ -114,9 +130,11 @@ function buildMessageSearch(
130
if (long.length === 0) return null;
131
// Two-character CJK words are too short for trigrams.
132
const indexed = hasIndex && long.every((token) => Array.from(token).length >= 3);
133
+ const narrowing = short.filter((token) => !NON_ASCII.test(token));
134
return {
135
match: indexed ? long.map((token) => `"${token.replaceAll('"', '""')}"`).join(" ") : null,
119
- substrings: indexed ? short : [...long, ...short],
136
+ substrings: indexed ? narrowing : [...long, ...narrowing],
137
+ shortSpellings: short.filter((token) => NON_ASCII.test(token)).map(shortTokenSpellings),
138
};
139
}
140
@@ -160,7 +178,7 @@ export function buildSearchSnippet(text: string, matchStart: number): string {
178
// The scan folds ASCII case only, so accent variants and non-ASCII case variants may not match.
179
export const messageHitsQuery = (
180
sql: SqlClient.SqlClient,
163
- { match, substrings, projectId, excludedThreadIds }: typeof SearchRequest.Encoded,
181
+ { match, substrings, shortSpellings, projectId, excludedThreadIds }: typeof SearchRequest.Encoded,
182
) => sql`
183
WITH eligible AS MATERIALIZED (
184
SELECT threads.thread_id, threads.project_id
@@ -225,6 +243,13 @@ export const messageHitsQuery = (
243
SELECT 1 FROM json_each(${substrings}) AS token
244
WHERE instr(lower(json_extract(messages.payload_json, '$.text')), token.value) = 0
245
)
246
+ AND NOT EXISTS (
247
+ SELECT 1 FROM json_each(${shortSpellings}) AS token
248
+ WHERE NOT EXISTS (
249
+ SELECT 1 FROM json_each(token.value) AS spelling
250
+ WHERE instr(json_extract(messages.payload_json, '$.text'), spelling.value) > 0
251
+ )
252
+ )
253
-- The index's own rowid, so FTS5 returns hits newest first and stops at
254
-- the limit; ordering by the messages table's rowid sorts every hit.
255
ORDER BY "messageRowid" DESC