The best way to control Claude Code, Codex and more from any computer or phone

main
ts 762 lines 31.4 KB
Raw
1 import {
2 IsoDateTime,
3 MessageId,
4 OrchestrationThreadSearchSource,
5 OrchestrationV2AppThreadJson,
6 OrchestrationV2ThreadOpens,
7 ThreadLinkedPullRequest,
8 ThreadPullRequestLink,
9 type OrchestrationSearchThreadsInput,
10 type OrchestrationSearchThreadsResult,
11 type OrchestrationV2SearchThreadsInput,
12 type OrchestrationV2SearchThreadsResult,
13 type OrchestrationV2ThreadShellSnapshot,
14 ProjectId,
15 ThreadId,
16 } from "@t3tools/contracts";
17 import * as DateTime from "effect/DateTime";
18 import * as Context from "effect/Context";
19 import * as Effect from "effect/Effect";
20 import * as Layer from "effect/Layer";
21 import * as Option from "effect/Option";
22 import * as Schema from "effect/Schema";
23 import * as SqlClient from "effect/sql/SqlClient";
24 import * as SqlSchema from "effect/sql/SqlSchema";
25 import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient";
26 import {
27 messageSearchTokens,
28 foldSearchQuery,
29 rankMessageMatches,
30 rankSearchItems,
31 messageMatcher,
32 scoreThreadSearch,
33 type ThreadSearchSubject,
34 } from "@t3tools/shared/paletteSearch";
35 import { threadSearchPullRequests } from "@t3tools/shared/threadPullRequests";
36 import {
37 foldedMessageSearchText,
38 indexedMessageCondition,
39 messageSearchText,
40 nonPlaceholderMessageCondition,
41 } from "./messageSearchIndex.ts";
42 import { loadOlderSettledThreadsPage } from "./ShellStream.ts";
43
44 // When the user last wrote in the thread aliased `threads`, as the clients
45 // rank recency; agent notices in the user role do not count.
46 const latestUserMessageAt = (sql: SqlClient.SqlClient) => sql`(
47 SELECT messages.updated_at
48 FROM orchestration_v2_projection_messages AS messages
49 WHERE messages.thread_id = threads.thread_id
50 AND messages.role = 'user'
51 AND COALESCE(json_extract(messages.payload_json, '$.createdBy'), 'user') = 'user'
52 ORDER BY messages.updated_at DESC, messages.message_id DESC
53 LIMIT 1
54 )`;
55
56 export class ThreadSearchError extends Schema.TaggedError<ThreadSearchError>()(
57 "ThreadSearchError",
58 {
59 operation: Schema.Literals(["query", "decode"]),
60 cause: Schema.Defect(),
61 },
62 ) {
63 override get message(): string {
64 return `Thread search ${this.operation} failed.`;
65 }
66 }
67
68 // Token lists bind as JSON arrays, so any number of them is one parameter.
69 const TokenList = Schema.fromJsonString(Schema.Array(Schema.String));
70 const SearchRequest = Schema.Struct({
71 /** The FTS5 query, or null to scan messages instead. */
72 match: Schema.NullOr(Schema.String),
73 /** Tokens every hit must contain, as typed or as their GLOB folding case and Latin accents. */
74 substrings: Schema.fromJsonString(
75 Schema.Array(Schema.Struct({ token: Schema.String, accentInsensitive: Schema.String })),
76 ),
77 /** For each short word outside ASCII, the spellings a hit must contain one of. */
78 shortSpellings: Schema.fromJsonString(Schema.Array(Schema.Array(Schema.String))),
79 projectId: Schema.NullOr(ProjectId),
80 /** Threads the caller already lists, left out before the hit limit. */
81 excludedThreadIds: TokenList,
82 });
83 const SearchRow = Schema.Struct({
84 messageRowid: Schema.Number,
85 messageId: MessageId,
86 threadId: ThreadId,
87 projectId: ProjectId,
88 source: OrchestrationThreadSearchSource,
89 text: Schema.String,
90 messageCreatedAt: Schema.NullOr(IsoDateTime),
91 });
92
93 const ThreadOpensRow = Schema.Struct({
94 threadId: ThreadId,
95 opens: Schema.NullOr(Schema.String),
96 latestUserMessageAt: Schema.NullOr(IsoDateTime),
97 });
98 const decodeThreadOpens = Schema.decodeUnknownOption(
99 Schema.fromJsonString(OrchestrationV2ThreadOpens),
100 );
101
102 const SNIPPET_LENGTH = 240;
103 // The newest hits read per query. A common word stops here instead of reading
104 // every message, so time stays flat as history grows.
105 const MESSAGE_HIT_LIMIT = 2000;
106 // How many of the newest index hits the filters may pass over to fill that
107 // limit: a rare short word or hidden threads would otherwise read every hit.
108 const MESSAGE_CANDIDATE_LIMIT = 10_000;
109 // Without the index, how many of the newest messages a search scans.
110 const MESSAGE_SCAN_LIMIT = 20_000;
111 const NON_ASCII = /[\u0080-\uffff]/;
112
113 /** SQLite's lower() folds ASCII only, so a short word is checked as each spelling it can match. */
114 function shortTokenSpellings(token: string): ReadonlyArray<string> {
115 const forms = (char: string) => {
116 const plain = char.normalize("NFD").replace(/\p{M}/gu, "");
117 return [char, char.toUpperCase(), plain, plain.toUpperCase()];
118 };
119 const spellings = Array.from(token).reduce<ReadonlyArray<string>>(
120 (prefixes, char) => prefixes.flatMap((prefix) => forms(char).map((form) => prefix + form)),
121 [""],
122 );
123 return [...new Set(spellings)];
124 }
125
126 // Latin-1 Supplement and Extended-A/B/Additional letters that fold to ASCII,
127 // including Romanian and Vietnamese letters SQLite's lower() leaves alone.
128 const accentedForms = Map.groupBy(
129 [
130 ...Array.from({ length: 0x250 - 0xc0 }, (_, offset) => String.fromCharCode(0xc0 + offset)),
131 ...Array.from({ length: 0x1f00 - 0x1e00 }, (_, offset) => String.fromCharCode(0x1e00 + offset)),
132 ].filter((char) => /^[a-z]$/.test(foldSearchQuery(char))),
133 (char) => foldSearchQuery(char),
134 );
135 // Only text that may hold one of those letters (by UTF-8 lead byte) needs the GLOB:
136 // lower() already folds ASCII case.
137 const MESSAGE_HOLDS_ACCENTED_LATIN = [
138 ...new Set(
139 [...accentedForms.values()].flat().map((char) => Buffer.from(char).toString("hex", 0, 1)),
140 ),
141 ]
142 .map((lead) => `instr(CAST(message.text AS BLOB), x'${lead}') > 0`)
143 .join(" OR ");
144
145 /** A GLOB pattern for `token` anywhere, in any case and with or without Latin accents. */
146 function accentInsensitiveGlob(token: string): string {
147 const pattern = Array.from(token, (char) => {
148 const letter = foldSearchQuery(char);
149 if (/^[a-z]$/.test(letter)) {
150 return `[${letter}${letter.toUpperCase()}${accentedForms.get(letter)?.join("") ?? ""}]`;
151 }
152 return /[*?[]/.test(char) ? `[${char}]` : char;
153 });
154 return `*${pattern.join("")}*`;
155 }
156
157 /**
158 * The SQL inputs for a message search, or null when the query has no token
159 * long enough to search. Tokens of three or more characters go to the
160 * trigram index as quoted phrases; shorter ones, and every token when the
161 * index cannot serve the query, are checked as substrings of the hits.
162 */
163 function buildMessageSearch(
164 query: string,
165 hasIndex: boolean,
166 ): Omit<typeof SearchRequest.Type, "projectId" | "excludedThreadIds"> | null {
167 const { long, short } = messageSearchTokens(query);
168 if (long.length === 0) return null;
169 // Two-character CJK words are too short for trigrams.
170 const indexed = hasIndex && long.every((token) => Array.from(token).length >= 3);
171 const narrowing = short.filter((token) => !NON_ASCII.test(token));
172 return {
173 match: indexed ? long.map((token) => `"${token.replaceAll('"', '""')}"`).join(" ") : null,
174 substrings: (indexed ? narrowing : [...long, ...narrowing]).map((token) => ({
175 token,
176 accentInsensitive: accentInsensitiveGlob(token),
177 })),
178 shortSpellings: short.filter((token) => NON_ASCII.test(token)).map(shortTokenSpellings),
179 };
180 }
181
182 // A thread shows a link as its text, so in-thread find cannot reach a match
183 // in the address; messages are matched as they read.
184 const MARKDOWN_LINK = /!?\[([^\]\n]*)\]\([^)\s]*\)/g;
185
186 // Result rows show one line, so the match must sit near the start to be seen.
187 const SNIPPET_LEAD = 24;
188
189 const isLowSurrogate = (code: number) => code >= 0xdc00 && code <= 0xdfff;
190
191 /**
192 * Up to 240 characters starting a few words before the match at `matchStart`,
193 * whitespace collapsed, with ellipses where the text was cut. The client finds
194 * the highlights again in the result.
195 */
196 export function buildSearchSnippet(text: string, matchStart: number): string {
197 let start = Math.max(0, matchStart - SNIPPET_LEAD);
198 const wordStart = text.slice(start, matchStart).search(/\s/);
199 if (start > 0 && wordStart !== -1) start += wordStart + 1;
200 // Cut between characters, not inside an emoji's surrogate pair.
201 if (isLowSurrogate(text.charCodeAt(start))) start++;
202 // Collapsing whitespace shortens the text, so read more of it.
203 const rawEnd = Math.min(text.length, start + SNIPPET_LENGTH * 2);
204 // Emphasis and code marks do not show in a thread either.
205 const shown = text
206 .slice(start, rawEnd)
207 .replace(/\*\*|`+/g, "")
208 .replace(/\s+/g, " ")
209 .trim();
210 let end = Math.min(shown.length, SNIPPET_LENGTH - 2);
211 if (isLowSurrogate(shown.charCodeAt(end))) end--;
212 const cut = end < shown.length || rawEnd < text.length;
213 return `${start > 0 ? "…" : ""}${shown.slice(0, end)}${cut ? "…" : ""}`;
214 }
215
216 // Filter hidden subagents and side chats before the hit limit; JS checks word starts and ranking.
217 // Both paths narrow ASCII tokens with Latin-1 and Extended-A/B/Additional accent variants.
218 // Without FTS5 trigrams, or for two-character CJK words, scan the same eligible messages;
219 // accented queries and non-Latin case can still miss: "fermée"/"fermee", "ОШИБКА"/"ошибка".
220 export const messageHitsQuery = (
221 sql: SqlClient.SqlClient,
222 { match, substrings, shortSpellings, projectId, excludedThreadIds }: typeof SearchRequest.Encoded,
223 ) => sql`
224 WITH eligible AS MATERIALIZED (
225 SELECT threads.thread_id, threads.project_id
226 FROM orchestration_v2_projection_threads AS threads
227 INNER JOIN projection_projects AS projects
228 ON projects.project_id = threads.project_id
229 WHERE threads.deleted_at IS NULL
230 AND threads.archived_at IS NULL
231 AND projects.deleted_at IS NULL
232 AND (${projectId} IS NULL OR threads.project_id = ${projectId})
233 AND threads.thread_id NOT IN (SELECT value FROM json_each(${excludedThreadIds}))
234 AND json_extract(threads.payload_json, '$.lineage.relationshipToParent')
235 IS NOT 'subagent'
236 AND json_extract(threads.payload_json, '$.lineage.sideChat') IS NOT 1
237 )
238 ${
239 match === null
240 ? sql`
241 SELECT
242 messages.rowid AS "messageRowid",
243 messages.message_id AS "messageId",
244 messages.thread_id AS "threadId",
245 eligible.project_id AS "projectId",
246 messages.role AS source,
247 ${messageSearchText(sql)} AS text,
248 messages.created_at AS "messageCreatedAt"
249 FROM orchestration_v2_projection_messages AS messages
250 INNER JOIN eligible ON eligible.thread_id = messages.thread_id
251 WHERE ${indexedMessageCondition(sql)}
252 -- A scan reads every message it passes, so it stops at the newest ones.
253 AND messages.rowid > (
254 SELECT coalesce(max(rowid), 0) FROM orchestration_v2_projection_messages
255 ) - ${MESSAGE_SCAN_LIMIT}
256 `
257 : sql`
258 SELECT
259 orchestration_v2_message_search.rowid AS "messageRowid",
260 messages.message_id AS "messageId",
261 messages.thread_id AS "threadId",
262 eligible.project_id AS "projectId",
263 messages.role AS source,
264 ${messageSearchText(sql)} AS text,
265 messages.created_at AS "messageCreatedAt"
266 FROM orchestration_v2_message_search
267 INNER JOIN orchestration_v2_projection_messages AS messages
268 ON messages.rowid = orchestration_v2_message_search.rowid
269 INNER JOIN eligible ON eligible.thread_id = messages.thread_id
270 WHERE orchestration_v2_message_search MATCH ${match}
271 AND orchestration_v2_message_search.rowid >= coalesce((
272 SELECT orchestration_v2_message_search.rowid FROM orchestration_v2_message_search
273 WHERE orchestration_v2_message_search MATCH ${match}
274 ORDER BY orchestration_v2_message_search.rowid DESC
275 LIMIT 1 OFFSET ${MESSAGE_CANDIDATE_LIMIT - 1}
276 ), 0)
277 AND ${nonPlaceholderMessageCondition(sql)}
278 `
279 }
280 -- Answers to an agent's question show inside its question card, where
281 -- neither find nor a jump can reach them.
282 AND messages.message_id NOT LIKE 'async-answer:%'
283 -- Nor can they reach a rolled-back turn, or a message of a stopped turn
284 -- that no shown item carries, such as a queued prompt that never ran.
285 AND NOT EXISTS (
286 SELECT 1 FROM orchestration_v2_projection_runs AS runs
287 WHERE runs.run_id = messages.run_id
288 AND (
289 runs.status = 'rolled_back'
290 OR (
291 runs.status = 'cancelled'
292 AND NOT EXISTS (
293 SELECT 1 FROM orchestration_v2_projection_turn_items AS items
294 WHERE items.run_id = runs.run_id
295 AND json_extract(items.payload_json, '$.messageId') = messages.message_id
296 AND json_extract(items.payload_json, '$.inputIntent') IS NOT 'queued_turn'
297 )
298 )
299 )
300 )
301 AND NOT EXISTS (
302 -- The text read once, not once per check.
303 SELECT 1 FROM (SELECT ${foldedMessageSearchText(sql)} AS text) AS message,
304 json_each(${substrings}) AS substring
305 WHERE instr(lower(message.text), json_extract(substring.value, '$.token')) = 0
306 AND NOT (
307 (${sql.literal(MESSAGE_HOLDS_ACCENTED_LATIN)})
308 AND message.text GLOB json_extract(substring.value, '$.accentInsensitive')
309 )
310 )
311 AND NOT EXISTS (
312 SELECT 1 FROM json_each(${shortSpellings}) AS token
313 WHERE NOT EXISTS (
314 SELECT 1 FROM json_each(token.value) AS spelling
315 WHERE instr(${foldedMessageSearchText(sql)}, spelling.value) > 0
316 )
317 )
318 -- The index's own rowid, so FTS5 returns hits newest first and stops at
319 -- the limit; ordering by the messages table's rowid sorts every hit.
320 ORDER BY "messageRowid" DESC
321 LIMIT ${MESSAGE_HIT_LIMIT}
322 `;
323
324 const TitleCandidateRow = Schema.Struct({
325 thread: Schema.fromJsonString(OrchestrationV2AppThreadJson),
326 projectTitle: Schema.String,
327 latestUserMessageAt: Schema.NullOr(IsoDateTime),
328 });
329
330 export interface ThreadSearchHit {
331 readonly threadId: ThreadId;
332 readonly projectId: ProjectId;
333 readonly title: string;
334 readonly source: "title" | OrchestrationThreadSearchSource;
335 readonly snippet?: string;
336 readonly messageCreatedAt?: string | null;
337 }
338
339 export class ThreadSearch extends Context.Service<
340 ThreadSearch,
341 {
342 /** Message text matches, for clients that rank their own thread list. */
343 readonly search: (
344 input: OrchestrationSearchThreadsInput,
345 ) => Effect.Effect<OrchestrationSearchThreadsResult, ThreadSearchError>;
346 /**
347 * Title, branch, pull request, and project matches ranked as the command
348 * palette ranks them, then message matches for the remaining threads.
349 */
350 readonly searchThreads: (
351 input: OrchestrationSearchThreadsInput & { readonly projectId?: ProjectId | undefined },
352 ) => Effect.Effect<{ readonly matches: ReadonlyArray<ThreadSearchHit> }, ThreadSearchError>;
353 /**
354 * The settled threads before `before` that clients omitting older settled threads should
355 * receive for a query: the best title, branch, pull request and project matches, ranked as
356 * the palette ranks them, then the older ones among `messageThreadIds`, in their order.
357 */
358 readonly searchOlderSettled: (input: {
359 readonly query: string;
360 readonly before: string;
361 readonly projectIds?: ReadonlyArray<ProjectId> | undefined;
362 /** The caller's name for this environment, which its palette also matches. */
363 readonly environmentLabel?: string | undefined;
364 readonly messageThreadIds: ReadonlyArray<ThreadId>;
365 }) => Effect.Effect<ReadonlyArray<ThreadId>, ThreadSearchError>;
366 }
367 >()("t3/orchestration-v2/ThreadSearch") {}
368
369 // Rows a client gets per search: enough for the shown results without sending every match.
370 const OLDER_SETTLED_TITLE_ROWS = 15;
371 const OLDER_SETTLED_MESSAGE_ROWS = 5;
372
373 // Only the fields the palette scorer reads, so an interactive search does not decode payloads.
374 const OlderSettledTitleRow = Schema.Struct({
375 id: ThreadId,
376 title: Schema.String,
377 branch: Schema.NullOr(Schema.String),
378 pullRequests: Schema.NullOr(Schema.fromJsonString(Schema.Array(ThreadPullRequestLink))),
379 linkedPullRequest: Schema.NullOr(Schema.fromJsonString(ThreadLinkedPullRequest)),
380 branchPullRequest: Schema.NullOr(Schema.fromJsonString(ThreadLinkedPullRequest)),
381 opens: Schema.NullOr(Schema.String),
382 createdAt: IsoDateTime,
383 projectTitle: Schema.String,
384 latestUserMessageAt: Schema.NullOr(IsoDateTime),
385 });
386
387 /** @public */
388 export const make = Effect.gen(function* () {
389 const sql = yield* SqlClient.SqlClient;
390
391 const searchRows = SqlSchema.findAll({
392 Request: SearchRequest,
393 Result: SearchRow,
394 execute: (request) => messageHitsQuery(sql, request),
395 });
396
397 const threadOpens = SqlSchema.findAll({
398 Request: Schema.Struct({ threadIds: TokenList }),
399 Result: ThreadOpensRow,
400 execute: ({ threadIds }) => sql`
401 SELECT
402 threads.thread_id AS "threadId",
403 json_extract(threads.payload_json, '$.opens') AS opens,
404 ${latestUserMessageAt(sql)} AS "latestUserMessageAt"
405 FROM orchestration_v2_projection_threads AS threads
406 WHERE threads.thread_id IN (SELECT value FROM json_each(${threadIds}))
407 `,
408 });
409
410 const titleCandidates = SqlSchema.findAll({
411 Request: Schema.Struct({ projectId: Schema.NullOr(ProjectId) }),
412 Result: TitleCandidateRow,
413 execute: ({ projectId }) => sql`
414 SELECT
415 threads.payload_json AS thread,
416 projects.title AS "projectTitle",
417 ${latestUserMessageAt(sql)} AS "latestUserMessageAt"
418 FROM orchestration_v2_projection_threads AS threads
419 INNER JOIN projection_projects AS projects
420 ON projects.project_id = threads.project_id
421 WHERE threads.deleted_at IS NULL
422 AND threads.archived_at IS NULL
423 AND projects.deleted_at IS NULL
424 AND json_extract(threads.payload_json, '$.lineage.relationshipToParent')
425 IS NOT 'subagent'
426 AND json_extract(threads.payload_json, '$.lineage.sideChat') IS NOT 1
427 AND (${projectId} IS NULL OR threads.project_id = ${projectId})
428 `,
429 });
430
431 const olderSettledTitleCandidates = SqlSchema.findAll({
432 Request: Schema.Struct({
433 before: Schema.String,
434 projectIds: Schema.NullOr(TokenList),
435 }),
436 Result: OlderSettledTitleRow,
437 execute: ({ before, projectIds }) => sql`
438 SELECT
439 threads.thread_id AS id,
440 json_extract(threads.payload_json, '$.title') AS title,
441 json_extract(threads.payload_json, '$.branch') AS branch,
442 json_extract(threads.payload_json, '$.pullRequests') AS "pullRequests",
443 json_extract(threads.payload_json, '$.linkedPullRequest') AS "linkedPullRequest",
444 json_extract(threads.payload_json, '$.branchPullRequest') AS "branchPullRequest",
445 json_extract(threads.payload_json, '$.opens') AS opens,
446 json_extract(threads.payload_json, '$.createdAt') AS "createdAt",
447 projects.title AS "projectTitle",
448 ${latestUserMessageAt(sql)} AS "latestUserMessageAt"
449 FROM orchestration_v2_projection_threads AS threads
450 INNER JOIN projection_projects AS projects
451 ON projects.project_id = threads.project_id
452 WHERE threads.deleted_at IS NULL
453 AND threads.archived_at IS NULL
454 AND projects.deleted_at IS NULL
455 AND json_extract(threads.payload_json, '$.lineage.relationshipToParent')
456 IS NOT 'subagent'
457 AND json_extract(threads.payload_json, '$.lineage.sideChat') IS NOT 1
458 AND json_extract(threads.payload_json, '$.settledOverride') = 'settled'
459 AND json_extract(threads.payload_json, '$.settledAt') < ${before}
460 AND (${projectIds} IS NULL
461 OR threads.project_id IN (SELECT value FROM json_each(${projectIds})))
462 `,
463 });
464
465 const olderSettledIds = SqlSchema.findAll({
466 Request: Schema.Struct({
467 before: Schema.String,
468 threadIds: TokenList,
469 projectIds: Schema.NullOr(TokenList),
470 }),
471 Result: Schema.Struct({ id: ThreadId }),
472 execute: ({ before, threadIds, projectIds }) => sql`
473 SELECT threads.thread_id AS id
474 FROM orchestration_v2_projection_threads AS threads
475 WHERE threads.thread_id IN (SELECT value FROM json_each(${threadIds}))
476 AND threads.deleted_at IS NULL
477 AND threads.archived_at IS NULL
478 AND json_extract(threads.payload_json, '$.lineage.relationshipToParent')
479 IS NOT 'subagent'
480 AND json_extract(threads.payload_json, '$.lineage.sideChat') IS NOT 1
481 AND json_extract(threads.payload_json, '$.settledOverride') = 'settled'
482 AND json_extract(threads.payload_json, '$.settledAt') < ${before}
483 AND (${projectIds} IS NULL
484 OR threads.project_id IN (SELECT value FROM json_each(${projectIds})))
485 `,
486 });
487
488 const read = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
489 sql.withTransaction(effect).pipe(
490 NodeSqliteClient.readOnly,
491 Effect.mapError(
492 (cause) =>
493 new ThreadSearchError({
494 operation: Schema.isSchemaError(cause) ? "decode" : "query",
495 cause,
496 }),
497 ),
498 );
499
500 // A project scope applies before ranking, so other projects' stronger hits
501 // cannot use up the limit.
502 const searchMessages = Effect.fn("ThreadSearch.search")(function* (
503 input: OrchestrationSearchThreadsInput & {
504 readonly projectId: ProjectId | null;
505 /** Threads the caller already lists, which must not use up the limit. */
506 readonly excludedThreadIds?: ReadonlySet<ThreadId>;
507 /** Off for an agent's search, which has no use for the user's picks. */
508 readonly picks?: boolean;
509 },
510 ) {
511 // One message per thread: the best word match, then what the user wrote,
512 // since that is what they remember asking, then the newest by when it was
513 // written. Row order is not age: imported histories arrive in one batch.
514 // Text relevance (bm25) would cost a pass over every hit in history.
515 type Hit = {
516 readonly row: typeof SearchRow.Type;
517 /** The message as it reads, links as their text. */
518 readonly text: string;
519 readonly quality: number;
520 readonly offset: number;
521 };
522 const newer = (left: Hit, right: Hit) =>
523 (right.row.messageCreatedAt ?? "").localeCompare(left.row.messageCreatedAt ?? "");
524 const compareHits = (left: Hit, right: Hit) =>
525 right.quality - left.quality ||
526 Number(left.row.source !== "user") - Number(right.row.source !== "user") ||
527 newer(left, right);
528 const rows = yield* read(
529 Effect.gen(function* () {
530 const index = yield* sql`
531 SELECT 1 FROM sqlite_master WHERE name = 'orchestration_v2_message_search'
532 `;
533 const messageSearch = buildMessageSearch(input.query, index.length > 0);
534 return messageSearch === null
535 ? []
536 : yield* searchRows({
537 ...messageSearch,
538 projectId: input.projectId,
539 excludedThreadIds: [...(input.excludedThreadIds ?? [])],
540 });
541 }),
542 );
543 const best = new Map<ThreadId, Hit>();
544 // Matching messages per thread at its best quality: a thread that keeps
545 // using the word ranks above a passing mention, and weaker hits ("Trust"
546 // for "rust") never add to a whole word's count.
547 const hitCounts = new Map<ThreadId, number>();
548 const hitCount = (threadId: ThreadId) => hitCounts.get(threadId) ?? 1;
549 const matchMessage = messageMatcher(input.query);
550 for (const row of rows) {
551 const text = row.text.replace(MARKDOWN_LINK, "$1");
552 const match = matchMessage(text);
553 if (match === null) continue;
554 const hit = { row, text, ...match };
555 const current = best.get(row.threadId);
556 if (current === undefined || hit.quality > current.quality) {
557 hitCounts.set(row.threadId, 1);
558 } else if (hit.quality === current.quality) {
559 hitCounts.set(row.threadId, hitCount(row.threadId) + 1);
560 }
561 if (current === undefined || compareHits(hit, current) < 0) best.set(row.threadId, hit);
562 }
563 // Ranked as clients rank them, with picks and use, before the limit cuts:
564 // a thread picked for this query must come back even when it only
565 // matches in a few old messages.
566 const opens = best.size === 0 ? [] : yield* read(threadOpens({ threadIds: [...best.keys()] }));
567 const subjectById = new Map(
568 opens.map((row) => [
569 row.threadId,
570 {
571 title: "",
572 activityAt: row.latestUserMessageAt === null ? null : Date.parse(row.latestUserMessageAt),
573 opens: Option.getOrNull(decodeThreadOpens(row.opens)),
574 },
575 ]),
576 );
577 const now = DateTime.toEpochMillis(yield* DateTime.now);
578 const ranked = rankMessageMatches(
579 [...best.values()].map((hit) => ({
580 ...hit,
581 hits: hitCount(hit.row.threadId),
582 messageCreatedAt: hit.row.messageCreatedAt,
583 })),
584 (hit) => subjectById.get(hit.row.threadId),
585 foldSearchQuery(input.query),
586 now,
587 { picks: input.picks ?? true },
588 );
589 return {
590 matches: ranked.slice(0, input.limit ?? 50).map(({ row, text, quality, offset }) => ({
591 hits: hitCount(row.threadId),
592 threadId: row.threadId,
593 messageId: row.messageId,
594 projectId: row.projectId,
595 source: row.source,
596 snippet: buildSearchSnippet(text, offset),
597 messageCreatedAt: row.messageCreatedAt,
598 quality,
599 })),
600 };
601 });
602
603 const search: ThreadSearch["Service"]["search"] = (input) =>
604 searchMessages({ ...input, projectId: null });
605
606 const searchThreads: ThreadSearch["Service"]["searchThreads"] = Effect.fn(
607 "ThreadSearch.searchThreads",
608 )(function* (input) {
609 const limit = input.limit ?? 50;
610 const normalizedQuery = foldSearchQuery(input.query);
611 const now = DateTime.toEpochMillis(yield* DateTime.now);
612 // Agents call this rarely, and a project has hundreds of threads, not
613 // millions, so ranking every active thread in memory stays cheap.
614 const candidates = yield* read(titleCandidates({ projectId: input.projectId ?? null }));
615 const subjectById = new Map(
616 candidates.map((row): [ThreadId, ThreadSearchSubject] => [
617 row.thread.id,
618 {
619 id: row.thread.id,
620 title: row.thread.title,
621 branch: row.thread.branch,
622 projectTitle: row.projectTitle,
623 pullRequests: threadSearchPullRequests(row.thread),
624 // Like the clients: what the user last wrote, else when the thread began.
625 activityAt: Date.parse(
626 row.latestUserMessageAt ?? DateTime.formatIso(row.thread.createdAt),
627 ),
628 opens: row.thread.opens ?? null,
629 },
630 ]),
631 );
632 const titleHits: ThreadSearchHit[] = rankSearchItems(candidates, (row, options) =>
633 // An agent's search has no use for the user's palette picks.
634 scoreThreadSearch(normalizedQuery, subjectById.get(row.thread.id)!, now, {
635 ...options,
636 picks: false,
637 }),
638 )
639 .slice(0, limit)
640 .map(({ item }) => ({
641 threadId: item.thread.id,
642 projectId: item.thread.projectId,
643 title: item.thread.title,
644 source: "title",
645 }));
646 const shown = new Set(titleHits.map((hit) => hit.threadId));
647 const { matches } = yield* searchMessages({
648 query: input.query,
649 limit: 50,
650 projectId: input.projectId ?? null,
651 excludedThreadIds: shown,
652 picks: false,
653 });
654 const messageHits = matches
655 .filter((match) => subjectById.has(match.threadId))
656 .map(({ quality: _quality, hits: _hits, messageId: _messageId, ...match }) => ({
657 ...match,
658 title: subjectById.get(match.threadId)!.title,
659 }));
660 // Titles come first, but a fifth of the slots stay for message matches,
661 // as the palette keeps its "In messages" section.
662 const messageSlots = Math.min(messageHits.length, Math.floor(limit / 5));
663 return {
664 matches: [...titleHits.slice(0, limit - messageSlots), ...messageHits].slice(0, limit),
665 };
666 });
667
668 const searchOlderSettled: ThreadSearch["Service"]["searchOlderSettled"] = Effect.fn(
669 "ThreadSearch.searchOlderSettled",
670 )(function* ({ query, before, projectIds, environmentLabel, messageThreadIds }) {
671 const normalizedQuery = foldSearchQuery(query);
672 const now = DateTime.toEpochMillis(yield* DateTime.now);
673 const candidates = yield* read(
674 olderSettledTitleCandidates({ before, projectIds: projectIds ?? null }),
675 );
676 const subjectOf = (row: typeof OlderSettledTitleRow.Type): ThreadSearchSubject => ({
677 id: row.id,
678 title: row.title,
679 branch: row.branch,
680 projectTitle: row.projectTitle,
681 environmentLabel: environmentLabel ?? null,
682 pullRequests: threadSearchPullRequests({
683 pullRequests: row.pullRequests ?? undefined,
684 linkedPullRequest: row.linkedPullRequest,
685 branchPullRequest: row.branchPullRequest,
686 }),
687 // Like the clients: what the user last wrote, else when the thread began.
688 activityAt: Date.parse(row.latestUserMessageAt ?? row.createdAt),
689 opens: Option.getOrNull(decodeThreadOpens(row.opens)),
690 });
691 // Picks on, as in the command palette: a thread picked for this query must make the cut.
692 const titleIds = rankSearchItems(candidates, (row, options) =>
693 scoreThreadSearch(normalizedQuery, subjectOf(row), now, { ...options, picks: true }),
694 )
695 .slice(0, OLDER_SETTLED_TITLE_ROWS)
696 .map(({ item }) => item.id);
697 const shown = new Set(titleIds);
698 const messageCandidates = [...new Set(messageThreadIds)].filter((id) => !shown.has(id));
699 const older =
700 messageCandidates.length === 0
701 ? new Set<ThreadId>()
702 : new Set(
703 (yield* read(
704 olderSettledIds({
705 before,
706 threadIds: messageCandidates,
707 projectIds: projectIds ?? null,
708 }),
709 )).map((row) => row.id),
710 );
711 return [
712 ...titleIds,
713 ...messageCandidates.filter((id) => older.has(id)).slice(0, OLDER_SETTLED_MESSAGE_ROWS),
714 ];
715 });
716
717 return ThreadSearch.of({ search, searchThreads, searchOlderSettled });
718 });
719
720 export const layer = Layer.effect(ThreadSearch, make);
721
722 /**
723 * The client search: message hits, and with `olderSettledBefore`, the rows of the best older
724 * settled matches, which shells that omit them do not hold.
725 */
726 export const searchThreadsWithOlderSettledRows = Effect.fn(
727 "ThreadSearch.searchThreadsWithOlderSettledRows",
728 )(function* <E1, E2, E3, E4, E5>(input: {
729 readonly request: OrchestrationV2SearchThreadsInput;
730 readonly search: (
731 input: OrchestrationSearchThreadsInput,
732 ) => Effect.Effect<OrchestrationSearchThreadsResult, E1>;
733 readonly searchOlderSettled: (
734 input: Parameters<ThreadSearch["Service"]["searchOlderSettled"]>[0],
735 ) => Effect.Effect<ReadonlyArray<ThreadId>, E2>;
736 readonly sql: SqlClient.SqlClient;
737 readonly readThreads: (
738 threadIds: ReadonlyArray<ThreadId>,
739 ) => Effect.Effect<Effect.Effect<OrchestrationV2ThreadShellSnapshot, E4>, E3>;
740 readonly latestSequence: Effect.Effect<number, E5>;
741 }) {
742 const { olderSettledBefore, projectIds, environmentLabel, titlesOnly, ...request } =
743 input.request;
744 const messages: OrchestrationV2SearchThreadsResult =
745 titlesOnly === true ? { matches: [] } : yield* input.search(request);
746 if (olderSettledBefore === undefined) return messages;
747 // Ranked before the rows' transaction, so scoring never holds the shared reader.
748 const threadIds = yield* input.searchOlderSettled({
749 query: request.query,
750 before: olderSettledBefore,
751 projectIds,
752 environmentLabel,
753 messageThreadIds: messages.matches.map((match) => match.threadId),
754 });
755 const older = yield* loadOlderSettledThreadsPage({
756 sql: input.sql,
757 listThreads: Effect.succeed({ threadIds, counts: [] }),
758 readThreads: input.readThreads,
759 latestSequence: input.latestSequence,
760 });
761 return { ...messages, snapshotSequence: older.snapshotSequence, threads: older.threads };
762 });