The best way to control Claude Code, Codex and more from any computer or phone

main
ts 1,502 lines 60.5 KB
Raw
1 import { assert, it } from "@effect/vitest";
2 import {
3 EnvironmentId,
4 EventId,
5 MessageId,
6 NodeId,
7 ProjectId,
8 ProviderInstanceId,
9 RunId,
10 ThreadId,
11 TurnItemId,
12 type OrchestrationV2DomainEvent,
13 type ThreadLinkedPullRequest,
14 } from "@t3tools/contracts";
15 import * as NodeServices from "@effect/platform-node/NodeServices";
16 import * as DateTime from "effect/DateTime";
17 import * as Deferred from "effect/Deferred";
18 import * as Effect from "effect/Effect";
19 import * as TestClock from "effect/testing/TestClock";
20 import * as FileSystem from "effect/FileSystem";
21 import * as Fiber from "effect/Fiber";
22 import * as Layer from "effect/Layer";
23 import * as Path from "effect/Path";
24 import * as SqlClient from "effect/sql/SqlClient";
25 import { serializeAssistantCitation } from "@t3tools/shared/assistantCitations";
26 import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "@t3tools/shared/composerContextReferences";
27
28 import * as SqlitePersistence from "../persistence/Sqlite.ts";
29 import { rebuildMessageSearch } from "./messageSearchIndex.ts";
30 import * as ProjectionStore from "./ProjectionStore.ts";
31 import * as ProjectStore from "./ProjectStore.ts";
32 import * as ThreadSearch from "./ThreadSearch.ts";
33
34 const layerTest = Layer.mergeAll(
35 ThreadSearch.layer,
36 ProjectionStore.layer,
37 ProjectStore.layer,
38 ).pipe(Layer.provideMerge(SqlitePersistence.layerMemory));
39
40 const providerInstanceId = ProviderInstanceId.make("codex");
41 const at = (minute: number) => DateTime.makeUnsafe(Date.UTC(2026, 8, 27, 0, minute));
42
43 const createProject = (projectId: ProjectId) =>
44 Effect.flatMap(ProjectStore.ProjectStoreV2, (projects) =>
45 projects.apply({
46 sequence: 0,
47 eventId: EventId.make(`created:${projectId}`),
48 aggregateKind: "project",
49 aggregateId: projectId,
50 occurredAt: DateTime.formatIso(at(0)),
51 commandId: null,
52 causationEventId: null,
53 correlationId: null,
54 metadata: {},
55 type: "project.created",
56 payload: {
57 projectId,
58 title: projectId,
59 workspaceRoot: `/work/${projectId}`,
60 defaultModelSelection: null,
61 scripts: [],
62 createdAt: DateTime.formatIso(at(0)),
63 updatedAt: DateTime.formatIso(at(0)),
64 },
65 }),
66 );
67
68 const thread = (
69 threadId: ThreadId,
70 projectId: ProjectId,
71 overrides: {
72 readonly archivedAt?: DateTime.Utc;
73 readonly deletedAt?: DateTime.Utc;
74 readonly subagentOf?: ThreadId;
75 readonly sideChatOf?: ThreadId;
76 readonly title?: string;
77 readonly settledAt?: DateTime.Utc;
78 readonly linkedPullRequest?: ThreadLinkedPullRequest;
79 readonly branchPullRequest?: ThreadLinkedPullRequest;
80 } = {},
81 ): OrchestrationV2DomainEvent => ({
82 id: EventId.make(`created:${threadId}`),
83 type: "thread.created",
84 threadId,
85 providerInstanceId,
86 occurredAt: at(0),
87 payload: {
88 createdBy: "user",
89 creationSource: "web",
90 id: threadId,
91 projectId,
92 title: overrides.title ?? threadId,
93 providerInstanceId,
94 modelSelection: { instanceId: providerInstanceId, model: "gpt-5" },
95 runtimeMode: "full-access",
96 interactionMode: "default",
97 branch: null,
98 worktreePath: null,
99 linkedPullRequest: overrides.linkedPullRequest ?? null,
100 branchPullRequest: overrides.branchPullRequest ?? null,
101 activeProviderThreadId: null,
102 lineage: overrides.subagentOf
103 ? {
104 parentThreadId: overrides.subagentOf,
105 relationshipToParent: "subagent",
106 rootThreadId: overrides.subagentOf,
107 }
108 : overrides.sideChatOf
109 ? {
110 parentThreadId: overrides.sideChatOf,
111 relationshipToParent: "fork",
112 rootThreadId: overrides.sideChatOf,
113 sideChat: true,
114 }
115 : { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId },
116 forkedFrom: null,
117 createdAt: at(0),
118 updatedAt: at(0),
119 archivedAt: overrides.archivedAt ?? null,
120 settledOverride: overrides.settledAt === undefined ? null : "settled",
121 settledAt: overrides.settledAt ?? null,
122 lastVisitedAt: null,
123 deletedAt: overrides.deletedAt ?? null,
124 },
125 });
126
127 const message = (
128 threadId: ThreadId,
129 id: string,
130 role: "user" | "assistant" | "system",
131 text: string,
132 options: {
133 readonly minute?: number;
134 readonly streaming?: boolean;
135 readonly createdBy?: "user" | "agent";
136 readonly runId?: RunId;
137 } = {},
138 ): OrchestrationV2DomainEvent => ({
139 id: EventId.make(`message:${id}`),
140 type: "message.updated",
141 threadId,
142 providerInstanceId,
143 occurredAt: at(options.minute ?? 1),
144 payload: {
145 createdBy: options.createdBy ?? (role === "user" ? "user" : "agent"),
146 creationSource: role === "user" ? "web" : "provider",
147 id: MessageId.make(id),
148 threadId,
149 runId: options.runId ?? null,
150 nodeId: null,
151 role,
152 text,
153 attachments: [],
154 streaming: options.streaming ?? false,
155 createdAt: at(options.minute ?? 1),
156 updatedAt: at(options.minute ?? 1),
157 },
158 });
159
160 // The prefix and suffix are hidden selection context, not text the reader sees.
161 const quote = serializeAssistantCitation({
162 version: 1,
163 environmentId: EnvironmentId.make("environment:local"),
164 threadId: ThreadId.make("thread:quoted"),
165 messageId: MessageId.make("quoted-answer"),
166 text: "A lighthouse guides ships safely through the darkness.",
167 comment: "what are you talking about",
168 start: 0,
169 end: 54,
170 prefix: "beacon01",
171 suffix: "harbor",
172 });
173
174 it.effect.each([true, false])(
175 "ThreadSearch excludes bootstrap text before the hit and candidate limits (indexed: %s)",
176 (indexed) =>
177 Effect.gen(function* () {
178 const projections = yield* ProjectionStore.ProjectionStoreV2;
179 const search = yield* ThreadSearch.ThreadSearch;
180 const sql = yield* SqlClient.SqlClient;
181 const project = ProjectId.make("project:attachments");
182 const threadId = ThreadId.make("thread:attachments");
183 yield* createProject(project);
184 yield* Effect.forEach(
185 [
186 thread(threadId, project),
187 message(threadId, "ordinary", "user", "Review the attached files in context", {
188 minute: 1,
189 }),
190 message(threadId, "attachment", "user", ATTACHMENT_ONLY_BOOTSTRAP_PROMPT, { minute: 2 }),
191 message(
192 threadId,
193 "ultrathink",
194 "user",
195 `Ultrathink:\n${ATTACHMENT_ONLY_BOOTSTRAP_PROMPT}`,
196 { minute: 3 },
197 ),
198 ],
199 projections.apply,
200 { discard: true },
201 );
202 if (!indexed) {
203 yield* sql`DROP TRIGGER orchestration_v2_message_search_insert`;
204 yield* sql`DROP TRIGGER orchestration_v2_message_search_update`;
205 yield* sql`DROP TRIGGER orchestration_v2_message_search_delete`;
206 yield* sql`DROP TABLE orchestration_v2_message_search`;
207 }
208 assert.deepEqual((yield* search.search({ query: "conversation context" })).matches, []);
209 assert.deepEqual((yield* search.search({ query: "ultrathink" })).matches, []);
210 assert.deepEqual(
211 (yield* search.search({ query: "attached files" })).matches.map((match) => [
212 match.messageId,
213 match.snippet,
214 ]),
215 [[MessageId.make("ordinary"), "Review the attached files in context"]],
216 );
217 for (const text of [
218 ATTACHMENT_ONLY_BOOTSTRAP_PROMPT,
219 `Ultrathink:\n${ATTACHMENT_ONLY_BOOTSTRAP_PROMPT}`,
220 `uLtRaThInK:\r\n\t\u00a0${ATTACHMENT_ONLY_BOOTSTRAP_PROMPT}`,
221 ]) {
222 yield* sql`
223 WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < ${indexed ? 10_000 : 2000})
224 INSERT INTO orchestration_v2_projection_messages
225 (message_id, thread_id, role, streaming, created_at, updated_at, payload_json)
226 SELECT 'attachment-limit:' || i, ${threadId}, 'user', 0,
227 ${DateTime.formatIso(at(5))}, ${DateTime.formatIso(at(5))}, json_object('text', ${text})
228 FROM n
229 `;
230 assert.deepEqual(
231 (yield* search.search({ query: "context" })).matches.map((match) => match.messageId),
232 [MessageId.make("ordinary")],
233 );
234 yield* sql`
235 DELETE FROM orchestration_v2_projection_messages WHERE message_id LIKE 'attachment-limit:%'
236 `;
237 }
238 }).pipe(Effect.provide(layerTest)),
239 );
240
241 it.effect("ThreadSearch excludes legacy placeholders still in the index", () =>
242 Effect.gen(function* () {
243 const projections = yield* ProjectionStore.ProjectionStoreV2;
244 const search = yield* ThreadSearch.ThreadSearch;
245 const sql = yield* SqlClient.SqlClient;
246 const project = ProjectId.make("project:legacy-attachments");
247 const threadId = ThreadId.make("thread:legacy-attachments");
248 yield* createProject(project);
249 // Simulate the pre-upgrade trigger, which indexed attachment placeholders.
250 yield* sql`DROP TRIGGER orchestration_v2_message_search_insert`;
251 yield* sql`
252 CREATE TRIGGER orchestration_v2_message_search_insert
253 AFTER INSERT ON orchestration_v2_projection_messages
254 BEGIN
255 INSERT INTO orchestration_v2_message_search (rowid, text)
256 VALUES (new.rowid, json_extract(new.payload_json, '$.text'));
257 END
258 `;
259 yield* Effect.forEach(
260 [
261 thread(threadId, project),
262 message(threadId, "ordinary", "user", "Review the attached files in context"),
263 message(threadId, "attachment", "user", ATTACHMENT_ONLY_BOOTSTRAP_PROMPT),
264 message(
265 threadId,
266 "ultrathink",
267 "user",
268 `uLtRaThInK:\r\n\t\u00a0${ATTACHMENT_ONLY_BOOTSTRAP_PROMPT}`,
269 ),
270 ],
271 projections.apply,
272 { discard: true },
273 );
274 assert.lengthOf(
275 yield* sql`SELECT rowid FROM orchestration_v2_message_search WHERE orchestration_v2_message_search MATCH '"context"'`,
276 3,
277 );
278 assert.deepEqual((yield* search.search({ query: "conversation context" })).matches, []);
279 assert.deepEqual((yield* search.search({ query: "ultrathink" })).matches, []);
280 assert.deepEqual(
281 (yield* search.search({ query: "context" })).matches.map((match) => match.messageId),
282 [MessageId.make("ordinary")],
283 );
284 }).pipe(Effect.provide(layerTest)),
285 );
286
287 it.layer(layerTest)("ThreadSearch", (it) => {
288 it.effect("reads the newest hits without sorting all of them", () =>
289 Effect.gen(function* () {
290 const sql = yield* SqlClient.SqlClient;
291 for (const match of ['"needle"', null]) {
292 const plan = yield* sql<{
293 readonly detail: string;
294 }>`EXPLAIN QUERY PLAN ${ThreadSearch.messageHitsQuery(sql, { match, substrings: "[]", shortSpellings: "[]", projectId: null, excludedThreadIds: "[]" })}`;
295 assert.isFalse(
296 plan.some((row) => row.detail.includes("TEMP B-TREE")),
297 plan.map((row) => row.detail).join("\n"),
298 );
299 if (match !== null) {
300 assert.lengthOf(
301 plan.filter((row) =>
302 row.detail.startsWith("SEARCH messages USING INTEGER PRIMARY KEY"),
303 ),
304 1,
305 plan.map((row) => row.detail).join("\n"),
306 );
307 }
308 }
309 }),
310 );
311
312 it.effect("returns one finished user or assistant match per active thread", () =>
313 Effect.gen(function* () {
314 const projections = yield* ProjectionStore.ProjectionStoreV2;
315 const search = yield* ThreadSearch.ThreadSearch;
316 const project = ProjectId.make("project:search");
317 const deletedProject = ProjectId.make("project:search-deleted");
318 yield* createProject(project);
319 yield* createProject(deletedProject);
320 yield* Effect.flatMap(ProjectStore.ProjectStoreV2, (projects) =>
321 projects.apply({
322 sequence: 0,
323 eventId: EventId.make("deleted:project"),
324 aggregateKind: "project",
325 aggregateId: deletedProject,
326 occurredAt: DateTime.formatIso(at(2)),
327 commandId: null,
328 causationEventId: null,
329 correlationId: null,
330 metadata: {},
331 type: "project.deleted",
332 payload: { projectId: deletedProject, deletedAt: DateTime.formatIso(at(2)) },
333 }),
334 );
335
336 const both = ThreadId.make("thread:both");
337 const assistantOnly = ThreadId.make("thread:assistant");
338 const events: ReadonlyArray<OrchestrationV2DomainEvent> = [
339 thread(both, project),
340 message(both, "both-assistant", "assistant", "needle from the answer", { minute: 3 }),
341 message(both, "both-user-old", "user", "older needle question", { minute: 1 }),
342 message(both, "both-user-new", "user", "newer needle question", { minute: 2 }),
343 thread(assistantOnly, project),
344 message(assistantOnly, "assistant-only", "assistant", "needle in an answer"),
345 message(assistantOnly, "assistant-streaming", "user", "needle still typing", {
346 streaming: true,
347 }),
348 message(assistantOnly, "assistant-system", "system", "needle system prompt"),
349 thread(ThreadId.make("thread:archived"), project, { archivedAt: at(1) }),
350 message(ThreadId.make("thread:archived"), "archived", "user", "needle archived"),
351 thread(ThreadId.make("thread:deleted"), project, { deletedAt: at(1) }),
352 message(ThreadId.make("thread:deleted"), "deleted", "user", "needle deleted"),
353 thread(ThreadId.make("thread:orphaned"), deletedProject),
354 message(ThreadId.make("thread:orphaned"), "orphaned", "user", "needle orphaned"),
355 ];
356 yield* Effect.forEach(events, projections.apply, { discard: true });
357
358 const result = yield* search.search({ query: "NEEDLE", limit: 20 });
359 assert.deepEqual(
360 result.matches
361 .map((match) => [match.threadId, match.source, match.snippet, match.messageId])
362 .toSorted(([left], [right]) => String(left).localeCompare(String(right))),
363 [
364 [assistantOnly, "assistant", "needle in an answer", "assistant-only"],
365 [both, "user", "newer needle question", "both-user-new"],
366 ],
367 );
368 assert.lengthOf((yield* search.search({ query: "needle", limit: 1 })).matches, 1);
369 assert.deepEqual((yield* search.search({ query: "ne%le" })).matches, []);
370 }),
371 );
372
373 it.effect("matches the readable text of quotes and their comments", () =>
374 Effect.gen(function* () {
375 const projections = yield* ProjectionStore.ProjectionStoreV2;
376 const search = yield* ThreadSearch.ThreadSearch;
377 const project = ProjectId.make("project:search-citation");
378 const threadId = ThreadId.make("thread:citation");
379 yield* createProject(project);
380 yield* Effect.forEach(
381 [
382 thread(threadId, project),
383 message(threadId, "citation", "user", `${quote} quiet silver lantern`),
384 ],
385 projections.apply,
386 { discard: true },
387 );
388 const snippets = (query: string) =>
389 Effect.map(search.search({ query }), (result) =>
390 result.matches
391 .filter((match) => match.threadId === threadId)
392 .map((match) => match.snippet),
393 );
394
395 const readable =
396 "A lighthouse guides ships safely through the darkness. Comment: what are you talking about quiet silver lantern";
397 assert.deepEqual(yield* snippets("what are you talking about"), [
398 `…${readable.slice(readable.indexOf("the darkness"))}`,
399 ]);
400 assert.deepEqual(yield* snippets("ships safely through"), [readable]);
401 assert.deepEqual(yield* snippets("beacon01"), []);
402 assert.deepEqual(yield* snippets("harbor"), []);
403
404 yield* projections.apply(message(threadId, "citation", "user", "quiet silver lantern"));
405 assert.deepEqual(yield* snippets("what are you talking about"), []);
406 assert.deepEqual(yield* snippets("lantern"), ["quiet silver lantern"]);
407 }),
408 );
409
410 it.effect("finds accented, CJK, and apostrophe words of a quote through the index", () =>
411 Effect.gen(function* () {
412 const sql = yield* SqlClient.SqlClient;
413 const projections = yield* ProjectionStore.ProjectionStoreV2;
414 const search = yield* ThreadSearch.ThreadSearch;
415 const project = ProjectId.make("project:search-citation-index");
416 const threadId = ThreadId.make("thread:citation-index");
417 // The stored link percent-encodes these words; the index must hold them as they read.
418 const encodedQuote = serializeAssistantCitation({
419 version: 1,
420 environmentId: EnvironmentId.make("environment:local"),
421 threadId: ThreadId.make("thread:quoted"),
422 messageId: MessageId.make("quoted-answer"),
423 text: "The kittiwake's résumé mentions 灯台の光 twice.",
424 comment: "naïve question в логах",
425 start: 0,
426 end: 44,
427 prefix: "",
428 suffix: "",
429 });
430 yield* createProject(project);
431 yield* Effect.forEach(
432 [thread(threadId, project), message(threadId, "citation-index", "user", encodedQuote)],
433 projections.apply,
434 { discard: true },
435 );
436 const index = yield* sql`
437 SELECT 1 FROM sqlite_master WHERE name = 'orchestration_v2_message_search'
438 `;
439 assert.lengthOf(index, 1);
440 const threadIds = (query: string) =>
441 search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId)));
442
443 for (const query of [
444 "kittiwake's",
445 "kittiwake’s",
446 "résumé",
447 "灯台の光",
448 "naïve",
449 "логах в",
450 ]) {
451 assert.deepEqual(yield* threadIds(query), [threadId], query);
452 }
453 // `t3 doctor` rebuilds the index after VACUUM; quotes must still read.
454 yield* rebuildMessageSearch(sql);
455 assert.deepEqual(yield* threadIds("résumé"), [threadId]);
456
457 yield* projections.apply(message(threadId, "citation-index", "user", "plain words"));
458 assert.deepEqual(yield* threadIds("résumé"), []);
459 assert.deepEqual(yield* threadIds("plain"), [threadId]);
460 }),
461 );
462
463 it.effect("indexes a message when it finishes streaming and follows later edits", () =>
464 Effect.gen(function* () {
465 const projections = yield* ProjectionStore.ProjectionStoreV2;
466 const search = yield* ThreadSearch.ThreadSearch;
467 const project = ProjectId.make("project:search-sync");
468 const threadId = ThreadId.make("thread:sync");
469 yield* createProject(project);
470 yield* projections.apply(thread(threadId, project));
471 const threadIds = (query: string) =>
472 search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId)));
473
474 yield* projections.apply(
475 message(threadId, "sync", "assistant", "streaming draft", { streaming: true }),
476 );
477 assert.deepEqual(yield* threadIds("draft"), []);
478
479 yield* projections.apply(message(threadId, "sync", "assistant", "finished answer"));
480 assert.deepEqual(yield* threadIds("finished"), [threadId]);
481
482 yield* projections.apply(message(threadId, "sync", "assistant", "edited reply"));
483 assert.deepEqual(yield* threadIds("finished"), []);
484 assert.deepEqual(yield* threadIds("edited"), [threadId]);
485 }),
486 );
487
488 it.effect("matches inside identifiers and skips hidden threads and short tokens", () =>
489 Effect.gen(function* () {
490 const projections = yield* ProjectionStore.ProjectionStoreV2;
491 const search = yield* ThreadSearch.ThreadSearch;
492 const project = ProjectId.make("project:search-tokens");
493 const parent = ThreadId.make("thread:tokens-parent");
494 const subagent = ThreadId.make("thread:tokens-subagent");
495 const sideChat = ThreadId.make("thread:tokens-side-chat");
496 yield* createProject(project);
497 yield* Effect.forEach(
498 [
499 thread(parent, project),
500 message(parent, "parent", "user", "Rename ThreadSearchService before shipping"),
501 thread(subagent, project, { subagentOf: parent }),
502 message(subagent, "subagent", "assistant", "ThreadSearchService renamed"),
503 thread(sideChat, project, { sideChatOf: parent, title: "ThreadSearchService question" }),
504 message(sideChat, "side-chat", "user", "Why rename ThreadSearchService?"),
505 ],
506 projections.apply,
507 { discard: true },
508 );
509
510 const result = yield* search.search({ query: "search servi" });
511 assert.deepEqual(
512 result.matches.map((match) => [match.threadId, match.snippet]),
513 [[parent, "Rename ThreadSearchService before shipping"]],
514 );
515 // "ab" alone cannot use the index; with a longer token it narrows the hits.
516 assert.deepEqual((yield* search.search({ query: "ab" })).matches, []);
517 assert.lengthOf((yield* search.search({ query: "renam be" })).matches, 1);
518 assert.lengthOf((yield* search.search({ query: "renam ab" })).matches, 0);
519 }),
520 );
521
522 it.effect("indexes neither subagent messages nor rewrites that keep the text", () =>
523 Effect.gen(function* () {
524 const sql = yield* SqlClient.SqlClient;
525 const projections = yield* ProjectionStore.ProjectionStoreV2;
526 const project = ProjectId.make("project:search-index-writes");
527 const parent = ThreadId.make("thread:index-parent");
528 const subagent = ThreadId.make("thread:index-subagent");
529 yield* createProject(project);
530 yield* Effect.forEach(
531 [
532 thread(parent, project),
533 message(parent, "index-parent", "assistant", "parent answer"),
534 thread(subagent, project, { subagentOf: parent }),
535 message(subagent, "index-subagent", "assistant", "subagent answer"),
536 ],
537 projections.apply,
538 { discard: true },
539 );
540 const indexedThreads = sql<{ readonly threadId: string }>`
541 SELECT messages.thread_id AS "threadId"
542 FROM orchestration_v2_message_search AS search
543 INNER JOIN orchestration_v2_projection_messages AS messages
544 ON messages.rowid = search.rowid
545 WHERE messages.thread_id IN (${parent}, ${subagent})
546 `.pipe(Effect.map((rows) => rows.map((row) => row.threadId)));
547 assert.deepEqual(yield* indexedThreads, [parent]);
548
549 // total_changes counts trigger writes, so 1 means the index was untouched.
550 const changes = sql<{ readonly n: number }>`SELECT total_changes() AS n`.pipe(
551 Effect.map(([row]) => row?.n ?? 0),
552 );
553 const before = yield* changes;
554 yield* sql`
555 UPDATE orchestration_v2_projection_messages
556 SET payload_json = json_set(payload_json, '$.updatedAt', '2026-09-27T01:00:00.000Z')
557 WHERE message_id = 'index-parent'
558 `;
559 assert.strictEqual((yield* changes) - before, 1);
560 assert.deepEqual(yield* indexedThreads, [parent]);
561 }),
562 );
563 });
564
565 it.layer(layerTest)("ThreadSearch filters before it ranks", (it) => {
566 it.effect("orders message matches by when they were written, not by row order", () =>
567 Effect.gen(function* () {
568 const projections = yield* ProjectionStore.ProjectionStoreV2;
569 const search = yield* ThreadSearch.ThreadSearch;
570 const project = ProjectId.make("project:written");
571 yield* createProject(project);
572 const recent = ThreadId.make("thread:written-recent");
573 const imported = ThreadId.make("thread:written-imported");
574 // An imported history lands after newer messages in row order.
575 yield* Effect.forEach(
576 [
577 thread(recent, project),
578 message(recent, "written-recent", "user", "kestrel deploy notes", { minute: 30 }),
579 thread(imported, project),
580 message(imported, "written-imported", "user", "kestrel deploy notes", { minute: 1 }),
581 ],
582 projections.apply,
583 { discard: true },
584 );
585 const { matches } = yield* search.search({ query: "kestrel" });
586 assert.deepEqual(
587 matches.map((match) => match.threadId),
588 [recent, imported],
589 );
590 }),
591 );
592
593 it.effect("ranks a thread about the word above a passing mention", () =>
594 Effect.gen(function* () {
595 const projections = yield* ProjectionStore.ProjectionStoreV2;
596 const search = yield* ThreadSearch.ThreadSearch;
597 const project = ProjectId.make("project:about");
598 yield* createProject(project);
599 const about = ThreadId.make("thread:about-osprey");
600 const mention = ThreadId.make("thread:mention-osprey");
601 yield* Effect.forEach(
602 [
603 thread(about, project),
604 ...Array.from({ length: 12 }, (_, index) =>
605 message(about, `about-osprey-${index}`, "assistant", `osprey step ${index}`, {
606 minute: 2,
607 }),
608 ),
609 thread(mention, project),
610 message(mention, "mention-osprey", "user", "also check osprey", { minute: 5 }),
611 ],
612 projections.apply,
613 { discard: true },
614 );
615 const { matches } = yield* search.search({ query: "osprey" });
616 assert.deepEqual(
617 matches.map((match) => [match.threadId, match.hits]),
618 [
619 [about, 12],
620 [mention, 1],
621 ],
622 );
623 }),
624 );
625
626 it.effect("counts a thread's hits at its best match only", () =>
627 Effect.gen(function* () {
628 const projections = yield* ProjectionStore.ProjectionStoreV2;
629 const search = yield* ThreadSearch.ThreadSearch;
630 const project = ProjectId.make("project:best-hits");
631 yield* createProject(project);
632 const clarks = ThreadId.make("thread:clarks");
633 const larks = ThreadId.make("thread:lark-many");
634 yield* Effect.forEach(
635 [
636 thread(clarks, project),
637 message(clarks, "clarks-lark", "user", "Lark once", { minute: 3 }),
638 ...Array.from({ length: 31 }, (_, index) =>
639 message(clarks, `clarks-${index}`, "assistant", "Clark the plan", { minute: 3 }),
640 ),
641 thread(larks, project),
642 ...Array.from({ length: 8 }, (_, index) =>
643 message(larks, `lark-many-${index}`, "assistant", "Lark again", { minute: 3 }),
644 ),
645 ],
646 projections.apply,
647 { discard: true },
648 );
649 const { matches } = yield* search.search({ query: "lark" });
650 assert.deepEqual(
651 matches.map((match) => [match.threadId, match.hits]),
652 [
653 [larks, 8],
654 [clarks, 1],
655 ],
656 );
657 }),
658 );
659
660 it.effect("finds a visible match behind hundreds of hidden or narrower hits", () =>
661 Effect.gen(function* () {
662 const projections = yield* ProjectionStore.ProjectionStoreV2;
663 const search = yield* ThreadSearch.ThreadSearch;
664 const project = ProjectId.make("project:crowded");
665 const parent = ThreadId.make("thread:crowded-parent");
666 const subagent = ThreadId.make("thread:crowded-subagent");
667 const older = ThreadId.make("thread:crowded-older");
668 yield* createProject(project);
669 yield* Effect.forEach(
670 [
671 thread(parent, project),
672 thread(subagent, project, { subagentOf: parent }),
673 thread(older, project),
674 ...Array.from({ length: 450 }, (_, index) =>
675 message(subagent, `subagent-${index}`, "assistant", "needle needle"),
676 ),
677 ...Array.from({ length: 450 }, (_, index) =>
678 message(older, `older-${index}`, "assistant", "pnpm 11 pnpm"),
679 ),
680 message(parent, "parent-needle", "user", "a needle in a long haystack of other words"),
681 // Agent-written notices in user messages are not indexed, so they never read as "You".
682 message(older, "older-notice", "user", "needle update from a watched pull request", {
683 createdBy: "agent",
684 }),
685 message(parent, "parent-pnpm", "user", "moved the repo to pnpm 12 today"),
686 ],
687 projections.apply,
688 { discard: true },
689 );
690
691 const threadIds = (query: string) =>
692 search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId)));
693 assert.deepEqual(yield* threadIds("needle"), [parent]);
694 assert.deepEqual(yield* threadIds("pnpm 12"), [parent]);
695 }),
696 );
697
698 it.effect("keeps a thread picked for the query when the limit cuts", () =>
699 Effect.gen(function* () {
700 const projections = yield* ProjectionStore.ProjectionStoreV2;
701 const search = yield* ThreadSearch.ThreadSearch;
702 const sql = yield* SqlClient.SqlClient;
703 const project = ProjectId.make("project:plover");
704 const busy = ThreadId.make("thread:plover-busy");
705 const picked = ThreadId.make("thread:plover-picked");
706 yield* createProject(project);
707 yield* Effect.forEach(
708 [
709 thread(busy, project),
710 message(busy, "plover-1", "user", "plover notes", { minute: 3 }),
711 message(busy, "plover-2", "user", "more plover notes", { minute: 4 }),
712 thread(picked, project),
713 message(picked, "plover-old", "user", "one plover mention", { minute: 1 }),
714 ],
715 projections.apply,
716 { discard: true },
717 );
718 const threadIds = () =>
719 search
720 .search({ query: "plover", limit: 1 })
721 .pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId)));
722 assert.deepEqual(yield* threadIds(), [busy]);
723 // A pick at time 0 would read as no pick.
724 yield* TestClock.adjust("1 day");
725 const at = DateTime.toEpochMillis(yield* DateTime.now);
726 yield* sql`
727 UPDATE orchestration_v2_projection_threads
728 SET payload_json = json_set(payload_json, '$.opens', json(${JSON.stringify({
729 openedAt: at,
730 score: 1,
731 picks: [{ query: "plover", at }],
732 })}))
733 WHERE thread_id = ${picked}
734 `;
735 assert.deepEqual(yield* threadIds(), [picked]);
736 }),
737 );
738
739 it.effect("leaves out answers in their question card and turns the thread hides", () =>
740 Effect.gen(function* () {
741 const projections = yield* ProjectionStore.ProjectionStoreV2;
742 const search = yield* ThreadSearch.ThreadSearch;
743 const project = ProjectId.make("project:answers");
744 const asked = ThreadId.make("thread:answers");
745 const rewound = ThreadId.make("thread:rewound");
746 const queued = ThreadId.make("thread:queued");
747 const stopped = ThreadId.make("thread:stopped");
748 const recovered = ThreadId.make("thread:recovered");
749 const run = (owner: ThreadId, status: "rolled_back" | "cancelled") =>
750 ({
751 id: EventId.make(`run:${owner}`),
752 type: "run.created",
753 threadId: owner,
754 runId: RunId.make(`run:${owner}`),
755 nodeId: NodeId.make(`node:${owner}`),
756 providerInstanceId,
757 occurredAt: at(2),
758 payload: {
759 id: RunId.make(`run:${owner}`),
760 threadId: owner,
761 ordinal: 1,
762 providerInstanceId,
763 modelSelection: { instanceId: providerInstanceId, model: "gpt-5" },
764 providerThreadId: null,
765 userMessageId: MessageId.make(`${owner}:prompt`),
766 rootNodeId: NodeId.make(`node:${owner}`),
767 activeAttemptId: null,
768 status,
769 requestedAt: at(1),
770 startedAt: at(1),
771 completedAt: at(2),
772 checkpointId: null,
773 contextHandoffId: null,
774 },
775 }) satisfies OrchestrationV2DomainEvent;
776 const shown = (owner: ThreadId, id: string, as: "turn_start" | "queued_turn" | "reply") =>
777 ({
778 id: EventId.make(`item:${id}`),
779 type: "turn-item.updated",
780 threadId: owner,
781 occurredAt: at(2),
782 payload: {
783 id: TurnItemId.make(id),
784 threadId: owner,
785 runId: RunId.make(`run:${owner}`),
786 nodeId: null,
787 providerThreadId: null,
788 providerTurnId: null,
789 nativeItemRef: null,
790 parentItemId: null,
791 ordinal: 1,
792 status: "completed",
793 title: null,
794 startedAt: at(1),
795 completedAt: at(1),
796 updatedAt: at(1),
797 messageId: MessageId.make(id),
798 text: "",
799 ...(as === "reply"
800 ? { type: "assistant_message", streaming: false }
801 : {
802 type: "user_message",
803 inputIntent: as,
804 attachments: [],
805 createdBy: "user",
806 creationSource: "web",
807 }),
808 },
809 }) satisfies OrchestrationV2DomainEvent;
810 yield* createProject(project);
811 yield* Effect.forEach(
812 [
813 thread(asked, project),
814 message(asked, "async-answer:request-1", "user", "use the albatross drive"),
815 thread(rewound, project),
816 run(rewound, "rolled_back"),
817 message(rewound, "rewound-reply", "assistant", "the albatross drive is ready", {
818 runId: RunId.make(`run:${rewound}`),
819 }),
820 thread(queued, project),
821 run(queued, "cancelled"),
822 message(queued, "queued-prompt", "user", "then mount the albatross drive", {
823 runId: RunId.make(`run:${queued}`),
824 }),
825 // A queued prompt usually has no item; an edited one has a queued item.
826 message(queued, "requeued-prompt", "user", "then eject the albatross drive", {
827 runId: RunId.make(`run:${queued}`),
828 }),
829 shown(queued, "requeued-prompt", "queued_turn"),
830 // A stopped turn keeps its prompt and whatever the agent already said.
831 thread(stopped, project),
832 run(stopped, "cancelled"),
833 message(stopped, "stopped-prompt", "user", "format the albatross drive", {
834 runId: RunId.make(`run:${stopped}`),
835 }),
836 shown(stopped, "stopped-prompt", "turn_start"),
837 thread(recovered, project),
838 run(recovered, "cancelled"),
839 message(recovered, "recovered-reply", "assistant", "the albatross drive is formatted", {
840 runId: RunId.make(`run:${recovered}`),
841 }),
842 shown(recovered, "recovered-reply", "reply"),
843 ],
844 projections.apply,
845 { discard: true },
846 );
847 assert.deepEqual(
848 (yield* search.search({ query: "albatross" })).matches
849 .map((match) => match.messageId)
850 .sort(),
851 [MessageId.make("recovered-reply"), MessageId.make("stopped-prompt")],
852 );
853 }),
854 );
855
856 it.effect("matches messages as they read, with links as their text", () =>
857 Effect.gen(function* () {
858 const projections = yield* ProjectionStore.ProjectionStoreV2;
859 const search = yield* ThreadSearch.ThreadSearch;
860 const project = ProjectId.make("project:heron");
861 const labeled = ThreadId.make("thread:heron-label");
862 const addressOnly = ThreadId.make("thread:heron-address");
863 const mention = ThreadId.make("thread:heron-mention");
864 yield* createProject(project);
865 yield* Effect.forEach(
866 [
867 thread(labeled, project),
868 message(labeled, "heron-label", "assistant", "See [the heron plan](https://x.dev/a) now"),
869 thread(addressOnly, project),
870 message(addressOnly, "heron-address", "user", "Read [this](https://heron.dev/b)"),
871 thread(mention, project),
872 message(
873 mention,
874 "heron-mention",
875 "user",
876 String.raw`Fix [\_\_init\_\_.py](pkg/__init__.py) and [\_café\_.py](pkg/_caf%C3%A9_.py)`,
877 ),
878 ],
879 projections.apply,
880 { discard: true },
881 );
882 const { matches } = yield* search.search({ query: "heron" });
883 assert.deepEqual(
884 matches.map((match) => [match.threadId, match.snippet]),
885 [[labeled, "See the heron plan now"]],
886 );
887 for (const query of ["__init__.py", "_café_.py"]) {
888 const mentions = yield* search.search({ query });
889 assert.deepEqual(
890 mentions.matches.map((match) => [match.threadId, match.snippet]),
891 [[mention, "Fix __init__.py and _café_.py"]],
892 );
893 }
894 }),
895 );
896
897 it.effect("reads at most the newest 10,000 index hits, however many the filters drop", () =>
898 Effect.gen(function* () {
899 const projections = yield* ProjectionStore.ProjectionStoreV2;
900 const search = yield* ThreadSearch.ThreadSearch;
901 const sql = yield* SqlClient.SqlClient;
902 const project = ProjectId.make("project:kestrel");
903 const old = ThreadId.make("thread:kestrel-old");
904 const busy = ThreadId.make("thread:kestrel-busy");
905 yield* createProject(project);
906 yield* Effect.forEach(
907 [
908 thread(old, project),
909 message(old, "kestrel-old", "user", "kestrel ci is red"),
910 thread(busy, project),
911 ],
912 projections.apply,
913 { discard: true },
914 );
915 const insertBusy = (count: number) => sql`
916 WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < ${count})
917 INSERT INTO orchestration_v2_projection_messages
918 (message_id, thread_id, role, streaming, created_at, updated_at, payload_json)
919 SELECT 'kestrel-busy-' || i, ${busy}, 'assistant', 0, ${DateTime.formatIso(at(5))},
920 ${DateTime.formatIso(at(5))}, json_object('text', 'kestrel packages')
921 FROM n
922 `;
923 const threadIds = (query: string) =>
924 search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId)));
925 yield* insertBusy(9_999);
926 assert.deepEqual(yield* threadIds("kestrel ci"), [old]);
927 yield* sql`DELETE FROM orchestration_v2_projection_messages WHERE thread_id = ${busy}`;
928 yield* insertBusy(10_000);
929 assert.deepEqual(yield* threadIds("kestrel ci"), []);
930 }),
931 );
932 });
933
934 it.effect.each([true, false])(
935 "ThreadSearch narrows Cyrillic and accent-folded Latin short words before the hit limit (indexed: %s)",
936 (indexed) =>
937 Effect.gen(function* () {
938 const projections = yield* ProjectionStore.ProjectionStoreV2;
939 const search = yield* ThreadSearch.ThreadSearch;
940 const sql = yield* SqlClient.SqlClient;
941 const project = ProjectId.make("project:short-limit");
942 const older = ThreadId.make("thread:short-older");
943 const olderFrench = ThreadId.make("thread:short-older-french");
944 const newer = ThreadId.make("thread:short-newer");
945 yield* createProject(project);
946 yield* Effect.forEach(
947 [
948 thread(older, project),
949 message(older, "older", "user", "В логах ошибка"),
950 thread(olderFrench, project),
951 message(olderFrench, "older-french", "user", "École fermée à Lyon"),
952 thread(newer, project),
953 ],
954 projections.apply,
955 { discard: true },
956 );
957 if (!indexed) {
958 yield* sql`DROP TRIGGER orchestration_v2_message_search_insert`;
959 yield* sql`DROP TRIGGER orchestration_v2_message_search_update`;
960 yield* sql`DROP TRIGGER orchestration_v2_message_search_delete`;
961 yield* sql`DROP TABLE orchestration_v2_message_search`;
962 }
963 // More newer hits for the long word than one read holds, none with the short one.
964 yield* sql`
965 WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 2000)
966 INSERT INTO orchestration_v2_projection_messages
967 (message_id, thread_id, role, streaming, created_at, updated_at, payload_json)
968 SELECT 'short-newer-' || i || '-' || word, ${newer}, 'assistant', 0,
969 ${DateTime.formatIso(at(5))}, ${DateTime.formatIso(at(5))}, json_object('text', text)
970 FROM n, (SELECT 'ru' AS word, 'ошибка здесь' AS text UNION ALL SELECT 'fr', 'fermée ici')
971 `;
972 const threadIds = (query: string) =>
973 search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId)));
974 assert.deepEqual(yield* threadIds("ошибка в"), [older]);
975 assert.deepEqual(yield* threadIds("fermée ec"), [olderFrench]);
976 }).pipe(Effect.provide(layerTest)),
977 );
978
979 // A two-character CJK word is too short for the index, so it scans even with the index.
980 it.effect.each([true, false])(
981 "ThreadSearch finds a curly apostrophe for a straight one and back (indexed: %s)",
982 (indexed) =>
983 Effect.gen(function* () {
984 const projections = yield* ProjectionStore.ProjectionStoreV2;
985 const search = yield* ThreadSearch.ThreadSearch;
986 const sql = yield* SqlClient.SqlClient;
987 const project = ProjectId.make("project:apostrophes");
988 const curly = ThreadId.make("thread:apostrophe-curly");
989 const straight = ThreadId.make("thread:apostrophe-straight");
990 yield* createProject(project);
991 yield* Effect.forEach(
992 [
993 thread(curly, project),
994 message(curly, "apostrophe-curly", "user", "部署 the gannet\u2019s nest"),
995 thread(straight, project),
996 message(straight, "apostrophe-straight", "user", "部署 a kittiwake's nest"),
997 ],
998 projections.apply,
999 { discard: true },
1000 );
1001 if (!indexed) {
1002 yield* sql`DROP TRIGGER orchestration_v2_message_search_insert`;
1003 yield* sql`DROP TRIGGER orchestration_v2_message_search_update`;
1004 yield* sql`DROP TRIGGER orchestration_v2_message_search_delete`;
1005 yield* sql`DROP TABLE orchestration_v2_message_search`;
1006 }
1007 const threadIds = (query: string) =>
1008 search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId)));
1009 for (const query of ["gannet's", "部署 gannet's"]) {
1010 assert.deepEqual(yield* threadIds(query), [curly], query);
1011 }
1012 for (const query of ["kittiwake\u2019s", "部署 kittiwake\u2019s"]) {
1013 assert.deepEqual(yield* threadIds(query), [straight], query);
1014 }
1015 }).pipe(Effect.provide(layerTest)),
1016 );
1017
1018 it.layer(layerTest)("ThreadSearch short words outside ASCII", (it) => {
1019 it.effect("finds them in any case or accent, with or without the index", () =>
1020 Effect.gen(function* () {
1021 const projections = yield* ProjectionStore.ProjectionStoreV2;
1022 const search = yield* ThreadSearch.ThreadSearch;
1023 const sql = yield* SqlClient.SqlClient;
1024 const project = ProjectId.make("project:short-non-ascii");
1025 const russian = ThreadId.make("thread:short-russian");
1026 const french = ThreadId.make("thread:short-french");
1027 yield* createProject(project);
1028 yield* Effect.forEach(
1029 [
1030 thread(russian, project),
1031 message(russian, "russian", "user", "В логах ошибка"),
1032 thread(french, project),
1033 message(french, "french", "user", "École fermée à Lyon"),
1034 message(french, "french-shout", "user", "Météo école"),
1035 message(french, "romanian", "user", "Școala închisă"),
1036 message(french, "vietnamese", "user", "Hồ Chí Minh"),
1037 ],
1038 projections.apply,
1039 { discard: true },
1040 );
1041 const threadIds = (query: string) =>
1042 search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId)));
1043
1044 const assertMatches = Effect.gen(function* () {
1045 assert.deepEqual(yield* threadIds("ошибка в"), [russian]);
1046 assert.deepEqual(yield* threadIds("fermée éc"), [french]);
1047 assert.deepEqual(yield* threadIds("ecole fermée"), [french]);
1048 assert.deepEqual(yield* threadIds("école me"), [french]);
1049 assert.deepEqual(yield* threadIds("închisă sc"), [french]);
1050 assert.deepEqual(yield* threadIds("chí ho"), [french]);
1051 // They still narrow: a word the message lacks rules it out.
1052 assert.deepEqual(yield* threadIds("ошибка я"), []);
1053 assert.deepEqual(yield* threadIds("fermée ce"), []);
1054 });
1055 yield* assertMatches;
1056 yield* sql`DROP TABLE orchestration_v2_message_search`;
1057 yield* assertMatches;
1058 }),
1059 );
1060 });
1061
1062 it.layer(layerTest)("ThreadSearch across scripts and without the index", (it) => {
1063 it.effect(
1064 "matches Korean, Japanese, and two-character Chinese words, and whole short words",
1065 () =>
1066 Effect.gen(function* () {
1067 const projections = yield* ProjectionStore.ProjectionStoreV2;
1068 const search = yield* ThreadSearch.ThreadSearch;
1069 const sql = yield* SqlClient.SqlClient;
1070 const project = ProjectId.make("project:scripts");
1071 const threadId = ThreadId.make("thread:scripts");
1072 const drafts = ThreadId.make("thread:drafts");
1073 const addresses = ThreadId.make("thread:addresses");
1074 yield* createProject(project);
1075 yield* Effect.forEach(
1076 [
1077 thread(threadId, project),
1078 message(threadId, "korean", "user", "검색 기능을 고쳐 주세요"),
1079 message(threadId, "japanese", "user", "がんばって検索します"),
1080 message(threadId, "chinese", "user", "明天部署到生产环境"),
1081 message(threadId, "fts", "assistant", "the FTS index is ready"),
1082 thread(drafts, project),
1083 message(drafts, "drafts", "assistant", "saved two drafts and shifts"),
1084 message(threadId, "camel", "assistant", "Configure enableFTS today"),
1085 message(threadId, "accented", "user", "Review Résumé indexing"),
1086 thread(addresses, project),
1087 message(addresses, "addresses", "user", "This addresses Résumé indexing"),
1088 ],
1089 projections.apply,
1090 { discard: true },
1091 );
1092 const threadIds = (query: string) =>
1093 search
1094 .search({ query })
1095 .pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId)));
1096
1097 // Short words must start a word, which camelCase and accented words do.
1098 for (const query of ["검색 기능", "がんばって", "部署", "fts", "enable"]) {
1099 assert.deepEqual(yield* threadIds(query), [threadId], query);
1100 }
1101 // "res" inside "addresses" does not hide the accented word after it.
1102 assert.sameMembers(yield* threadIds("res"), [threadId, addresses]);
1103 // A word beats the same letters inside a longer word, even in an older message.
1104 const trust = ThreadId.make("thread:trust");
1105 const rust = ThreadId.make("thread:rust");
1106 yield* Effect.forEach(
1107 [
1108 thread(rust, project),
1109 message(rust, "rust", "assistant", "Rust rewrite notes"),
1110 thread(trust, project),
1111 message(trust, "trust", "user", "We trust the build"),
1112 ],
1113 projections.apply,
1114 { discard: true },
1115 );
1116 const { matches } = yield* search.search({ query: "rust" });
1117 assert.deepEqual(
1118 matches.map((match) => [match.threadId, (match.quality ?? 1) <= 0.3]),
1119 [
1120 [rust, false],
1121 [trust, true],
1122 ],
1123 );
1124 // Without the index (an SQLite build without FTS5 trigrams) search scans instead.
1125 yield* sql`DROP TABLE orchestration_v2_message_search`;
1126 assert.deepEqual(yield* threadIds("検索"), [threadId]);
1127 assert.deepEqual(yield* threadIds("fts"), [threadId]);
1128 }),
1129 );
1130 });
1131
1132 it.layer(layerTest)("ThreadSearch.searchOlderSettled", (it) => {
1133 it.effect("returns older settled title matches in scope, then older message hits", () =>
1134 Effect.gen(function* () {
1135 const projections = yield* ProjectionStore.ProjectionStoreV2;
1136 const search = yield* ThreadSearch.ThreadSearch;
1137 const project = ProjectId.make("project:kestrel");
1138 const other = ProjectId.make("project:kestrel-other");
1139 yield* createProject(project);
1140 yield* createProject(other);
1141 const id = (name: string) => ThreadId.make(`thread:kestrel-${name}`);
1142 const old = at(10);
1143 yield* Effect.forEach(
1144 [
1145 thread(id("old"), project, { title: "Kestrel nest", settledAt: old }),
1146 thread(id("recent"), project, { title: "Kestrel nest", settledAt: at(50) }),
1147 thread(id("open"), project, { title: "Kestrel nest" }),
1148 thread(id("child"), project, { title: "Kestrel nest", subagentOf: id("old") }),
1149 thread(id("elsewhere"), other, { title: "Kestrel nest", settledAt: old }),
1150 thread(id("note"), project, { title: "Unrelated", settledAt: old }),
1151 thread(id("new-note"), project, { title: "Unrelated", settledAt: at(50) }),
1152 thread(id("archived"), project, {
1153 title: "Kestrel nest",
1154 settledAt: old,
1155 archivedAt: old,
1156 }),
1157 thread(id("archived-note"), project, {
1158 title: "Unrelated",
1159 settledAt: old,
1160 archivedAt: old,
1161 }),
1162 ],
1163 projections.apply,
1164 { discard: true },
1165 );
1166 const before = DateTime.formatIso(at(30));
1167 const messageThreadIds = [id("new-note"), id("note"), id("archived-note"), id("old")];
1168 assert.sameMembers(
1169 [...(yield* search.searchOlderSettled({ query: "kestrel", before, messageThreadIds }))],
1170 [id("old"), id("elsewhere"), id("note")],
1171 );
1172 // The scope applies to titles and message hits; message hits follow the titles in order.
1173 assert.deepEqual(
1174 yield* search.searchOlderSettled({
1175 query: "kestrel",
1176 before,
1177 projectIds: [project],
1178 messageThreadIds: [id("elsewhere"), ...messageThreadIds],
1179 }),
1180 [id("old"), id("note")],
1181 );
1182 // A palette also matches the environment's name.
1183 assert.include(
1184 yield* search.searchOlderSettled({
1185 query: "laptop unrelated",
1186 before,
1187 environmentLabel: "Laptop",
1188 messageThreadIds: [],
1189 }),
1190 id("note"),
1191 );
1192 }),
1193 );
1194
1195 it.effect("matches the branch and linked pull requests, like the palette", () =>
1196 Effect.gen(function* () {
1197 const projections = yield* ProjectionStore.ProjectionStoreV2;
1198 const search = yield* ThreadSearch.ThreadSearch;
1199 const project = ProjectId.make("project:heron");
1200 yield* createProject(project);
1201 const pullRequest = (number: number): ThreadLinkedPullRequest => ({
1202 projectId: project,
1203 repository: "acme/heron",
1204 number,
1205 url: `https://github.com/acme/heron/pull/${number}`,
1206 });
1207 const branch = ThreadId.make("thread:heron-branch");
1208 const linked = ThreadId.make("thread:heron-linked");
1209 yield* Effect.forEach(
1210 [
1211 thread(branch, project, {
1212 title: "Heron",
1213 settledAt: at(10),
1214 branchPullRequest: pullRequest(4101),
1215 }),
1216 thread(linked, project, {
1217 title: "Heron",
1218 settledAt: at(10),
1219 linkedPullRequest: pullRequest(4202),
1220 }),
1221 ],
1222 projections.apply,
1223 { discard: true },
1224 );
1225 const find = (query: string) =>
1226 search.searchOlderSettled({
1227 query,
1228 before: DateTime.formatIso(at(30)),
1229 messageThreadIds: [],
1230 });
1231 assert.deepEqual(yield* find("#4101"), [branch]);
1232 assert.deepEqual(yield* find("acme/heron#4202"), [linked]);
1233 }),
1234 );
1235 });
1236
1237 it.layer(layerTest)("ThreadSearch.searchThreadsWithOlderSettledRows", (it) => {
1238 it.effect("adds older settled rows to message hits, or alone for titles only", () =>
1239 Effect.gen(function* () {
1240 const projections = yield* ProjectionStore.ProjectionStoreV2;
1241 const search = yield* ThreadSearch.ThreadSearch;
1242 const sql = yield* SqlClient.SqlClient;
1243 const project = ProjectId.make("project:plover");
1244 yield* createProject(project);
1245 const old = ThreadId.make("thread:plover-old");
1246 const recent = ThreadId.make("thread:plover-recent");
1247 yield* Effect.forEach(
1248 [
1249 thread(old, project, { title: "Plover nest", settledAt: at(10) }),
1250 thread(recent, project, { title: "Plover nest", settledAt: at(50) }),
1251 message(recent, "plover-note", "user", "a plover note", { minute: 2 }),
1252 ],
1253 projections.apply,
1254 { discard: true },
1255 );
1256 const run = (request: { readonly titlesOnly?: boolean }) =>
1257 ThreadSearch.searchThreadsWithOlderSettledRows({
1258 request: { query: "plover", olderSettledBefore: DateTime.formatIso(at(30)), ...request },
1259 search: search.search,
1260 searchOlderSettled: search.searchOlderSettled,
1261 sql,
1262 readThreads: (threadIds) =>
1263 projections.readShellSnapshot({ location: "active", threadIds }),
1264 latestSequence: Effect.succeed(7),
1265 });
1266 const withMessages = yield* run({});
1267 assert.deepEqual(
1268 withMessages.matches.map((match) => match.threadId),
1269 [recent],
1270 );
1271 assert.equal(withMessages.snapshotSequence, 7);
1272 assert.deepEqual(
1273 withMessages.threads?.map((row) => row.id),
1274 [old],
1275 );
1276 const titles = yield* run({ titlesOnly: true });
1277 assert.deepEqual(titles.matches, []);
1278 assert.deepEqual(
1279 titles.threads?.map((row) => row.id),
1280 [old],
1281 );
1282 // Without the cut it is the plain message search.
1283 assert.isUndefined(
1284 (yield* ThreadSearch.searchThreadsWithOlderSettledRows({
1285 request: { query: "plover" },
1286 search: search.search,
1287 searchOlderSettled: search.searchOlderSettled,
1288 sql,
1289 readThreads: (threadIds) =>
1290 projections.readShellSnapshot({ location: "active", threadIds }),
1291 latestSequence: Effect.succeed(7),
1292 })).threads,
1293 );
1294 }),
1295 );
1296 });
1297
1298 it.layer(layerTest)("ThreadSearch.searchThreads", (it) => {
1299 it.effect("keeps room for a message-only thread behind many titled ones", () =>
1300 Effect.gen(function* () {
1301 const projections = yield* ProjectionStore.ProjectionStoreV2;
1302 const search = yield* ThreadSearch.ThreadSearch;
1303 const project = ProjectId.make("project:petrel");
1304 yield* createProject(project);
1305 const messageOnly = ThreadId.make("thread:petrel-message");
1306 yield* Effect.forEach(
1307 [
1308 thread(messageOnly, project),
1309 // The oldest message, so a cut taken before titles drop would lose it.
1310 message(messageOnly, "petrel-old", "user", "a petrel note", { minute: 1 }),
1311 ...Array.from({ length: 50 }, (_, index) => {
1312 const id = ThreadId.make(`thread:petrel-${index}`);
1313 return [
1314 thread(id, project, { title: `Petrel ${index}` }),
1315 message(id, `petrel-${index}`, "user", "petrel again", { minute: 2 }),
1316 ];
1317 }).flat(),
1318 ],
1319 projections.apply,
1320 { discard: true },
1321 );
1322 const { matches } = yield* search.searchThreads({ query: "petrel", limit: 5 });
1323 assert.include(
1324 matches.map((match) => match.threadId),
1325 messageOnly,
1326 );
1327 }),
1328 );
1329
1330 it.effect("leaves titled threads out before the hit limit", () =>
1331 Effect.gen(function* () {
1332 const projections = yield* ProjectionStore.ProjectionStoreV2;
1333 const search = yield* ThreadSearch.ThreadSearch;
1334 const sql = yield* SqlClient.SqlClient;
1335 const project = ProjectId.make("project:hit-limit");
1336 const messageOnly = ThreadId.make("thread:grebe-message");
1337 const titled = ThreadId.make("thread:grebe-titled");
1338 yield* createProject(project);
1339 yield* Effect.forEach(
1340 [
1341 thread(messageOnly, project, { title: "Unrelated" }),
1342 message(messageOnly, "grebe-old", "user", "a grebe note", { minute: 1 }),
1343 thread(titled, project, { title: "Grebe" }),
1344 ],
1345 projections.apply,
1346 { discard: true },
1347 );
1348 // Newer hits than one read holds, all in the titled thread.
1349 yield* sql`
1350 WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 2000)
1351 INSERT INTO orchestration_v2_projection_messages
1352 (message_id, thread_id, role, streaming, created_at, updated_at, payload_json)
1353 SELECT 'grebe-titled-' || i, ${titled}, 'assistant', 0, ${DateTime.formatIso(at(5))},
1354 ${DateTime.formatIso(at(5))}, json_object('text', 'grebe again')
1355 FROM n
1356 `;
1357 const { matches } = yield* search.searchThreads({ query: "grebe", limit: 5 });
1358 assert.include(
1359 matches.map((match) => match.threadId),
1360 messageOnly,
1361 );
1362 }),
1363 );
1364
1365 it.effect("ranks title matches like the palette, then message matches, within one project", () =>
1366 Effect.gen(function* () {
1367 const projections = yield* ProjectionStore.ProjectionStoreV2;
1368 const search = yield* ThreadSearch.ThreadSearch;
1369 const project = ProjectId.make("project:titles");
1370 const otherProject = ProjectId.make("project:titles-other");
1371 yield* createProject(project);
1372 yield* createProject(otherProject);
1373 const wordStart = ThreadId.make("thread:word-start");
1374 const midWord = ThreadId.make("thread:mid-word");
1375 const messageOnly = ThreadId.make("thread:message-only");
1376 const elsewhere = ThreadId.make("thread:elsewhere");
1377 yield* Effect.forEach(
1378 [
1379 thread(midWord, project, { title: "Fix oauth redirect" }),
1380 thread(wordStart, project, { title: "Clean up auth tokens" }),
1381 thread(messageOnly, project, { title: "Unrelated" }),
1382 message(messageOnly, "message-only", "user", "the auth flow broke again"),
1383 thread(elsewhere, otherProject, { title: "Auth everywhere" }),
1384 ],
1385 projections.apply,
1386 { discard: true },
1387 );
1388
1389 // Stronger message hits in another project must not use up this project's limit.
1390 yield* Effect.forEach(
1391 Array.from({ length: 55 }, (_, index) => {
1392 const busy = ThreadId.make(`thread:elsewhere-${index}`);
1393 return [
1394 thread(busy, otherProject),
1395 message(busy, `elsewhere-${index}`, "user", "auth auth auth"),
1396 ];
1397 }).flat(),
1398 projections.apply,
1399 { discard: true },
1400 );
1401 const result = yield* search.searchThreads({ query: "auth", projectId: project, limit: 3 });
1402 assert.deepEqual(
1403 result.matches.map((match) => [match.threadId, match.source]),
1404 [
1405 [wordStart, "title"],
1406 [midWord, "title"],
1407 [messageOnly, "user"],
1408 ],
1409 );
1410 // With one slot, an exact title keeps it.
1411 assert.deepEqual(
1412 (yield* search.searchThreads({ query: "auth", projectId: project, limit: 1 })).matches.map(
1413 (match) => match.threadId,
1414 ),
1415 [wordStart],
1416 );
1417 // Exact matching finds nothing, so one typo is forgiven.
1418 assert.deepEqual(
1419 (yield* search.searchThreads({ query: "tokesn", projectId: project })).matches.map(
1420 (match) => match.title,
1421 ),
1422 ["Clean up auth tokens"],
1423 );
1424 }),
1425 );
1426 });
1427
1428 it.effect("starts snippets a few words before the match so one line shows it", () =>
1429 Effect.sync(() => {
1430 const snippet = (text: string, word: string) =>
1431 ThreadSearch.buildSearchSnippet(text, text.indexOf(word));
1432 const long = `${"lead ".repeat(60)}the NEEDLE sits here ${"tail ".repeat(60)}`;
1433 const cut = snippet(long, "NEEDLE");
1434 assert.isAtMost(cut.length, 240);
1435 assert.match(cut, /^…(lead ){2,4}the NEEDLE sits here/);
1436 assert.isTrue(cut.endsWith("…"));
1437 // Whitespace collapses only in the shown part, so the offset still lands.
1438 assert.match(snippet(`${"word\n\n ".repeat(60)}un café`, "café"), /un café$/);
1439 // A cut never splits an emoji's surrogate pair.
1440 assert.notMatch(
1441 snippet(`x${"🚀".repeat(200)}needle`, "needle"),
1442 /[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]/,
1443 );
1444 // Emphasis and code marks go, as the thread shows the text.
1445 assert.equal(
1446 ThreadSearch.buildSearchSnippet("Use **bold** and `code` here", 0),
1447 "Use bold and code here",
1448 );
1449 assert.equal(ThreadSearch.buildSearchSnippet("Call __init__ first", 0), "Call __init__ first");
1450 // Short text still moves a late match into view.
1451 assert.equal(snippet(`${"word ".repeat(20)}needle`, "needle"), "…word word word word needle");
1452 }),
1453 );
1454
1455 it.effect("searches on the reader worker while the writer is busy", () =>
1456 Effect.gen(function* () {
1457 const fileSystem = yield* FileSystem.FileSystem;
1458 const path = yield* Path.Path;
1459 const directory = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-thread-search-" });
1460 const layerFile = Layer.mergeAll(
1461 ThreadSearch.layer,
1462 ProjectionStore.layer,
1463 ProjectStore.layer,
1464 ).pipe(
1465 Layer.provideMerge(SqlitePersistence.layerFromPath(path.join(directory, "state.sqlite"))),
1466 Layer.provide(NodeServices.layer),
1467 );
1468
1469 yield* Effect.gen(function* () {
1470 const projections = yield* ProjectionStore.ProjectionStoreV2;
1471 const search = yield* ThreadSearch.ThreadSearch;
1472 const sql = yield* SqlClient.SqlClient;
1473 const project = ProjectId.make("project:search-reader");
1474 const threadId = ThreadId.make("thread:reader");
1475 yield* createProject(project);
1476 yield* Effect.forEach(
1477 [thread(threadId, project), message(threadId, "reader", "user", "needle on the reader")],
1478 projections.apply,
1479 { discard: true },
1480 );
1481
1482 const writerHeld = yield* Deferred.make<void>();
1483 const releaseWriter = yield* Deferred.make<void>();
1484 const writer = yield* Effect.forkChild(
1485 sql.withTransaction(
1486 Deferred.succeed(writerHeld, undefined).pipe(
1487 Effect.andThen(Deferred.await(releaseWriter)),
1488 ),
1489 ),
1490 );
1491 yield* Deferred.await(writerHeld);
1492 const result = yield* search.search({ query: "needle" });
1493 yield* Deferred.succeed(releaseWriter, undefined);
1494 yield* Fiber.join(writer);
1495
1496 assert.deepEqual(
1497 result.matches.map((match) => match.threadId),
1498 [threadId],
1499 );
1500 }).pipe(Effect.provide(layerFile));
1501 }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
1502 );