The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import { assert, it } from "@effect/vitest"; |
| 2 | import { |
| 3 | EnvironmentId, |
| 4 | EventId, |
| 5 | MessageId, |
| 6 | NodeId, |
| 7 | ProjectId, |
| 8 | ProviderInstanceId, |
| 9 | RunId, |
| 10 | ThreadId, |
| 11 | TurnItemId, |
| 12 | type OrchestrationV2DomainEvent, |
| 13 | type ThreadLinkedPullRequest, |
| 14 | } from "@t3tools/contracts"; |
| 15 | import * as NodeServices from "@effect/platform-node/NodeServices"; |
| 16 | import * as DateTime from "effect/DateTime"; |
| 17 | import * as Deferred from "effect/Deferred"; |
| 18 | import * as Effect from "effect/Effect"; |
| 19 | import * as TestClock from "effect/testing/TestClock"; |
| 20 | import * as FileSystem from "effect/FileSystem"; |
| 21 | import * as Fiber from "effect/Fiber"; |
| 22 | import * as Layer from "effect/Layer"; |
| 23 | import * as Path from "effect/Path"; |
| 24 | import * as SqlClient from "effect/sql/SqlClient"; |
| 25 | import { serializeAssistantCitation } from "@t3tools/shared/assistantCitations"; |
| 26 | import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "@t3tools/shared/composerContextReferences"; |
| 27 | |
| 28 | import * as SqlitePersistence from "../persistence/Sqlite.ts"; |
| 29 | import { rebuildMessageSearch } from "./messageSearchIndex.ts"; |
| 30 | import * as ProjectionStore from "./ProjectionStore.ts"; |
| 31 | import * as ProjectStore from "./ProjectStore.ts"; |
| 32 | import * as ThreadSearch from "./ThreadSearch.ts"; |
| 33 | |
| 34 | const layerTest = Layer.mergeAll( |
| 35 | ThreadSearch.layer, |
| 36 | ProjectionStore.layer, |
| 37 | ProjectStore.layer, |
| 38 | ).pipe(Layer.provideMerge(SqlitePersistence.layerMemory)); |
| 39 | |
| 40 | const providerInstanceId = ProviderInstanceId.make("codex"); |
| 41 | const at = (minute: number) => DateTime.makeUnsafe(Date.UTC(2026, 8, 27, 0, minute)); |
| 42 | |
| 43 | const createProject = (projectId: ProjectId) => |
| 44 | Effect.flatMap(ProjectStore.ProjectStoreV2, (projects) => |
| 45 | projects.apply({ |
| 46 | sequence: 0, |
| 47 | eventId: EventId.make(`created:${projectId}`), |
| 48 | aggregateKind: "project", |
| 49 | aggregateId: projectId, |
| 50 | occurredAt: DateTime.formatIso(at(0)), |
| 51 | commandId: null, |
| 52 | causationEventId: null, |
| 53 | correlationId: null, |
| 54 | metadata: {}, |
| 55 | type: "project.created", |
| 56 | payload: { |
| 57 | projectId, |
| 58 | title: projectId, |
| 59 | workspaceRoot: `/work/${projectId}`, |
| 60 | defaultModelSelection: null, |
| 61 | scripts: [], |
| 62 | createdAt: DateTime.formatIso(at(0)), |
| 63 | updatedAt: DateTime.formatIso(at(0)), |
| 64 | }, |
| 65 | }), |
| 66 | ); |
| 67 | |
| 68 | const thread = ( |
| 69 | threadId: ThreadId, |
| 70 | projectId: ProjectId, |
| 71 | overrides: { |
| 72 | readonly archivedAt?: DateTime.Utc; |
| 73 | readonly deletedAt?: DateTime.Utc; |
| 74 | readonly subagentOf?: ThreadId; |
| 75 | readonly sideChatOf?: ThreadId; |
| 76 | readonly title?: string; |
| 77 | readonly settledAt?: DateTime.Utc; |
| 78 | readonly linkedPullRequest?: ThreadLinkedPullRequest; |
| 79 | readonly branchPullRequest?: ThreadLinkedPullRequest; |
| 80 | } = {}, |
| 81 | ): OrchestrationV2DomainEvent => ({ |
| 82 | id: EventId.make(`created:${threadId}`), |
| 83 | type: "thread.created", |
| 84 | threadId, |
| 85 | providerInstanceId, |
| 86 | occurredAt: at(0), |
| 87 | payload: { |
| 88 | createdBy: "user", |
| 89 | creationSource: "web", |
| 90 | id: threadId, |
| 91 | projectId, |
| 92 | title: overrides.title ?? threadId, |
| 93 | providerInstanceId, |
| 94 | modelSelection: { instanceId: providerInstanceId, model: "gpt-5" }, |
| 95 | runtimeMode: "full-access", |
| 96 | interactionMode: "default", |
| 97 | branch: null, |
| 98 | worktreePath: null, |
| 99 | linkedPullRequest: overrides.linkedPullRequest ?? null, |
| 100 | branchPullRequest: overrides.branchPullRequest ?? null, |
| 101 | activeProviderThreadId: null, |
| 102 | lineage: overrides.subagentOf |
| 103 | ? { |
| 104 | parentThreadId: overrides.subagentOf, |
| 105 | relationshipToParent: "subagent", |
| 106 | rootThreadId: overrides.subagentOf, |
| 107 | } |
| 108 | : overrides.sideChatOf |
| 109 | ? { |
| 110 | parentThreadId: overrides.sideChatOf, |
| 111 | relationshipToParent: "fork", |
| 112 | rootThreadId: overrides.sideChatOf, |
| 113 | sideChat: true, |
| 114 | } |
| 115 | : { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId }, |
| 116 | forkedFrom: null, |
| 117 | createdAt: at(0), |
| 118 | updatedAt: at(0), |
| 119 | archivedAt: overrides.archivedAt ?? null, |
| 120 | settledOverride: overrides.settledAt === undefined ? null : "settled", |
| 121 | settledAt: overrides.settledAt ?? null, |
| 122 | lastVisitedAt: null, |
| 123 | deletedAt: overrides.deletedAt ?? null, |
| 124 | }, |
| 125 | }); |
| 126 | |
| 127 | const message = ( |
| 128 | threadId: ThreadId, |
| 129 | id: string, |
| 130 | role: "user" | "assistant" | "system", |
| 131 | text: string, |
| 132 | options: { |
| 133 | readonly minute?: number; |
| 134 | readonly streaming?: boolean; |
| 135 | readonly createdBy?: "user" | "agent"; |
| 136 | readonly runId?: RunId; |
| 137 | } = {}, |
| 138 | ): OrchestrationV2DomainEvent => ({ |
| 139 | id: EventId.make(`message:${id}`), |
| 140 | type: "message.updated", |
| 141 | threadId, |
| 142 | providerInstanceId, |
| 143 | occurredAt: at(options.minute ?? 1), |
| 144 | payload: { |
| 145 | createdBy: options.createdBy ?? (role === "user" ? "user" : "agent"), |
| 146 | creationSource: role === "user" ? "web" : "provider", |
| 147 | id: MessageId.make(id), |
| 148 | threadId, |
| 149 | runId: options.runId ?? null, |
| 150 | nodeId: null, |
| 151 | role, |
| 152 | text, |
| 153 | attachments: [], |
| 154 | streaming: options.streaming ?? false, |
| 155 | createdAt: at(options.minute ?? 1), |
| 156 | updatedAt: at(options.minute ?? 1), |
| 157 | }, |
| 158 | }); |
| 159 | |
| 160 | // The prefix and suffix are hidden selection context, not text the reader sees. |
| 161 | const quote = serializeAssistantCitation({ |
| 162 | version: 1, |
| 163 | environmentId: EnvironmentId.make("environment:local"), |
| 164 | threadId: ThreadId.make("thread:quoted"), |
| 165 | messageId: MessageId.make("quoted-answer"), |
| 166 | text: "A lighthouse guides ships safely through the darkness.", |
| 167 | comment: "what are you talking about", |
| 168 | start: 0, |
| 169 | end: 54, |
| 170 | prefix: "beacon01", |
| 171 | suffix: "harbor", |
| 172 | }); |
| 173 | |
| 174 | it.effect.each([true, false])( |
| 175 | "ThreadSearch excludes bootstrap text before the hit and candidate limits (indexed: %s)", |
| 176 | (indexed) => |
| 177 | Effect.gen(function* () { |
| 178 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 179 | const search = yield* ThreadSearch.ThreadSearch; |
| 180 | const sql = yield* SqlClient.SqlClient; |
| 181 | const project = ProjectId.make("project:attachments"); |
| 182 | const threadId = ThreadId.make("thread:attachments"); |
| 183 | yield* createProject(project); |
| 184 | yield* Effect.forEach( |
| 185 | [ |
| 186 | thread(threadId, project), |
| 187 | message(threadId, "ordinary", "user", "Review the attached files in context", { |
| 188 | minute: 1, |
| 189 | }), |
| 190 | message(threadId, "attachment", "user", ATTACHMENT_ONLY_BOOTSTRAP_PROMPT, { minute: 2 }), |
| 191 | message( |
| 192 | threadId, |
| 193 | "ultrathink", |
| 194 | "user", |
| 195 | `Ultrathink:\n${ATTACHMENT_ONLY_BOOTSTRAP_PROMPT}`, |
| 196 | { minute: 3 }, |
| 197 | ), |
| 198 | ], |
| 199 | projections.apply, |
| 200 | { discard: true }, |
| 201 | ); |
| 202 | if (!indexed) { |
| 203 | yield* sql`DROP TRIGGER orchestration_v2_message_search_insert`; |
| 204 | yield* sql`DROP TRIGGER orchestration_v2_message_search_update`; |
| 205 | yield* sql`DROP TRIGGER orchestration_v2_message_search_delete`; |
| 206 | yield* sql`DROP TABLE orchestration_v2_message_search`; |
| 207 | } |
| 208 | assert.deepEqual((yield* search.search({ query: "conversation context" })).matches, []); |
| 209 | assert.deepEqual((yield* search.search({ query: "ultrathink" })).matches, []); |
| 210 | assert.deepEqual( |
| 211 | (yield* search.search({ query: "attached files" })).matches.map((match) => [ |
| 212 | match.messageId, |
| 213 | match.snippet, |
| 214 | ]), |
| 215 | [[MessageId.make("ordinary"), "Review the attached files in context"]], |
| 216 | ); |
| 217 | for (const text of [ |
| 218 | ATTACHMENT_ONLY_BOOTSTRAP_PROMPT, |
| 219 | `Ultrathink:\n${ATTACHMENT_ONLY_BOOTSTRAP_PROMPT}`, |
| 220 | `uLtRaThInK:\r\n\t\u00a0${ATTACHMENT_ONLY_BOOTSTRAP_PROMPT}`, |
| 221 | ]) { |
| 222 | yield* sql` |
| 223 | WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < ${indexed ? 10_000 : 2000}) |
| 224 | INSERT INTO orchestration_v2_projection_messages |
| 225 | (message_id, thread_id, role, streaming, created_at, updated_at, payload_json) |
| 226 | SELECT 'attachment-limit:' || i, ${threadId}, 'user', 0, |
| 227 | ${DateTime.formatIso(at(5))}, ${DateTime.formatIso(at(5))}, json_object('text', ${text}) |
| 228 | FROM n |
| 229 | `; |
| 230 | assert.deepEqual( |
| 231 | (yield* search.search({ query: "context" })).matches.map((match) => match.messageId), |
| 232 | [MessageId.make("ordinary")], |
| 233 | ); |
| 234 | yield* sql` |
| 235 | DELETE FROM orchestration_v2_projection_messages WHERE message_id LIKE 'attachment-limit:%' |
| 236 | `; |
| 237 | } |
| 238 | }).pipe(Effect.provide(layerTest)), |
| 239 | ); |
| 240 | |
| 241 | it.effect("ThreadSearch excludes legacy placeholders still in the index", () => |
| 242 | Effect.gen(function* () { |
| 243 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 244 | const search = yield* ThreadSearch.ThreadSearch; |
| 245 | const sql = yield* SqlClient.SqlClient; |
| 246 | const project = ProjectId.make("project:legacy-attachments"); |
| 247 | const threadId = ThreadId.make("thread:legacy-attachments"); |
| 248 | yield* createProject(project); |
| 249 | // Simulate the pre-upgrade trigger, which indexed attachment placeholders. |
| 250 | yield* sql`DROP TRIGGER orchestration_v2_message_search_insert`; |
| 251 | yield* sql` |
| 252 | CREATE TRIGGER orchestration_v2_message_search_insert |
| 253 | AFTER INSERT ON orchestration_v2_projection_messages |
| 254 | BEGIN |
| 255 | INSERT INTO orchestration_v2_message_search (rowid, text) |
| 256 | VALUES (new.rowid, json_extract(new.payload_json, '$.text')); |
| 257 | END |
| 258 | `; |
| 259 | yield* Effect.forEach( |
| 260 | [ |
| 261 | thread(threadId, project), |
| 262 | message(threadId, "ordinary", "user", "Review the attached files in context"), |
| 263 | message(threadId, "attachment", "user", ATTACHMENT_ONLY_BOOTSTRAP_PROMPT), |
| 264 | message( |
| 265 | threadId, |
| 266 | "ultrathink", |
| 267 | "user", |
| 268 | `uLtRaThInK:\r\n\t\u00a0${ATTACHMENT_ONLY_BOOTSTRAP_PROMPT}`, |
| 269 | ), |
| 270 | ], |
| 271 | projections.apply, |
| 272 | { discard: true }, |
| 273 | ); |
| 274 | assert.lengthOf( |
| 275 | yield* sql`SELECT rowid FROM orchestration_v2_message_search WHERE orchestration_v2_message_search MATCH '"context"'`, |
| 276 | 3, |
| 277 | ); |
| 278 | assert.deepEqual((yield* search.search({ query: "conversation context" })).matches, []); |
| 279 | assert.deepEqual((yield* search.search({ query: "ultrathink" })).matches, []); |
| 280 | assert.deepEqual( |
| 281 | (yield* search.search({ query: "context" })).matches.map((match) => match.messageId), |
| 282 | [MessageId.make("ordinary")], |
| 283 | ); |
| 284 | }).pipe(Effect.provide(layerTest)), |
| 285 | ); |
| 286 | |
| 287 | it.layer(layerTest)("ThreadSearch", (it) => { |
| 288 | it.effect("reads the newest hits without sorting all of them", () => |
| 289 | Effect.gen(function* () { |
| 290 | const sql = yield* SqlClient.SqlClient; |
| 291 | for (const match of ['"needle"', null]) { |
| 292 | const plan = yield* sql<{ |
| 293 | readonly detail: string; |
| 294 | }>`EXPLAIN QUERY PLAN ${ThreadSearch.messageHitsQuery(sql, { match, substrings: "[]", shortSpellings: "[]", projectId: null, excludedThreadIds: "[]" })}`; |
| 295 | assert.isFalse( |
| 296 | plan.some((row) => row.detail.includes("TEMP B-TREE")), |
| 297 | plan.map((row) => row.detail).join("\n"), |
| 298 | ); |
| 299 | if (match !== null) { |
| 300 | assert.lengthOf( |
| 301 | plan.filter((row) => |
| 302 | row.detail.startsWith("SEARCH messages USING INTEGER PRIMARY KEY"), |
| 303 | ), |
| 304 | 1, |
| 305 | plan.map((row) => row.detail).join("\n"), |
| 306 | ); |
| 307 | } |
| 308 | } |
| 309 | }), |
| 310 | ); |
| 311 | |
| 312 | it.effect("returns one finished user or assistant match per active thread", () => |
| 313 | Effect.gen(function* () { |
| 314 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 315 | const search = yield* ThreadSearch.ThreadSearch; |
| 316 | const project = ProjectId.make("project:search"); |
| 317 | const deletedProject = ProjectId.make("project:search-deleted"); |
| 318 | yield* createProject(project); |
| 319 | yield* createProject(deletedProject); |
| 320 | yield* Effect.flatMap(ProjectStore.ProjectStoreV2, (projects) => |
| 321 | projects.apply({ |
| 322 | sequence: 0, |
| 323 | eventId: EventId.make("deleted:project"), |
| 324 | aggregateKind: "project", |
| 325 | aggregateId: deletedProject, |
| 326 | occurredAt: DateTime.formatIso(at(2)), |
| 327 | commandId: null, |
| 328 | causationEventId: null, |
| 329 | correlationId: null, |
| 330 | metadata: {}, |
| 331 | type: "project.deleted", |
| 332 | payload: { projectId: deletedProject, deletedAt: DateTime.formatIso(at(2)) }, |
| 333 | }), |
| 334 | ); |
| 335 | |
| 336 | const both = ThreadId.make("thread:both"); |
| 337 | const assistantOnly = ThreadId.make("thread:assistant"); |
| 338 | const events: ReadonlyArray<OrchestrationV2DomainEvent> = [ |
| 339 | thread(both, project), |
| 340 | message(both, "both-assistant", "assistant", "needle from the answer", { minute: 3 }), |
| 341 | message(both, "both-user-old", "user", "older needle question", { minute: 1 }), |
| 342 | message(both, "both-user-new", "user", "newer needle question", { minute: 2 }), |
| 343 | thread(assistantOnly, project), |
| 344 | message(assistantOnly, "assistant-only", "assistant", "needle in an answer"), |
| 345 | message(assistantOnly, "assistant-streaming", "user", "needle still typing", { |
| 346 | streaming: true, |
| 347 | }), |
| 348 | message(assistantOnly, "assistant-system", "system", "needle system prompt"), |
| 349 | thread(ThreadId.make("thread:archived"), project, { archivedAt: at(1) }), |
| 350 | message(ThreadId.make("thread:archived"), "archived", "user", "needle archived"), |
| 351 | thread(ThreadId.make("thread:deleted"), project, { deletedAt: at(1) }), |
| 352 | message(ThreadId.make("thread:deleted"), "deleted", "user", "needle deleted"), |
| 353 | thread(ThreadId.make("thread:orphaned"), deletedProject), |
| 354 | message(ThreadId.make("thread:orphaned"), "orphaned", "user", "needle orphaned"), |
| 355 | ]; |
| 356 | yield* Effect.forEach(events, projections.apply, { discard: true }); |
| 357 | |
| 358 | const result = yield* search.search({ query: "NEEDLE", limit: 20 }); |
| 359 | assert.deepEqual( |
| 360 | result.matches |
| 361 | .map((match) => [match.threadId, match.source, match.snippet, match.messageId]) |
| 362 | .toSorted(([left], [right]) => String(left).localeCompare(String(right))), |
| 363 | [ |
| 364 | [assistantOnly, "assistant", "needle in an answer", "assistant-only"], |
| 365 | [both, "user", "newer needle question", "both-user-new"], |
| 366 | ], |
| 367 | ); |
| 368 | assert.lengthOf((yield* search.search({ query: "needle", limit: 1 })).matches, 1); |
| 369 | assert.deepEqual((yield* search.search({ query: "ne%le" })).matches, []); |
| 370 | }), |
| 371 | ); |
| 372 | |
| 373 | it.effect("matches the readable text of quotes and their comments", () => |
| 374 | Effect.gen(function* () { |
| 375 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 376 | const search = yield* ThreadSearch.ThreadSearch; |
| 377 | const project = ProjectId.make("project:search-citation"); |
| 378 | const threadId = ThreadId.make("thread:citation"); |
| 379 | yield* createProject(project); |
| 380 | yield* Effect.forEach( |
| 381 | [ |
| 382 | thread(threadId, project), |
| 383 | message(threadId, "citation", "user", `${quote} quiet silver lantern`), |
| 384 | ], |
| 385 | projections.apply, |
| 386 | { discard: true }, |
| 387 | ); |
| 388 | const snippets = (query: string) => |
| 389 | Effect.map(search.search({ query }), (result) => |
| 390 | result.matches |
| 391 | .filter((match) => match.threadId === threadId) |
| 392 | .map((match) => match.snippet), |
| 393 | ); |
| 394 | |
| 395 | const readable = |
| 396 | "A lighthouse guides ships safely through the darkness. Comment: what are you talking about quiet silver lantern"; |
| 397 | assert.deepEqual(yield* snippets("what are you talking about"), [ |
| 398 | `…${readable.slice(readable.indexOf("the darkness"))}`, |
| 399 | ]); |
| 400 | assert.deepEqual(yield* snippets("ships safely through"), [readable]); |
| 401 | assert.deepEqual(yield* snippets("beacon01"), []); |
| 402 | assert.deepEqual(yield* snippets("harbor"), []); |
| 403 | |
| 404 | yield* projections.apply(message(threadId, "citation", "user", "quiet silver lantern")); |
| 405 | assert.deepEqual(yield* snippets("what are you talking about"), []); |
| 406 | assert.deepEqual(yield* snippets("lantern"), ["quiet silver lantern"]); |
| 407 | }), |
| 408 | ); |
| 409 | |
| 410 | it.effect("finds accented, CJK, and apostrophe words of a quote through the index", () => |
| 411 | Effect.gen(function* () { |
| 412 | const sql = yield* SqlClient.SqlClient; |
| 413 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 414 | const search = yield* ThreadSearch.ThreadSearch; |
| 415 | const project = ProjectId.make("project:search-citation-index"); |
| 416 | const threadId = ThreadId.make("thread:citation-index"); |
| 417 | // The stored link percent-encodes these words; the index must hold them as they read. |
| 418 | const encodedQuote = serializeAssistantCitation({ |
| 419 | version: 1, |
| 420 | environmentId: EnvironmentId.make("environment:local"), |
| 421 | threadId: ThreadId.make("thread:quoted"), |
| 422 | messageId: MessageId.make("quoted-answer"), |
| 423 | text: "The kittiwake's résumé mentions 灯台の光 twice.", |
| 424 | comment: "naïve question в логах", |
| 425 | start: 0, |
| 426 | end: 44, |
| 427 | prefix: "", |
| 428 | suffix: "", |
| 429 | }); |
| 430 | yield* createProject(project); |
| 431 | yield* Effect.forEach( |
| 432 | [thread(threadId, project), message(threadId, "citation-index", "user", encodedQuote)], |
| 433 | projections.apply, |
| 434 | { discard: true }, |
| 435 | ); |
| 436 | const index = yield* sql` |
| 437 | SELECT 1 FROM sqlite_master WHERE name = 'orchestration_v2_message_search' |
| 438 | `; |
| 439 | assert.lengthOf(index, 1); |
| 440 | const threadIds = (query: string) => |
| 441 | search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId))); |
| 442 | |
| 443 | for (const query of [ |
| 444 | "kittiwake's", |
| 445 | "kittiwake’s", |
| 446 | "résumé", |
| 447 | "灯台の光", |
| 448 | "naïve", |
| 449 | "логах в", |
| 450 | ]) { |
| 451 | assert.deepEqual(yield* threadIds(query), [threadId], query); |
| 452 | } |
| 453 | // `t3 doctor` rebuilds the index after VACUUM; quotes must still read. |
| 454 | yield* rebuildMessageSearch(sql); |
| 455 | assert.deepEqual(yield* threadIds("résumé"), [threadId]); |
| 456 | |
| 457 | yield* projections.apply(message(threadId, "citation-index", "user", "plain words")); |
| 458 | assert.deepEqual(yield* threadIds("résumé"), []); |
| 459 | assert.deepEqual(yield* threadIds("plain"), [threadId]); |
| 460 | }), |
| 461 | ); |
| 462 | |
| 463 | it.effect("indexes a message when it finishes streaming and follows later edits", () => |
| 464 | Effect.gen(function* () { |
| 465 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 466 | const search = yield* ThreadSearch.ThreadSearch; |
| 467 | const project = ProjectId.make("project:search-sync"); |
| 468 | const threadId = ThreadId.make("thread:sync"); |
| 469 | yield* createProject(project); |
| 470 | yield* projections.apply(thread(threadId, project)); |
| 471 | const threadIds = (query: string) => |
| 472 | search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId))); |
| 473 | |
| 474 | yield* projections.apply( |
| 475 | message(threadId, "sync", "assistant", "streaming draft", { streaming: true }), |
| 476 | ); |
| 477 | assert.deepEqual(yield* threadIds("draft"), []); |
| 478 | |
| 479 | yield* projections.apply(message(threadId, "sync", "assistant", "finished answer")); |
| 480 | assert.deepEqual(yield* threadIds("finished"), [threadId]); |
| 481 | |
| 482 | yield* projections.apply(message(threadId, "sync", "assistant", "edited reply")); |
| 483 | assert.deepEqual(yield* threadIds("finished"), []); |
| 484 | assert.deepEqual(yield* threadIds("edited"), [threadId]); |
| 485 | }), |
| 486 | ); |
| 487 | |
| 488 | it.effect("matches inside identifiers and skips hidden threads and short tokens", () => |
| 489 | Effect.gen(function* () { |
| 490 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 491 | const search = yield* ThreadSearch.ThreadSearch; |
| 492 | const project = ProjectId.make("project:search-tokens"); |
| 493 | const parent = ThreadId.make("thread:tokens-parent"); |
| 494 | const subagent = ThreadId.make("thread:tokens-subagent"); |
| 495 | const sideChat = ThreadId.make("thread:tokens-side-chat"); |
| 496 | yield* createProject(project); |
| 497 | yield* Effect.forEach( |
| 498 | [ |
| 499 | thread(parent, project), |
| 500 | message(parent, "parent", "user", "Rename ThreadSearchService before shipping"), |
| 501 | thread(subagent, project, { subagentOf: parent }), |
| 502 | message(subagent, "subagent", "assistant", "ThreadSearchService renamed"), |
| 503 | thread(sideChat, project, { sideChatOf: parent, title: "ThreadSearchService question" }), |
| 504 | message(sideChat, "side-chat", "user", "Why rename ThreadSearchService?"), |
| 505 | ], |
| 506 | projections.apply, |
| 507 | { discard: true }, |
| 508 | ); |
| 509 | |
| 510 | const result = yield* search.search({ query: "search servi" }); |
| 511 | assert.deepEqual( |
| 512 | result.matches.map((match) => [match.threadId, match.snippet]), |
| 513 | [[parent, "Rename ThreadSearchService before shipping"]], |
| 514 | ); |
| 515 | // "ab" alone cannot use the index; with a longer token it narrows the hits. |
| 516 | assert.deepEqual((yield* search.search({ query: "ab" })).matches, []); |
| 517 | assert.lengthOf((yield* search.search({ query: "renam be" })).matches, 1); |
| 518 | assert.lengthOf((yield* search.search({ query: "renam ab" })).matches, 0); |
| 519 | }), |
| 520 | ); |
| 521 | |
| 522 | it.effect("indexes neither subagent messages nor rewrites that keep the text", () => |
| 523 | Effect.gen(function* () { |
| 524 | const sql = yield* SqlClient.SqlClient; |
| 525 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 526 | const project = ProjectId.make("project:search-index-writes"); |
| 527 | const parent = ThreadId.make("thread:index-parent"); |
| 528 | const subagent = ThreadId.make("thread:index-subagent"); |
| 529 | yield* createProject(project); |
| 530 | yield* Effect.forEach( |
| 531 | [ |
| 532 | thread(parent, project), |
| 533 | message(parent, "index-parent", "assistant", "parent answer"), |
| 534 | thread(subagent, project, { subagentOf: parent }), |
| 535 | message(subagent, "index-subagent", "assistant", "subagent answer"), |
| 536 | ], |
| 537 | projections.apply, |
| 538 | { discard: true }, |
| 539 | ); |
| 540 | const indexedThreads = sql<{ readonly threadId: string }>` |
| 541 | SELECT messages.thread_id AS "threadId" |
| 542 | FROM orchestration_v2_message_search AS search |
| 543 | INNER JOIN orchestration_v2_projection_messages AS messages |
| 544 | ON messages.rowid = search.rowid |
| 545 | WHERE messages.thread_id IN (${parent}, ${subagent}) |
| 546 | `.pipe(Effect.map((rows) => rows.map((row) => row.threadId))); |
| 547 | assert.deepEqual(yield* indexedThreads, [parent]); |
| 548 | |
| 549 | // total_changes counts trigger writes, so 1 means the index was untouched. |
| 550 | const changes = sql<{ readonly n: number }>`SELECT total_changes() AS n`.pipe( |
| 551 | Effect.map(([row]) => row?.n ?? 0), |
| 552 | ); |
| 553 | const before = yield* changes; |
| 554 | yield* sql` |
| 555 | UPDATE orchestration_v2_projection_messages |
| 556 | SET payload_json = json_set(payload_json, '$.updatedAt', '2026-09-27T01:00:00.000Z') |
| 557 | WHERE message_id = 'index-parent' |
| 558 | `; |
| 559 | assert.strictEqual((yield* changes) - before, 1); |
| 560 | assert.deepEqual(yield* indexedThreads, [parent]); |
| 561 | }), |
| 562 | ); |
| 563 | }); |
| 564 | |
| 565 | it.layer(layerTest)("ThreadSearch filters before it ranks", (it) => { |
| 566 | it.effect("orders message matches by when they were written, not by row order", () => |
| 567 | Effect.gen(function* () { |
| 568 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 569 | const search = yield* ThreadSearch.ThreadSearch; |
| 570 | const project = ProjectId.make("project:written"); |
| 571 | yield* createProject(project); |
| 572 | const recent = ThreadId.make("thread:written-recent"); |
| 573 | const imported = ThreadId.make("thread:written-imported"); |
| 574 | // An imported history lands after newer messages in row order. |
| 575 | yield* Effect.forEach( |
| 576 | [ |
| 577 | thread(recent, project), |
| 578 | message(recent, "written-recent", "user", "kestrel deploy notes", { minute: 30 }), |
| 579 | thread(imported, project), |
| 580 | message(imported, "written-imported", "user", "kestrel deploy notes", { minute: 1 }), |
| 581 | ], |
| 582 | projections.apply, |
| 583 | { discard: true }, |
| 584 | ); |
| 585 | const { matches } = yield* search.search({ query: "kestrel" }); |
| 586 | assert.deepEqual( |
| 587 | matches.map((match) => match.threadId), |
| 588 | [recent, imported], |
| 589 | ); |
| 590 | }), |
| 591 | ); |
| 592 | |
| 593 | it.effect("ranks a thread about the word above a passing mention", () => |
| 594 | Effect.gen(function* () { |
| 595 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 596 | const search = yield* ThreadSearch.ThreadSearch; |
| 597 | const project = ProjectId.make("project:about"); |
| 598 | yield* createProject(project); |
| 599 | const about = ThreadId.make("thread:about-osprey"); |
| 600 | const mention = ThreadId.make("thread:mention-osprey"); |
| 601 | yield* Effect.forEach( |
| 602 | [ |
| 603 | thread(about, project), |
| 604 | ...Array.from({ length: 12 }, (_, index) => |
| 605 | message(about, `about-osprey-${index}`, "assistant", `osprey step ${index}`, { |
| 606 | minute: 2, |
| 607 | }), |
| 608 | ), |
| 609 | thread(mention, project), |
| 610 | message(mention, "mention-osprey", "user", "also check osprey", { minute: 5 }), |
| 611 | ], |
| 612 | projections.apply, |
| 613 | { discard: true }, |
| 614 | ); |
| 615 | const { matches } = yield* search.search({ query: "osprey" }); |
| 616 | assert.deepEqual( |
| 617 | matches.map((match) => [match.threadId, match.hits]), |
| 618 | [ |
| 619 | [about, 12], |
| 620 | [mention, 1], |
| 621 | ], |
| 622 | ); |
| 623 | }), |
| 624 | ); |
| 625 | |
| 626 | it.effect("counts a thread's hits at its best match only", () => |
| 627 | Effect.gen(function* () { |
| 628 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 629 | const search = yield* ThreadSearch.ThreadSearch; |
| 630 | const project = ProjectId.make("project:best-hits"); |
| 631 | yield* createProject(project); |
| 632 | const clarks = ThreadId.make("thread:clarks"); |
| 633 | const larks = ThreadId.make("thread:lark-many"); |
| 634 | yield* Effect.forEach( |
| 635 | [ |
| 636 | thread(clarks, project), |
| 637 | message(clarks, "clarks-lark", "user", "Lark once", { minute: 3 }), |
| 638 | ...Array.from({ length: 31 }, (_, index) => |
| 639 | message(clarks, `clarks-${index}`, "assistant", "Clark the plan", { minute: 3 }), |
| 640 | ), |
| 641 | thread(larks, project), |
| 642 | ...Array.from({ length: 8 }, (_, index) => |
| 643 | message(larks, `lark-many-${index}`, "assistant", "Lark again", { minute: 3 }), |
| 644 | ), |
| 645 | ], |
| 646 | projections.apply, |
| 647 | { discard: true }, |
| 648 | ); |
| 649 | const { matches } = yield* search.search({ query: "lark" }); |
| 650 | assert.deepEqual( |
| 651 | matches.map((match) => [match.threadId, match.hits]), |
| 652 | [ |
| 653 | [larks, 8], |
| 654 | [clarks, 1], |
| 655 | ], |
| 656 | ); |
| 657 | }), |
| 658 | ); |
| 659 | |
| 660 | it.effect("finds a visible match behind hundreds of hidden or narrower hits", () => |
| 661 | Effect.gen(function* () { |
| 662 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 663 | const search = yield* ThreadSearch.ThreadSearch; |
| 664 | const project = ProjectId.make("project:crowded"); |
| 665 | const parent = ThreadId.make("thread:crowded-parent"); |
| 666 | const subagent = ThreadId.make("thread:crowded-subagent"); |
| 667 | const older = ThreadId.make("thread:crowded-older"); |
| 668 | yield* createProject(project); |
| 669 | yield* Effect.forEach( |
| 670 | [ |
| 671 | thread(parent, project), |
| 672 | thread(subagent, project, { subagentOf: parent }), |
| 673 | thread(older, project), |
| 674 | ...Array.from({ length: 450 }, (_, index) => |
| 675 | message(subagent, `subagent-${index}`, "assistant", "needle needle"), |
| 676 | ), |
| 677 | ...Array.from({ length: 450 }, (_, index) => |
| 678 | message(older, `older-${index}`, "assistant", "pnpm 11 pnpm"), |
| 679 | ), |
| 680 | message(parent, "parent-needle", "user", "a needle in a long haystack of other words"), |
| 681 | // Agent-written notices in user messages are not indexed, so they never read as "You". |
| 682 | message(older, "older-notice", "user", "needle update from a watched pull request", { |
| 683 | createdBy: "agent", |
| 684 | }), |
| 685 | message(parent, "parent-pnpm", "user", "moved the repo to pnpm 12 today"), |
| 686 | ], |
| 687 | projections.apply, |
| 688 | { discard: true }, |
| 689 | ); |
| 690 | |
| 691 | const threadIds = (query: string) => |
| 692 | search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId))); |
| 693 | assert.deepEqual(yield* threadIds("needle"), [parent]); |
| 694 | assert.deepEqual(yield* threadIds("pnpm 12"), [parent]); |
| 695 | }), |
| 696 | ); |
| 697 | |
| 698 | it.effect("keeps a thread picked for the query when the limit cuts", () => |
| 699 | Effect.gen(function* () { |
| 700 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 701 | const search = yield* ThreadSearch.ThreadSearch; |
| 702 | const sql = yield* SqlClient.SqlClient; |
| 703 | const project = ProjectId.make("project:plover"); |
| 704 | const busy = ThreadId.make("thread:plover-busy"); |
| 705 | const picked = ThreadId.make("thread:plover-picked"); |
| 706 | yield* createProject(project); |
| 707 | yield* Effect.forEach( |
| 708 | [ |
| 709 | thread(busy, project), |
| 710 | message(busy, "plover-1", "user", "plover notes", { minute: 3 }), |
| 711 | message(busy, "plover-2", "user", "more plover notes", { minute: 4 }), |
| 712 | thread(picked, project), |
| 713 | message(picked, "plover-old", "user", "one plover mention", { minute: 1 }), |
| 714 | ], |
| 715 | projections.apply, |
| 716 | { discard: true }, |
| 717 | ); |
| 718 | const threadIds = () => |
| 719 | search |
| 720 | .search({ query: "plover", limit: 1 }) |
| 721 | .pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId))); |
| 722 | assert.deepEqual(yield* threadIds(), [busy]); |
| 723 | // A pick at time 0 would read as no pick. |
| 724 | yield* TestClock.adjust("1 day"); |
| 725 | const at = DateTime.toEpochMillis(yield* DateTime.now); |
| 726 | yield* sql` |
| 727 | UPDATE orchestration_v2_projection_threads |
| 728 | SET payload_json = json_set(payload_json, '$.opens', json(${JSON.stringify({ |
| 729 | openedAt: at, |
| 730 | score: 1, |
| 731 | picks: [{ query: "plover", at }], |
| 732 | })})) |
| 733 | WHERE thread_id = ${picked} |
| 734 | `; |
| 735 | assert.deepEqual(yield* threadIds(), [picked]); |
| 736 | }), |
| 737 | ); |
| 738 | |
| 739 | it.effect("leaves out answers in their question card and turns the thread hides", () => |
| 740 | Effect.gen(function* () { |
| 741 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 742 | const search = yield* ThreadSearch.ThreadSearch; |
| 743 | const project = ProjectId.make("project:answers"); |
| 744 | const asked = ThreadId.make("thread:answers"); |
| 745 | const rewound = ThreadId.make("thread:rewound"); |
| 746 | const queued = ThreadId.make("thread:queued"); |
| 747 | const stopped = ThreadId.make("thread:stopped"); |
| 748 | const recovered = ThreadId.make("thread:recovered"); |
| 749 | const run = (owner: ThreadId, status: "rolled_back" | "cancelled") => |
| 750 | ({ |
| 751 | id: EventId.make(`run:${owner}`), |
| 752 | type: "run.created", |
| 753 | threadId: owner, |
| 754 | runId: RunId.make(`run:${owner}`), |
| 755 | nodeId: NodeId.make(`node:${owner}`), |
| 756 | providerInstanceId, |
| 757 | occurredAt: at(2), |
| 758 | payload: { |
| 759 | id: RunId.make(`run:${owner}`), |
| 760 | threadId: owner, |
| 761 | ordinal: 1, |
| 762 | providerInstanceId, |
| 763 | modelSelection: { instanceId: providerInstanceId, model: "gpt-5" }, |
| 764 | providerThreadId: null, |
| 765 | userMessageId: MessageId.make(`${owner}:prompt`), |
| 766 | rootNodeId: NodeId.make(`node:${owner}`), |
| 767 | activeAttemptId: null, |
| 768 | status, |
| 769 | requestedAt: at(1), |
| 770 | startedAt: at(1), |
| 771 | completedAt: at(2), |
| 772 | checkpointId: null, |
| 773 | contextHandoffId: null, |
| 774 | }, |
| 775 | }) satisfies OrchestrationV2DomainEvent; |
| 776 | const shown = (owner: ThreadId, id: string, as: "turn_start" | "queued_turn" | "reply") => |
| 777 | ({ |
| 778 | id: EventId.make(`item:${id}`), |
| 779 | type: "turn-item.updated", |
| 780 | threadId: owner, |
| 781 | occurredAt: at(2), |
| 782 | payload: { |
| 783 | id: TurnItemId.make(id), |
| 784 | threadId: owner, |
| 785 | runId: RunId.make(`run:${owner}`), |
| 786 | nodeId: null, |
| 787 | providerThreadId: null, |
| 788 | providerTurnId: null, |
| 789 | nativeItemRef: null, |
| 790 | parentItemId: null, |
| 791 | ordinal: 1, |
| 792 | status: "completed", |
| 793 | title: null, |
| 794 | startedAt: at(1), |
| 795 | completedAt: at(1), |
| 796 | updatedAt: at(1), |
| 797 | messageId: MessageId.make(id), |
| 798 | text: "", |
| 799 | ...(as === "reply" |
| 800 | ? { type: "assistant_message", streaming: false } |
| 801 | : { |
| 802 | type: "user_message", |
| 803 | inputIntent: as, |
| 804 | attachments: [], |
| 805 | createdBy: "user", |
| 806 | creationSource: "web", |
| 807 | }), |
| 808 | }, |
| 809 | }) satisfies OrchestrationV2DomainEvent; |
| 810 | yield* createProject(project); |
| 811 | yield* Effect.forEach( |
| 812 | [ |
| 813 | thread(asked, project), |
| 814 | message(asked, "async-answer:request-1", "user", "use the albatross drive"), |
| 815 | thread(rewound, project), |
| 816 | run(rewound, "rolled_back"), |
| 817 | message(rewound, "rewound-reply", "assistant", "the albatross drive is ready", { |
| 818 | runId: RunId.make(`run:${rewound}`), |
| 819 | }), |
| 820 | thread(queued, project), |
| 821 | run(queued, "cancelled"), |
| 822 | message(queued, "queued-prompt", "user", "then mount the albatross drive", { |
| 823 | runId: RunId.make(`run:${queued}`), |
| 824 | }), |
| 825 | // A queued prompt usually has no item; an edited one has a queued item. |
| 826 | message(queued, "requeued-prompt", "user", "then eject the albatross drive", { |
| 827 | runId: RunId.make(`run:${queued}`), |
| 828 | }), |
| 829 | shown(queued, "requeued-prompt", "queued_turn"), |
| 830 | // A stopped turn keeps its prompt and whatever the agent already said. |
| 831 | thread(stopped, project), |
| 832 | run(stopped, "cancelled"), |
| 833 | message(stopped, "stopped-prompt", "user", "format the albatross drive", { |
| 834 | runId: RunId.make(`run:${stopped}`), |
| 835 | }), |
| 836 | shown(stopped, "stopped-prompt", "turn_start"), |
| 837 | thread(recovered, project), |
| 838 | run(recovered, "cancelled"), |
| 839 | message(recovered, "recovered-reply", "assistant", "the albatross drive is formatted", { |
| 840 | runId: RunId.make(`run:${recovered}`), |
| 841 | }), |
| 842 | shown(recovered, "recovered-reply", "reply"), |
| 843 | ], |
| 844 | projections.apply, |
| 845 | { discard: true }, |
| 846 | ); |
| 847 | assert.deepEqual( |
| 848 | (yield* search.search({ query: "albatross" })).matches |
| 849 | .map((match) => match.messageId) |
| 850 | .sort(), |
| 851 | [MessageId.make("recovered-reply"), MessageId.make("stopped-prompt")], |
| 852 | ); |
| 853 | }), |
| 854 | ); |
| 855 | |
| 856 | it.effect("matches messages as they read, with links as their text", () => |
| 857 | Effect.gen(function* () { |
| 858 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 859 | const search = yield* ThreadSearch.ThreadSearch; |
| 860 | const project = ProjectId.make("project:heron"); |
| 861 | const labeled = ThreadId.make("thread:heron-label"); |
| 862 | const addressOnly = ThreadId.make("thread:heron-address"); |
| 863 | const mention = ThreadId.make("thread:heron-mention"); |
| 864 | yield* createProject(project); |
| 865 | yield* Effect.forEach( |
| 866 | [ |
| 867 | thread(labeled, project), |
| 868 | message(labeled, "heron-label", "assistant", "See [the heron plan](https://x.dev/a) now"), |
| 869 | thread(addressOnly, project), |
| 870 | message(addressOnly, "heron-address", "user", "Read [this](https://heron.dev/b)"), |
| 871 | thread(mention, project), |
| 872 | message( |
| 873 | mention, |
| 874 | "heron-mention", |
| 875 | "user", |
| 876 | String.raw`Fix [\_\_init\_\_.py](pkg/__init__.py) and [\_café\_.py](pkg/_caf%C3%A9_.py)`, |
| 877 | ), |
| 878 | ], |
| 879 | projections.apply, |
| 880 | { discard: true }, |
| 881 | ); |
| 882 | const { matches } = yield* search.search({ query: "heron" }); |
| 883 | assert.deepEqual( |
| 884 | matches.map((match) => [match.threadId, match.snippet]), |
| 885 | [[labeled, "See the heron plan now"]], |
| 886 | ); |
| 887 | for (const query of ["__init__.py", "_café_.py"]) { |
| 888 | const mentions = yield* search.search({ query }); |
| 889 | assert.deepEqual( |
| 890 | mentions.matches.map((match) => [match.threadId, match.snippet]), |
| 891 | [[mention, "Fix __init__.py and _café_.py"]], |
| 892 | ); |
| 893 | } |
| 894 | }), |
| 895 | ); |
| 896 | |
| 897 | it.effect("reads at most the newest 10,000 index hits, however many the filters drop", () => |
| 898 | Effect.gen(function* () { |
| 899 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 900 | const search = yield* ThreadSearch.ThreadSearch; |
| 901 | const sql = yield* SqlClient.SqlClient; |
| 902 | const project = ProjectId.make("project:kestrel"); |
| 903 | const old = ThreadId.make("thread:kestrel-old"); |
| 904 | const busy = ThreadId.make("thread:kestrel-busy"); |
| 905 | yield* createProject(project); |
| 906 | yield* Effect.forEach( |
| 907 | [ |
| 908 | thread(old, project), |
| 909 | message(old, "kestrel-old", "user", "kestrel ci is red"), |
| 910 | thread(busy, project), |
| 911 | ], |
| 912 | projections.apply, |
| 913 | { discard: true }, |
| 914 | ); |
| 915 | const insertBusy = (count: number) => sql` |
| 916 | WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < ${count}) |
| 917 | INSERT INTO orchestration_v2_projection_messages |
| 918 | (message_id, thread_id, role, streaming, created_at, updated_at, payload_json) |
| 919 | SELECT 'kestrel-busy-' || i, ${busy}, 'assistant', 0, ${DateTime.formatIso(at(5))}, |
| 920 | ${DateTime.formatIso(at(5))}, json_object('text', 'kestrel packages') |
| 921 | FROM n |
| 922 | `; |
| 923 | const threadIds = (query: string) => |
| 924 | search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId))); |
| 925 | yield* insertBusy(9_999); |
| 926 | assert.deepEqual(yield* threadIds("kestrel ci"), [old]); |
| 927 | yield* sql`DELETE FROM orchestration_v2_projection_messages WHERE thread_id = ${busy}`; |
| 928 | yield* insertBusy(10_000); |
| 929 | assert.deepEqual(yield* threadIds("kestrel ci"), []); |
| 930 | }), |
| 931 | ); |
| 932 | }); |
| 933 | |
| 934 | it.effect.each([true, false])( |
| 935 | "ThreadSearch narrows Cyrillic and accent-folded Latin short words before the hit limit (indexed: %s)", |
| 936 | (indexed) => |
| 937 | Effect.gen(function* () { |
| 938 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 939 | const search = yield* ThreadSearch.ThreadSearch; |
| 940 | const sql = yield* SqlClient.SqlClient; |
| 941 | const project = ProjectId.make("project:short-limit"); |
| 942 | const older = ThreadId.make("thread:short-older"); |
| 943 | const olderFrench = ThreadId.make("thread:short-older-french"); |
| 944 | const newer = ThreadId.make("thread:short-newer"); |
| 945 | yield* createProject(project); |
| 946 | yield* Effect.forEach( |
| 947 | [ |
| 948 | thread(older, project), |
| 949 | message(older, "older", "user", "В логах ошибка"), |
| 950 | thread(olderFrench, project), |
| 951 | message(olderFrench, "older-french", "user", "École fermée à Lyon"), |
| 952 | thread(newer, project), |
| 953 | ], |
| 954 | projections.apply, |
| 955 | { discard: true }, |
| 956 | ); |
| 957 | if (!indexed) { |
| 958 | yield* sql`DROP TRIGGER orchestration_v2_message_search_insert`; |
| 959 | yield* sql`DROP TRIGGER orchestration_v2_message_search_update`; |
| 960 | yield* sql`DROP TRIGGER orchestration_v2_message_search_delete`; |
| 961 | yield* sql`DROP TABLE orchestration_v2_message_search`; |
| 962 | } |
| 963 | // More newer hits for the long word than one read holds, none with the short one. |
| 964 | yield* sql` |
| 965 | WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 2000) |
| 966 | INSERT INTO orchestration_v2_projection_messages |
| 967 | (message_id, thread_id, role, streaming, created_at, updated_at, payload_json) |
| 968 | SELECT 'short-newer-' || i || '-' || word, ${newer}, 'assistant', 0, |
| 969 | ${DateTime.formatIso(at(5))}, ${DateTime.formatIso(at(5))}, json_object('text', text) |
| 970 | FROM n, (SELECT 'ru' AS word, 'ошибка здесь' AS text UNION ALL SELECT 'fr', 'fermée ici') |
| 971 | `; |
| 972 | const threadIds = (query: string) => |
| 973 | search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId))); |
| 974 | assert.deepEqual(yield* threadIds("ошибка в"), [older]); |
| 975 | assert.deepEqual(yield* threadIds("fermée ec"), [olderFrench]); |
| 976 | }).pipe(Effect.provide(layerTest)), |
| 977 | ); |
| 978 | |
| 979 | // A two-character CJK word is too short for the index, so it scans even with the index. |
| 980 | it.effect.each([true, false])( |
| 981 | "ThreadSearch finds a curly apostrophe for a straight one and back (indexed: %s)", |
| 982 | (indexed) => |
| 983 | Effect.gen(function* () { |
| 984 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 985 | const search = yield* ThreadSearch.ThreadSearch; |
| 986 | const sql = yield* SqlClient.SqlClient; |
| 987 | const project = ProjectId.make("project:apostrophes"); |
| 988 | const curly = ThreadId.make("thread:apostrophe-curly"); |
| 989 | const straight = ThreadId.make("thread:apostrophe-straight"); |
| 990 | yield* createProject(project); |
| 991 | yield* Effect.forEach( |
| 992 | [ |
| 993 | thread(curly, project), |
| 994 | message(curly, "apostrophe-curly", "user", "部署 the gannet\u2019s nest"), |
| 995 | thread(straight, project), |
| 996 | message(straight, "apostrophe-straight", "user", "部署 a kittiwake's nest"), |
| 997 | ], |
| 998 | projections.apply, |
| 999 | { discard: true }, |
| 1000 | ); |
| 1001 | if (!indexed) { |
| 1002 | yield* sql`DROP TRIGGER orchestration_v2_message_search_insert`; |
| 1003 | yield* sql`DROP TRIGGER orchestration_v2_message_search_update`; |
| 1004 | yield* sql`DROP TRIGGER orchestration_v2_message_search_delete`; |
| 1005 | yield* sql`DROP TABLE orchestration_v2_message_search`; |
| 1006 | } |
| 1007 | const threadIds = (query: string) => |
| 1008 | search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId))); |
| 1009 | for (const query of ["gannet's", "部署 gannet's"]) { |
| 1010 | assert.deepEqual(yield* threadIds(query), [curly], query); |
| 1011 | } |
| 1012 | for (const query of ["kittiwake\u2019s", "部署 kittiwake\u2019s"]) { |
| 1013 | assert.deepEqual(yield* threadIds(query), [straight], query); |
| 1014 | } |
| 1015 | }).pipe(Effect.provide(layerTest)), |
| 1016 | ); |
| 1017 | |
| 1018 | it.layer(layerTest)("ThreadSearch short words outside ASCII", (it) => { |
| 1019 | it.effect("finds them in any case or accent, with or without the index", () => |
| 1020 | Effect.gen(function* () { |
| 1021 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 1022 | const search = yield* ThreadSearch.ThreadSearch; |
| 1023 | const sql = yield* SqlClient.SqlClient; |
| 1024 | const project = ProjectId.make("project:short-non-ascii"); |
| 1025 | const russian = ThreadId.make("thread:short-russian"); |
| 1026 | const french = ThreadId.make("thread:short-french"); |
| 1027 | yield* createProject(project); |
| 1028 | yield* Effect.forEach( |
| 1029 | [ |
| 1030 | thread(russian, project), |
| 1031 | message(russian, "russian", "user", "В логах ошибка"), |
| 1032 | thread(french, project), |
| 1033 | message(french, "french", "user", "École fermée à Lyon"), |
| 1034 | message(french, "french-shout", "user", "Météo école"), |
| 1035 | message(french, "romanian", "user", "Școala închisă"), |
| 1036 | message(french, "vietnamese", "user", "Hồ Chí Minh"), |
| 1037 | ], |
| 1038 | projections.apply, |
| 1039 | { discard: true }, |
| 1040 | ); |
| 1041 | const threadIds = (query: string) => |
| 1042 | search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId))); |
| 1043 | |
| 1044 | const assertMatches = Effect.gen(function* () { |
| 1045 | assert.deepEqual(yield* threadIds("ошибка в"), [russian]); |
| 1046 | assert.deepEqual(yield* threadIds("fermée éc"), [french]); |
| 1047 | assert.deepEqual(yield* threadIds("ecole fermée"), [french]); |
| 1048 | assert.deepEqual(yield* threadIds("école me"), [french]); |
| 1049 | assert.deepEqual(yield* threadIds("închisă sc"), [french]); |
| 1050 | assert.deepEqual(yield* threadIds("chí ho"), [french]); |
| 1051 | // They still narrow: a word the message lacks rules it out. |
| 1052 | assert.deepEqual(yield* threadIds("ошибка я"), []); |
| 1053 | assert.deepEqual(yield* threadIds("fermée ce"), []); |
| 1054 | }); |
| 1055 | yield* assertMatches; |
| 1056 | yield* sql`DROP TABLE orchestration_v2_message_search`; |
| 1057 | yield* assertMatches; |
| 1058 | }), |
| 1059 | ); |
| 1060 | }); |
| 1061 | |
| 1062 | it.layer(layerTest)("ThreadSearch across scripts and without the index", (it) => { |
| 1063 | it.effect( |
| 1064 | "matches Korean, Japanese, and two-character Chinese words, and whole short words", |
| 1065 | () => |
| 1066 | Effect.gen(function* () { |
| 1067 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 1068 | const search = yield* ThreadSearch.ThreadSearch; |
| 1069 | const sql = yield* SqlClient.SqlClient; |
| 1070 | const project = ProjectId.make("project:scripts"); |
| 1071 | const threadId = ThreadId.make("thread:scripts"); |
| 1072 | const drafts = ThreadId.make("thread:drafts"); |
| 1073 | const addresses = ThreadId.make("thread:addresses"); |
| 1074 | yield* createProject(project); |
| 1075 | yield* Effect.forEach( |
| 1076 | [ |
| 1077 | thread(threadId, project), |
| 1078 | message(threadId, "korean", "user", "검색 기능을 고쳐 주세요"), |
| 1079 | message(threadId, "japanese", "user", "がんばって検索します"), |
| 1080 | message(threadId, "chinese", "user", "明天部署到生产环境"), |
| 1081 | message(threadId, "fts", "assistant", "the FTS index is ready"), |
| 1082 | thread(drafts, project), |
| 1083 | message(drafts, "drafts", "assistant", "saved two drafts and shifts"), |
| 1084 | message(threadId, "camel", "assistant", "Configure enableFTS today"), |
| 1085 | message(threadId, "accented", "user", "Review Résumé indexing"), |
| 1086 | thread(addresses, project), |
| 1087 | message(addresses, "addresses", "user", "This addresses Résumé indexing"), |
| 1088 | ], |
| 1089 | projections.apply, |
| 1090 | { discard: true }, |
| 1091 | ); |
| 1092 | const threadIds = (query: string) => |
| 1093 | search |
| 1094 | .search({ query }) |
| 1095 | .pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId))); |
| 1096 | |
| 1097 | // Short words must start a word, which camelCase and accented words do. |
| 1098 | for (const query of ["검색 기능", "がんばって", "部署", "fts", "enable"]) { |
| 1099 | assert.deepEqual(yield* threadIds(query), [threadId], query); |
| 1100 | } |
| 1101 | // "res" inside "addresses" does not hide the accented word after it. |
| 1102 | assert.sameMembers(yield* threadIds("res"), [threadId, addresses]); |
| 1103 | // A word beats the same letters inside a longer word, even in an older message. |
| 1104 | const trust = ThreadId.make("thread:trust"); |
| 1105 | const rust = ThreadId.make("thread:rust"); |
| 1106 | yield* Effect.forEach( |
| 1107 | [ |
| 1108 | thread(rust, project), |
| 1109 | message(rust, "rust", "assistant", "Rust rewrite notes"), |
| 1110 | thread(trust, project), |
| 1111 | message(trust, "trust", "user", "We trust the build"), |
| 1112 | ], |
| 1113 | projections.apply, |
| 1114 | { discard: true }, |
| 1115 | ); |
| 1116 | const { matches } = yield* search.search({ query: "rust" }); |
| 1117 | assert.deepEqual( |
| 1118 | matches.map((match) => [match.threadId, (match.quality ?? 1) <= 0.3]), |
| 1119 | [ |
| 1120 | [rust, false], |
| 1121 | [trust, true], |
| 1122 | ], |
| 1123 | ); |
| 1124 | // Without the index (an SQLite build without FTS5 trigrams) search scans instead. |
| 1125 | yield* sql`DROP TABLE orchestration_v2_message_search`; |
| 1126 | assert.deepEqual(yield* threadIds("検索"), [threadId]); |
| 1127 | assert.deepEqual(yield* threadIds("fts"), [threadId]); |
| 1128 | }), |
| 1129 | ); |
| 1130 | }); |
| 1131 | |
| 1132 | it.layer(layerTest)("ThreadSearch.searchOlderSettled", (it) => { |
| 1133 | it.effect("returns older settled title matches in scope, then older message hits", () => |
| 1134 | Effect.gen(function* () { |
| 1135 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 1136 | const search = yield* ThreadSearch.ThreadSearch; |
| 1137 | const project = ProjectId.make("project:kestrel"); |
| 1138 | const other = ProjectId.make("project:kestrel-other"); |
| 1139 | yield* createProject(project); |
| 1140 | yield* createProject(other); |
| 1141 | const id = (name: string) => ThreadId.make(`thread:kestrel-${name}`); |
| 1142 | const old = at(10); |
| 1143 | yield* Effect.forEach( |
| 1144 | [ |
| 1145 | thread(id("old"), project, { title: "Kestrel nest", settledAt: old }), |
| 1146 | thread(id("recent"), project, { title: "Kestrel nest", settledAt: at(50) }), |
| 1147 | thread(id("open"), project, { title: "Kestrel nest" }), |
| 1148 | thread(id("child"), project, { title: "Kestrel nest", subagentOf: id("old") }), |
| 1149 | thread(id("elsewhere"), other, { title: "Kestrel nest", settledAt: old }), |
| 1150 | thread(id("note"), project, { title: "Unrelated", settledAt: old }), |
| 1151 | thread(id("new-note"), project, { title: "Unrelated", settledAt: at(50) }), |
| 1152 | thread(id("archived"), project, { |
| 1153 | title: "Kestrel nest", |
| 1154 | settledAt: old, |
| 1155 | archivedAt: old, |
| 1156 | }), |
| 1157 | thread(id("archived-note"), project, { |
| 1158 | title: "Unrelated", |
| 1159 | settledAt: old, |
| 1160 | archivedAt: old, |
| 1161 | }), |
| 1162 | ], |
| 1163 | projections.apply, |
| 1164 | { discard: true }, |
| 1165 | ); |
| 1166 | const before = DateTime.formatIso(at(30)); |
| 1167 | const messageThreadIds = [id("new-note"), id("note"), id("archived-note"), id("old")]; |
| 1168 | assert.sameMembers( |
| 1169 | [...(yield* search.searchOlderSettled({ query: "kestrel", before, messageThreadIds }))], |
| 1170 | [id("old"), id("elsewhere"), id("note")], |
| 1171 | ); |
| 1172 | // The scope applies to titles and message hits; message hits follow the titles in order. |
| 1173 | assert.deepEqual( |
| 1174 | yield* search.searchOlderSettled({ |
| 1175 | query: "kestrel", |
| 1176 | before, |
| 1177 | projectIds: [project], |
| 1178 | messageThreadIds: [id("elsewhere"), ...messageThreadIds], |
| 1179 | }), |
| 1180 | [id("old"), id("note")], |
| 1181 | ); |
| 1182 | // A palette also matches the environment's name. |
| 1183 | assert.include( |
| 1184 | yield* search.searchOlderSettled({ |
| 1185 | query: "laptop unrelated", |
| 1186 | before, |
| 1187 | environmentLabel: "Laptop", |
| 1188 | messageThreadIds: [], |
| 1189 | }), |
| 1190 | id("note"), |
| 1191 | ); |
| 1192 | }), |
| 1193 | ); |
| 1194 | |
| 1195 | it.effect("matches the branch and linked pull requests, like the palette", () => |
| 1196 | Effect.gen(function* () { |
| 1197 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 1198 | const search = yield* ThreadSearch.ThreadSearch; |
| 1199 | const project = ProjectId.make("project:heron"); |
| 1200 | yield* createProject(project); |
| 1201 | const pullRequest = (number: number): ThreadLinkedPullRequest => ({ |
| 1202 | projectId: project, |
| 1203 | repository: "acme/heron", |
| 1204 | number, |
| 1205 | url: `https://github.com/acme/heron/pull/${number}`, |
| 1206 | }); |
| 1207 | const branch = ThreadId.make("thread:heron-branch"); |
| 1208 | const linked = ThreadId.make("thread:heron-linked"); |
| 1209 | yield* Effect.forEach( |
| 1210 | [ |
| 1211 | thread(branch, project, { |
| 1212 | title: "Heron", |
| 1213 | settledAt: at(10), |
| 1214 | branchPullRequest: pullRequest(4101), |
| 1215 | }), |
| 1216 | thread(linked, project, { |
| 1217 | title: "Heron", |
| 1218 | settledAt: at(10), |
| 1219 | linkedPullRequest: pullRequest(4202), |
| 1220 | }), |
| 1221 | ], |
| 1222 | projections.apply, |
| 1223 | { discard: true }, |
| 1224 | ); |
| 1225 | const find = (query: string) => |
| 1226 | search.searchOlderSettled({ |
| 1227 | query, |
| 1228 | before: DateTime.formatIso(at(30)), |
| 1229 | messageThreadIds: [], |
| 1230 | }); |
| 1231 | assert.deepEqual(yield* find("#4101"), [branch]); |
| 1232 | assert.deepEqual(yield* find("acme/heron#4202"), [linked]); |
| 1233 | }), |
| 1234 | ); |
| 1235 | }); |
| 1236 | |
| 1237 | it.layer(layerTest)("ThreadSearch.searchThreadsWithOlderSettledRows", (it) => { |
| 1238 | it.effect("adds older settled rows to message hits, or alone for titles only", () => |
| 1239 | Effect.gen(function* () { |
| 1240 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 1241 | const search = yield* ThreadSearch.ThreadSearch; |
| 1242 | const sql = yield* SqlClient.SqlClient; |
| 1243 | const project = ProjectId.make("project:plover"); |
| 1244 | yield* createProject(project); |
| 1245 | const old = ThreadId.make("thread:plover-old"); |
| 1246 | const recent = ThreadId.make("thread:plover-recent"); |
| 1247 | yield* Effect.forEach( |
| 1248 | [ |
| 1249 | thread(old, project, { title: "Plover nest", settledAt: at(10) }), |
| 1250 | thread(recent, project, { title: "Plover nest", settledAt: at(50) }), |
| 1251 | message(recent, "plover-note", "user", "a plover note", { minute: 2 }), |
| 1252 | ], |
| 1253 | projections.apply, |
| 1254 | { discard: true }, |
| 1255 | ); |
| 1256 | const run = (request: { readonly titlesOnly?: boolean }) => |
| 1257 | ThreadSearch.searchThreadsWithOlderSettledRows({ |
| 1258 | request: { query: "plover", olderSettledBefore: DateTime.formatIso(at(30)), ...request }, |
| 1259 | search: search.search, |
| 1260 | searchOlderSettled: search.searchOlderSettled, |
| 1261 | sql, |
| 1262 | readThreads: (threadIds) => |
| 1263 | projections.readShellSnapshot({ location: "active", threadIds }), |
| 1264 | latestSequence: Effect.succeed(7), |
| 1265 | }); |
| 1266 | const withMessages = yield* run({}); |
| 1267 | assert.deepEqual( |
| 1268 | withMessages.matches.map((match) => match.threadId), |
| 1269 | [recent], |
| 1270 | ); |
| 1271 | assert.equal(withMessages.snapshotSequence, 7); |
| 1272 | assert.deepEqual( |
| 1273 | withMessages.threads?.map((row) => row.id), |
| 1274 | [old], |
| 1275 | ); |
| 1276 | const titles = yield* run({ titlesOnly: true }); |
| 1277 | assert.deepEqual(titles.matches, []); |
| 1278 | assert.deepEqual( |
| 1279 | titles.threads?.map((row) => row.id), |
| 1280 | [old], |
| 1281 | ); |
| 1282 | // Without the cut it is the plain message search. |
| 1283 | assert.isUndefined( |
| 1284 | (yield* ThreadSearch.searchThreadsWithOlderSettledRows({ |
| 1285 | request: { query: "plover" }, |
| 1286 | search: search.search, |
| 1287 | searchOlderSettled: search.searchOlderSettled, |
| 1288 | sql, |
| 1289 | readThreads: (threadIds) => |
| 1290 | projections.readShellSnapshot({ location: "active", threadIds }), |
| 1291 | latestSequence: Effect.succeed(7), |
| 1292 | })).threads, |
| 1293 | ); |
| 1294 | }), |
| 1295 | ); |
| 1296 | }); |
| 1297 | |
| 1298 | it.layer(layerTest)("ThreadSearch.searchThreads", (it) => { |
| 1299 | it.effect("keeps room for a message-only thread behind many titled ones", () => |
| 1300 | Effect.gen(function* () { |
| 1301 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 1302 | const search = yield* ThreadSearch.ThreadSearch; |
| 1303 | const project = ProjectId.make("project:petrel"); |
| 1304 | yield* createProject(project); |
| 1305 | const messageOnly = ThreadId.make("thread:petrel-message"); |
| 1306 | yield* Effect.forEach( |
| 1307 | [ |
| 1308 | thread(messageOnly, project), |
| 1309 | // The oldest message, so a cut taken before titles drop would lose it. |
| 1310 | message(messageOnly, "petrel-old", "user", "a petrel note", { minute: 1 }), |
| 1311 | ...Array.from({ length: 50 }, (_, index) => { |
| 1312 | const id = ThreadId.make(`thread:petrel-${index}`); |
| 1313 | return [ |
| 1314 | thread(id, project, { title: `Petrel ${index}` }), |
| 1315 | message(id, `petrel-${index}`, "user", "petrel again", { minute: 2 }), |
| 1316 | ]; |
| 1317 | }).flat(), |
| 1318 | ], |
| 1319 | projections.apply, |
| 1320 | { discard: true }, |
| 1321 | ); |
| 1322 | const { matches } = yield* search.searchThreads({ query: "petrel", limit: 5 }); |
| 1323 | assert.include( |
| 1324 | matches.map((match) => match.threadId), |
| 1325 | messageOnly, |
| 1326 | ); |
| 1327 | }), |
| 1328 | ); |
| 1329 | |
| 1330 | it.effect("leaves titled threads out before the hit limit", () => |
| 1331 | Effect.gen(function* () { |
| 1332 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 1333 | const search = yield* ThreadSearch.ThreadSearch; |
| 1334 | const sql = yield* SqlClient.SqlClient; |
| 1335 | const project = ProjectId.make("project:hit-limit"); |
| 1336 | const messageOnly = ThreadId.make("thread:grebe-message"); |
| 1337 | const titled = ThreadId.make("thread:grebe-titled"); |
| 1338 | yield* createProject(project); |
| 1339 | yield* Effect.forEach( |
| 1340 | [ |
| 1341 | thread(messageOnly, project, { title: "Unrelated" }), |
| 1342 | message(messageOnly, "grebe-old", "user", "a grebe note", { minute: 1 }), |
| 1343 | thread(titled, project, { title: "Grebe" }), |
| 1344 | ], |
| 1345 | projections.apply, |
| 1346 | { discard: true }, |
| 1347 | ); |
| 1348 | // Newer hits than one read holds, all in the titled thread. |
| 1349 | yield* sql` |
| 1350 | WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 2000) |
| 1351 | INSERT INTO orchestration_v2_projection_messages |
| 1352 | (message_id, thread_id, role, streaming, created_at, updated_at, payload_json) |
| 1353 | SELECT 'grebe-titled-' || i, ${titled}, 'assistant', 0, ${DateTime.formatIso(at(5))}, |
| 1354 | ${DateTime.formatIso(at(5))}, json_object('text', 'grebe again') |
| 1355 | FROM n |
| 1356 | `; |
| 1357 | const { matches } = yield* search.searchThreads({ query: "grebe", limit: 5 }); |
| 1358 | assert.include( |
| 1359 | matches.map((match) => match.threadId), |
| 1360 | messageOnly, |
| 1361 | ); |
| 1362 | }), |
| 1363 | ); |
| 1364 | |
| 1365 | it.effect("ranks title matches like the palette, then message matches, within one project", () => |
| 1366 | Effect.gen(function* () { |
| 1367 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 1368 | const search = yield* ThreadSearch.ThreadSearch; |
| 1369 | const project = ProjectId.make("project:titles"); |
| 1370 | const otherProject = ProjectId.make("project:titles-other"); |
| 1371 | yield* createProject(project); |
| 1372 | yield* createProject(otherProject); |
| 1373 | const wordStart = ThreadId.make("thread:word-start"); |
| 1374 | const midWord = ThreadId.make("thread:mid-word"); |
| 1375 | const messageOnly = ThreadId.make("thread:message-only"); |
| 1376 | const elsewhere = ThreadId.make("thread:elsewhere"); |
| 1377 | yield* Effect.forEach( |
| 1378 | [ |
| 1379 | thread(midWord, project, { title: "Fix oauth redirect" }), |
| 1380 | thread(wordStart, project, { title: "Clean up auth tokens" }), |
| 1381 | thread(messageOnly, project, { title: "Unrelated" }), |
| 1382 | message(messageOnly, "message-only", "user", "the auth flow broke again"), |
| 1383 | thread(elsewhere, otherProject, { title: "Auth everywhere" }), |
| 1384 | ], |
| 1385 | projections.apply, |
| 1386 | { discard: true }, |
| 1387 | ); |
| 1388 | |
| 1389 | // Stronger message hits in another project must not use up this project's limit. |
| 1390 | yield* Effect.forEach( |
| 1391 | Array.from({ length: 55 }, (_, index) => { |
| 1392 | const busy = ThreadId.make(`thread:elsewhere-${index}`); |
| 1393 | return [ |
| 1394 | thread(busy, otherProject), |
| 1395 | message(busy, `elsewhere-${index}`, "user", "auth auth auth"), |
| 1396 | ]; |
| 1397 | }).flat(), |
| 1398 | projections.apply, |
| 1399 | { discard: true }, |
| 1400 | ); |
| 1401 | const result = yield* search.searchThreads({ query: "auth", projectId: project, limit: 3 }); |
| 1402 | assert.deepEqual( |
| 1403 | result.matches.map((match) => [match.threadId, match.source]), |
| 1404 | [ |
| 1405 | [wordStart, "title"], |
| 1406 | [midWord, "title"], |
| 1407 | [messageOnly, "user"], |
| 1408 | ], |
| 1409 | ); |
| 1410 | // With one slot, an exact title keeps it. |
| 1411 | assert.deepEqual( |
| 1412 | (yield* search.searchThreads({ query: "auth", projectId: project, limit: 1 })).matches.map( |
| 1413 | (match) => match.threadId, |
| 1414 | ), |
| 1415 | [wordStart], |
| 1416 | ); |
| 1417 | // Exact matching finds nothing, so one typo is forgiven. |
| 1418 | assert.deepEqual( |
| 1419 | (yield* search.searchThreads({ query: "tokesn", projectId: project })).matches.map( |
| 1420 | (match) => match.title, |
| 1421 | ), |
| 1422 | ["Clean up auth tokens"], |
| 1423 | ); |
| 1424 | }), |
| 1425 | ); |
| 1426 | }); |
| 1427 | |
| 1428 | it.effect("starts snippets a few words before the match so one line shows it", () => |
| 1429 | Effect.sync(() => { |
| 1430 | const snippet = (text: string, word: string) => |
| 1431 | ThreadSearch.buildSearchSnippet(text, text.indexOf(word)); |
| 1432 | const long = `${"lead ".repeat(60)}the NEEDLE sits here ${"tail ".repeat(60)}`; |
| 1433 | const cut = snippet(long, "NEEDLE"); |
| 1434 | assert.isAtMost(cut.length, 240); |
| 1435 | assert.match(cut, /^…(lead ){2,4}the NEEDLE sits here/); |
| 1436 | assert.isTrue(cut.endsWith("…")); |
| 1437 | // Whitespace collapses only in the shown part, so the offset still lands. |
| 1438 | assert.match(snippet(`${"word\n\n ".repeat(60)}un café`, "café"), /un café$/); |
| 1439 | // A cut never splits an emoji's surrogate pair. |
| 1440 | assert.notMatch( |
| 1441 | snippet(`x${"🚀".repeat(200)}needle`, "needle"), |
| 1442 | /[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]/, |
| 1443 | ); |
| 1444 | // Emphasis and code marks go, as the thread shows the text. |
| 1445 | assert.equal( |
| 1446 | ThreadSearch.buildSearchSnippet("Use **bold** and `code` here", 0), |
| 1447 | "Use bold and code here", |
| 1448 | ); |
| 1449 | assert.equal(ThreadSearch.buildSearchSnippet("Call __init__ first", 0), "Call __init__ first"); |
| 1450 | // Short text still moves a late match into view. |
| 1451 | assert.equal(snippet(`${"word ".repeat(20)}needle`, "needle"), "…word word word word needle"); |
| 1452 | }), |
| 1453 | ); |
| 1454 | |
| 1455 | it.effect("searches on the reader worker while the writer is busy", () => |
| 1456 | Effect.gen(function* () { |
| 1457 | const fileSystem = yield* FileSystem.FileSystem; |
| 1458 | const path = yield* Path.Path; |
| 1459 | const directory = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-thread-search-" }); |
| 1460 | const layerFile = Layer.mergeAll( |
| 1461 | ThreadSearch.layer, |
| 1462 | ProjectionStore.layer, |
| 1463 | ProjectStore.layer, |
| 1464 | ).pipe( |
| 1465 | Layer.provideMerge(SqlitePersistence.layerFromPath(path.join(directory, "state.sqlite"))), |
| 1466 | Layer.provide(NodeServices.layer), |
| 1467 | ); |
| 1468 | |
| 1469 | yield* Effect.gen(function* () { |
| 1470 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 1471 | const search = yield* ThreadSearch.ThreadSearch; |
| 1472 | const sql = yield* SqlClient.SqlClient; |
| 1473 | const project = ProjectId.make("project:search-reader"); |
| 1474 | const threadId = ThreadId.make("thread:reader"); |
| 1475 | yield* createProject(project); |
| 1476 | yield* Effect.forEach( |
| 1477 | [thread(threadId, project), message(threadId, "reader", "user", "needle on the reader")], |
| 1478 | projections.apply, |
| 1479 | { discard: true }, |
| 1480 | ); |
| 1481 | |
| 1482 | const writerHeld = yield* Deferred.make<void>(); |
| 1483 | const releaseWriter = yield* Deferred.make<void>(); |
| 1484 | const writer = yield* Effect.forkChild( |
| 1485 | sql.withTransaction( |
| 1486 | Deferred.succeed(writerHeld, undefined).pipe( |
| 1487 | Effect.andThen(Deferred.await(releaseWriter)), |
| 1488 | ), |
| 1489 | ), |
| 1490 | ); |
| 1491 | yield* Deferred.await(writerHeld); |
| 1492 | const result = yield* search.search({ query: "needle" }); |
| 1493 | yield* Deferred.succeed(releaseWriter, undefined); |
| 1494 | yield* Fiber.join(writer); |
| 1495 | |
| 1496 | assert.deepEqual( |
| 1497 | result.matches.map((match) => match.threadId), |
| 1498 | [threadId], |
| 1499 | ); |
| 1500 | }).pipe(Effect.provide(layerFile)); |
| 1501 | }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), |
| 1502 | ); |