The best way to control Claude Code, Codex and more from any computer or phone

main
ts 541 lines 20.8 KB
Raw
1 import {
2 OrchestrationV2TurnItemJson,
3 type OrchestrationV2ProjectedTurnItem,
4 OrchestrationV2SearchThreadInput,
5 type OrchestrationV2SearchThreadResult,
6 type OrchestrationV2TurnItem,
7 ThreadId,
8 TurnItemId,
9 } from "@t3tools/contracts";
10 import { searchableMessageSegments, searchablePlanSegments } from "@t3tools/shared/threadFindText";
11 import { countThreadSearchOccurrences } from "@t3tools/shared/threadSearch";
12 import { htmlRenderFromToolItem, mcpAppFromToolItem } from "@t3tools/shared/toolOutput";
13 import * as Cache from "effect/Cache";
14 import * as Data from "effect/Data";
15 import * as Effect from "effect/Effect";
16 import * as RcRef from "effect/RcRef";
17 import * as Schema from "effect/Schema";
18 import * as Stream from "effect/Stream";
19 import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient";
20 import * as SqlClient from "effect/sql/SqlClient";
21 import {
22 ProjectionStoreReadError,
23 ProjectionStoreThreadNotFoundError,
24 type ProjectionStoreV2Error,
25 } from "./ProjectionStore.ts";
26
27 interface FindRow {
28 readonly position: number;
29 readonly visibility: OrchestrationV2ProjectedTurnItem["visibility"];
30 readonly sourceThreadId: ThreadId;
31 readonly sourceItemId: TurnItemId;
32 }
33 interface FindPayload extends FindRow {
34 readonly payload: string;
35 readonly entryId: string;
36 readonly runId: OrchestrationV2TurnItem["runId"];
37 readonly type: OrchestrationV2TurnItem["type"];
38 readonly emptyText: boolean;
39 }
40 interface FindDocument extends FindRow {
41 readonly entryId: string;
42 readonly runId: OrchestrationV2TurnItem["runId"];
43 readonly count: number;
44 }
45 interface RenderedDocument extends Omit<FindDocument, "count"> {
46 readonly segments: readonly string[];
47 readonly sourceLength: number;
48 }
49 type CompletedScans = Map<
50 string,
51 {
52 sequence: number;
53 documents: readonly FindDocument[];
54 sourceThreadIds: readonly ThreadId[];
55 }
56 >;
57 // Segments may retain slices of their source text, so the budget includes both.
58 interface RenderedSnapshot {
59 current:
60 | {
61 key: string;
62 sequence: number;
63 documents: readonly RenderedDocument[];
64 sourceThreadIds: readonly ThreadId[];
65 }
66 | undefined;
67 }
68 class TextKey extends Data.Class<{
69 text: string;
70 role: "user" | "assistant" | "plan";
71 streaming: boolean;
72 hasContext: boolean;
73 cwd: string | undefined;
74 skills: NonNullable<OrchestrationV2SearchThreadInput["skills"]>;
75 theme: OrchestrationV2SearchThreadInput["theme"];
76 }> {}
77
78 function textKey(
79 item: OrchestrationV2TurnItem,
80 cwd: string | undefined,
81 skills: TextKey["skills"],
82 theme: TextKey["theme"],
83 ) {
84 return new TextKey({
85 text:
86 item.type === "proposed_plan"
87 ? item.markdown
88 : item.type === "user_message" || item.type === "assistant_message"
89 ? item.text
90 : "",
91 role:
92 item.type === "proposed_plan" ? "plan" : item.type === "user_message" ? "user" : "assistant",
93 streaming: item.type === "assistant_message" && item.streaming,
94 hasContext: item.type === "user_message" && item.context !== undefined,
95 cwd,
96 skills,
97 theme,
98 });
99 }
100
101 function parseText(key: TextKey): readonly string[] {
102 return key.role === "plan"
103 ? searchablePlanSegments(key.text, key.cwd, key.theme)
104 : (searchableMessageSegments(
105 {
106 role: key.role,
107 text: key.text,
108 streaming: key.streaming,
109 context: key.hasContext ? { version: 1, records: [] } : undefined,
110 },
111 key.cwd,
112 key.skills,
113 key.theme,
114 ) ?? []);
115 }
116
117 function documentFor(row: OrchestrationV2ProjectedTurnItem, count: number): FindDocument {
118 return {
119 position: row.position,
120 visibility: row.visibility,
121 sourceThreadId: row.sourceThreadId,
122 sourceItemId: row.sourceItemId,
123 entryId:
124 row.item.type === "user_message" || row.item.type === "assistant_message"
125 ? row.item.messageId
126 : row.item.id,
127 runId: row.item.runId,
128 count,
129 };
130 }
131
132 function countSegments(segments: readonly string[], query: string) {
133 return segments.reduce((sum, text) => sum + countThreadSearchOccurrences(text, query), 0);
134 }
135
136 function selectMatch(
137 documents: readonly FindDocument[],
138 input: Pick<OrchestrationV2SearchThreadInput, "index" | "start" | "offset">,
139 ) {
140 const totalMatches = documents.reduce((sum, doc) => sum + doc.count, 0);
141 let requestedIndex = input.index ?? 0;
142 if (input.index === undefined && input.start) {
143 const startIndex = documents.findIndex((doc) => doc.entryId === input.start?.entryId);
144 if (startIndex >= 0) {
145 requestedIndex =
146 documents.slice(0, startIndex).reduce((sum, doc) => sum + doc.count, 0) +
147 Math.min(input.start.occurrence, documents[startIndex]!.count);
148 if (requestedIndex >= totalMatches) requestedIndex = 0;
149 }
150 }
151 const activeIndex =
152 input.index === undefined && totalMatches > 0
153 ? (((requestedIndex + (input.offset ?? 0)) % totalMatches) + totalMatches) % totalMatches
154 : Math.min(requestedIndex, Math.max(0, totalMatches - 1));
155 let occurrence = activeIndex;
156 for (const document of documents) {
157 if (occurrence < document.count) return { totalMatches, activeIndex, document, occurrence };
158 occurrence -= document.count;
159 }
160 return { totalMatches, activeIndex, document: null, occurrence: 0 };
161 }
162
163 function resultForSelection(
164 selected: ReturnType<typeof selectMatch>,
165 documents: readonly FindDocument[],
166 snapshotSequence: number,
167 ): OrchestrationV2SearchThreadResult {
168 let startIndex = 0;
169 const entries = documents.flatMap(({ entryId, runId, count }) => {
170 const entry = { entryId, runId, count, startIndex };
171 startIndex += count;
172 return count > 0 ? [entry] : [];
173 });
174 const selectedEntry = entries.findIndex((entry) => entry.entryId === selected.document?.entryId);
175 const navigation =
176 entries.length <= 17
177 ? entries
178 : Array.from(
179 { length: 17 },
180 (_, i) => entries[(selectedEntry + i - 8 + entries.length) % entries.length]!,
181 );
182 return {
183 snapshotSequence,
184 totalMatches: selected.totalMatches,
185 activeIndex: selected.activeIndex,
186 navigation,
187 match:
188 selected.document === null
189 ? null
190 : {
191 entryId: selected.document.entryId,
192 runId: selected.document.runId,
193 occurrence: selected.occurrence,
194 },
195 };
196 }
197
198 export const makeThreadFind = Effect.fn("makeThreadFind")(function* (
199 readIndex: (threadId: ThreadId) => Effect.Effect<
200 readonly (Omit<FindRow, "position"> & {
201 readonly item: Pick<OrchestrationV2TurnItem, "type" | "runId">;
202 })[],
203 ProjectionStoreV2Error
204 >,
205 ) {
206 const sql = yield* SqlClient.SqlClient;
207 const encodeCacheKey = Schema.encodeEffect(
208 Schema.fromJsonString(
209 Schema.Struct({
210 threadId: ThreadId,
211 cwd: Schema.optional(Schema.String),
212 query: Schema.optional(Schema.String),
213 skills: OrchestrationV2SearchThreadInput.fields.skills,
214 theme: OrchestrationV2SearchThreadInput.fields.theme,
215 }),
216 ),
217 );
218 const encodeThreadIds = Schema.encodeEffect(Schema.fromJsonString(Schema.Array(ThreadId)));
219 const encodeSources = Schema.encodeEffect(
220 Schema.fromJsonString(Schema.Array(Schema.Struct({ threadId: ThreadId, id: TurnItemId }))),
221 );
222 const decodeItem = Schema.decodeUnknownEffect(Schema.fromJsonString(OrchestrationV2TurnItemJson));
223 const isSnapshotError = Schema.is(
224 Schema.Union([ProjectionStoreReadError, ProjectionStoreThreadNotFoundError]),
225 );
226 const load = Effect.fn("ThreadFind.load")(function* (
227 threadId: ThreadId,
228 rows: readonly FindRow[],
229 ) {
230 if (rows.length === 0) return [];
231 const sources = yield* encodeSources(
232 rows.map((row) => ({ threadId: row.sourceThreadId, id: row.sourceItemId })),
233 ).pipe(Effect.mapError((cause) => new ProjectionStoreReadError({ threadId, cause })));
234 const payloads = yield* sql<{
235 thread_id: string;
236 turn_item_id: string;
237 payload_json: string;
238 entry_id: string;
239 run_id: OrchestrationV2TurnItem["runId"];
240 item_type: OrchestrationV2TurnItem["type"];
241 empty_text: number | null;
242 }>`
243 SELECT item.thread_id, item.turn_item_id, item.payload_json,
244 COALESCE(json_extract(item.payload_json, '$.messageId'), item.turn_item_id) AS entry_id,
245 json_extract(item.payload_json, '$.runId') AS run_id,
246 json_extract(item.payload_json, '$.type') AS item_type,
247 json_extract(item.payload_json, '$.text') = '' AS empty_text
248 FROM orchestration_v2_projection_turn_items AS item
249 JOIN json_each(${sources}) AS wanted
250 ON item.thread_id = json_extract(wanted.value, '$.threadId')
251 AND item.turn_item_id = json_extract(wanted.value, '$.id')
252 `.pipe(Effect.mapError((cause) => new ProjectionStoreReadError({ threadId, cause })));
253 const byThread = new Map<string, Map<string, (typeof payloads)[number]>>();
254 for (const row of payloads) {
255 let items = byThread.get(row.thread_id);
256 if (!items) byThread.set(row.thread_id, (items = new Map()));
257 items.set(row.turn_item_id, row);
258 }
259 return yield* Effect.forEach(rows, (row) => {
260 const payload = byThread.get(row.sourceThreadId)?.get(row.sourceItemId);
261 return payload === undefined
262 ? Effect.fail(new ProjectionStoreReadError({ threadId }))
263 : Effect.succeed({
264 ...row,
265 payload: payload.payload_json,
266 entryId: payload.entry_id,
267 runId: payload.run_id,
268 type: payload.item_type,
269 emptyText: payload.empty_text === 1,
270 });
271 });
272 });
273 const makeTextCache = (capacity: number) =>
274 Cache.make({ capacity, lookup: (key: TextKey) => Effect.sync(() => parseText(key)) });
275 // Text keys are message content, so entries never go stale. Cache TTL only expires on access
276 // and would keep the last searched threads forever. Each search borrows this for its whole
277 // stream, and all of it is dropped once Find is idle.
278 const findCaches = yield* RcRef.make({
279 acquire: Effect.all({
280 // Scans visit every message; a cache smaller than the thread churns without hits.
281 text: makeTextCache(8192),
282 // Bounded separately because each entry can hold a large body; typing reuses the parse.
283 largeText: makeTextCache(32),
284 scans: Effect.sync((): CompletedScans => new Map()),
285 rendered: Effect.sync((): RenderedSnapshot => ({ current: undefined })),
286 }),
287 idleTimeToLive: "1 minute",
288 });
289 const readRevision = Effect.fn("ThreadFind.readRevision")(function* (
290 threadIds: readonly ThreadId[],
291 ) {
292 const sources = yield* encodeThreadIds(threadIds);
293 const revision = yield* sql<{ sequence: number }>`
294 WITH RECURSIVE source_threads(thread_id) AS (
295 SELECT value FROM json_each(${sources})
296 UNION
297 SELECT json_extract(t.payload_json, '$.forkedFrom.threadId')
298 FROM orchestration_v2_projection_threads t
299 JOIN source_threads s ON s.thread_id = t.thread_id
300 WHERE json_extract(t.payload_json, '$.forkedFrom.type') = 'run'
301 )
302 SELECT COALESCE(MAX(sequence), 0) AS sequence FROM orchestration_events
303 -- The planner otherwise walks every V2 event backwards from the newest,
304 -- which takes seconds for a thread that has been idle a long time.
305 INDEXED BY idx_orch_events_stream_sequence
306 WHERE application_event_version = 2 AND aggregate_kind = 'thread'
307 AND stream_id IN (SELECT thread_id FROM source_threads)
308 `;
309 return revision[0]?.sequence ?? 0;
310 });
311 const snapshot = Effect.fn("ThreadFind.snapshot")(
312 function* (
313 input: OrchestrationV2SearchThreadInput,
314 scans: CompletedScans,
315 renderedSnapshot: RenderedSnapshot,
316 ) {
317 const { threadId, theme } = input;
318 const active = yield* sql<{ cwd: string | null }>`
319 SELECT COALESCE(json_extract(t.payload_json, '$.worktreePath'), p.workspace_root) AS cwd
320 FROM orchestration_v2_projection_threads t JOIN projection_projects p ON p.project_id = t.project_id
321 WHERE t.thread_id = ${threadId} AND t.deleted_at IS NULL AND p.deleted_at IS NULL
322 `;
323 if (!active[0]) return yield* new ProjectionStoreThreadNotFoundError({ threadId });
324 const cwd = active[0].cwd ?? undefined;
325 const cacheKey = yield* encodeCacheKey({
326 threadId,
327 cwd,
328 query: input.query,
329 skills: input.skills ?? [],
330 ...(theme === undefined ? {} : { theme }),
331 });
332 const cached = scans.get(cacheKey);
333 if (cached && cached.sequence === (yield* readRevision(cached.sourceThreadIds))) {
334 const selected = selectMatch(cached.documents, input);
335 scans.delete(cacheKey);
336 scans.set(cacheKey, cached);
337 return { result: resultForSelection(selected, cached.documents, cached.sequence) };
338 }
339 const documentKey = yield* encodeCacheKey({
340 threadId,
341 cwd,
342 skills: input.skills ?? [],
343 ...(theme === undefined ? {} : { theme }),
344 });
345 const rendered = renderedSnapshot.current;
346 if (
347 rendered?.key === documentKey &&
348 rendered.sequence === (yield* readRevision(rendered.sourceThreadIds))
349 ) {
350 return {
351 rendered: rendered.documents,
352 cwd,
353 sequence: rendered.sequence,
354 cacheKey,
355 documentKey,
356 sourceThreadIds: rendered.sourceThreadIds,
357 };
358 }
359 const index = yield* readIndex(threadId);
360 const sourceThreadIds = [...new Set([threadId, ...index.map((row) => row.sourceThreadId)])];
361 const sequence = yield* readRevision(sourceThreadIds);
362 const candidates = index
363 .map((row, position) => ({ ...row, position }))
364 .filter((row) =>
365 ["user_message", "assistant_message", "proposed_plan", "user_input_request"].includes(
366 row.item.type,
367 ),
368 );
369 const loadAll = Effect.fnUntraced(function* (rows: readonly FindRow[]) {
370 const loaded: FindPayload[] = [];
371 for (let start = 0; start < rows.length; start += 128) {
372 loaded.push(...(yield* load(threadId, rows.slice(start, start + 128))));
373 }
374 return loaded;
375 });
376 const payloads = yield* loadAll(candidates);
377 // Only runs with an empty reply can hide its fallback, so other tool payloads stay unread.
378 const emptyReplyRunIds = new Set(
379 payloads.flatMap((row) =>
380 row.type === "assistant_message" && row.emptyText && row.runId !== null
381 ? [row.runId]
382 : [],
383 ),
384 );
385 const toolPayloads = yield* loadAll(
386 index
387 .map((row, position) => ({ ...row, position }))
388 .filter(
389 (row) =>
390 row.item.type === "dynamic_tool" &&
391 row.item.runId !== null &&
392 emptyReplyRunIds.has(row.item.runId),
393 ),
394 );
395 return { payloads, toolPayloads, cwd, sequence, cacheKey, documentKey, sourceThreadIds };
396 },
397 (effect) => sql.withTransaction(effect).pipe(NodeSqliteClient.readOnly),
398 (effect, input) =>
399 effect.pipe(
400 Effect.mapError((cause) =>
401 isSnapshotError(cause)
402 ? cause
403 : new ProjectionStoreReadError({ threadId: input.threadId, cause }),
404 ),
405 ),
406 );
407 const searchThreadStream = (input: OrchestrationV2SearchThreadInput) =>
408 Stream.unwrap(
409 Effect.gen(function* () {
410 const { scans, rendered: renderedSnapshot, ...textCache } = yield* RcRef.get(findCaches);
411 // Release SQLite before decoding and parsing. Both frames describe one snapshot.
412 const data = yield* snapshot(input, scans, renderedSnapshot);
413 if (data.result) return Stream.succeed(data.result);
414 const { cwd, sequence, cacheKey, documentKey, sourceThreadIds } = data;
415 const payloads = data.payloads ?? [];
416 const toolPayloads = data.toolPayloads ?? [];
417 const decode = (row: FindPayload) =>
418 decodeItem(row.payload).pipe(
419 Effect.map((item) => ({ ...row, item })),
420 Effect.mapError(
421 (cause) => new ProjectionStoreReadError({ threadId: input.threadId, cause }),
422 ),
423 );
424 // Answers rendered inside question cards must not appear as separate matches.
425 const foldedAnswerRunIds = new Map<string, OrchestrationV2TurnItem["runId"]>();
426 for (const payload of payloads) {
427 if (payload.type !== "user_input_request") continue;
428 const { item } = yield* decode(payload);
429 if (item.type === "user_input_request" && item.questionAnswer)
430 foldedAnswerRunIds.set(`async-answer:${item.questionAnswer.requestId}`, item.runId);
431 }
432 // The timeline hides an empty reply's fallback when its run shows an HTML render or MCP app.
433 const embeddedResultRunIds = new Set<OrchestrationV2TurnItem["runId"]>();
434 for (const payload of toolPayloads) {
435 const { item } = yield* decode(payload);
436 if (
437 item.type === "dynamic_tool" &&
438 ((item.status === "completed" && htmlRenderFromToolItem(item)) ||
439 mcpAppFromToolItem(item))
440 )
441 embeddedResultRunIds.add(item.runId);
442 }
443 const rows =
444 data.rendered ??
445 payloads.filter(
446 (row) =>
447 row.type !== "user_input_request" &&
448 !(row.type === "user_message" && foldedAnswerRunIds.get(row.entryId) === row.runId),
449 );
450 const documents = new Map<number, FindDocument & RenderedDocument>();
451 const count = Effect.fnUntraced(function* (row: FindPayload | RenderedDocument) {
452 const cached = documents.get(row.position);
453 if (cached) return cached;
454 let rendered: RenderedDocument;
455 if ("segments" in row) {
456 rendered = row;
457 } else {
458 const decoded = yield* decode(row);
459 const key = textKey(decoded.item, cwd, input.skills ?? [], input.theme);
460 const segments =
461 decoded.item.type === "assistant_message" &&
462 decoded.item.text === "" &&
463 embeddedResultRunIds.has(decoded.item.runId)
464 ? []
465 : yield* Cache.get(
466 key.text.length <= 32_768 ? textCache.text : textCache.largeText,
467 key,
468 );
469 rendered = { ...documentFor(decoded, 0), segments, sourceLength: key.text.length };
470 }
471 const document = {
472 ...rendered,
473 count: countSegments(rendered.segments, input.query),
474 };
475 documents.set(row.position, document);
476 yield* Effect.yieldNow;
477 return document;
478 });
479 const finish = Effect.gen(function* () {
480 for (const row of rows) yield* count(row);
481 const ordered = rows.map((row) => documents.get(row.position)!);
482 renderedSnapshot.current =
483 ordered.length <= 8192 &&
484 ordered.reduce(
485 (size, doc) =>
486 size + doc.sourceLength + doc.segments.reduce((sum, text) => sum + text.length, 0),
487 0,
488 ) <= 2_097_152
489 ? { key: documentKey, sequence, documents: ordered, sourceThreadIds }
490 : undefined;
491 scans.delete(cacheKey);
492 scans.set(cacheKey, {
493 sequence,
494 documents: ordered.map(
495 ({ segments: _segments, sourceLength: _sourceLength, ...document }) => document,
496 ),
497 sourceThreadIds,
498 });
499 if (scans.size > 8) {
500 const oldest = scans.keys().next().value;
501 if (oldest !== undefined) scans.delete(oldest);
502 }
503 return resultForSelection(selectMatch(ordered, input), ordered, sequence);
504 });
505 if (input.index !== undefined || (input.offset ?? 0) !== 0)
506 return Stream.fromEffect(finish);
507 const anchor = Math.max(
508 0,
509 rows.findIndex((row) => row.entryId === input.start?.entryId),
510 );
511 for (let step = 0; step <= rows.length; step++) {
512 const index = (anchor + step) % rows.length;
513 const row = rows[index];
514 if (!row) break;
515 const document = yield* count(row);
516 const occurrence =
517 step === 0 && row.entryId === input.start?.entryId
518 ? Math.min(input.start.occurrence, document.count)
519 : 0;
520 if (occurrence >= document.count) continue;
521 const first: OrchestrationV2SearchThreadResult = {
522 complete: false,
523 snapshotSequence: sequence,
524 totalMatches: 1,
525 activeIndex: 0,
526 match: { entryId: document.entryId, runId: document.runId, occurrence },
527 };
528 return Stream.concat(Stream.succeed(first), Stream.fromEffect(finish));
529 }
530 return Stream.fromEffect(finish);
531 }),
532 );
533 return {
534 searchThreadStream,
535 searchThread: (input: OrchestrationV2SearchThreadInput) =>
536 searchThreadStream(input).pipe(
537 Stream.runCollect,
538 Effect.map((results) => results[results.length - 1]!),
539 ),
540 };
541 });