@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 1,731 lines 62.5 KB
Raw
1 import { assert, it } from "@effect/vitest";
2 import {
3 CommandId,
4 ContextTransferId,
5 EventId,
6 MessageId,
7 NodeId,
8 type OrchestrationV2AppThread,
9 type OrchestrationV2Run,
10 ProjectId,
11 ProviderDriverKind,
12 ProviderInstanceId,
13 ProviderSessionId,
14 ProviderThreadId,
15 ProviderTurnId,
16 RunAttemptId,
17 RunId,
18 RuntimeRequestId,
19 ThreadId,
20 TurnItemId,
21 } from "@t3tools/contracts";
22 import * as DateTime from "effect/DateTime";
23 import * as Effect from "effect/Effect";
24 import * as Layer from "effect/Layer";
25 import * as Stream from "effect/Stream";
26 import * as SqlClient from "effect/sql/SqlClient";
27 import * as Statement from "effect/sql/Statement";
28
29 import delegatedRecoveryIndexes from "../persistence/Migrations/079_DelegatedRecoveryIndexes.ts";
30 import * as SqlitePersistence from "../persistence/Sqlite.ts";
31 import * as ServerSettings from "../serverSettings.ts";
32 import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
33 import * as EffectOutbox from "./EffectOutbox.ts";
34 import * as ProjectionStore from "./ProjectionStore.ts";
35 import * as EventSink from "./EventSink.ts";
36 import * as EventStore from "./EventStore.ts";
37 import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
38 import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts";
39 import { restartContinuationRun } from "./RestartContinuation.ts";
40
41 const layerTest = Layer.mergeAll(ProjectionStore.layer, EffectOutbox.layer).pipe(
42 Layer.provideMerge(SqlitePersistence.layerMemory),
43 );
44 const layerRecoveryTest = ProviderRuntimeRecovery.layer.pipe(
45 Layer.provideMerge(
46 EventSink.layer.pipe(
47 Layer.provideMerge(
48 Layer.mergeAll(
49 EventStore.layer,
50 ProjectionStore.layer,
51 EffectOutbox.layer,
52 IdAllocator.layer,
53 ).pipe(Layer.provideMerge(SqlitePersistence.layerMemory)),
54 ),
55 ),
56 ),
57 Layer.provide(ServerSettings.layerTest()),
58 );
59 const providerInstanceId = ProviderInstanceId.make("codex");
60 const modelSelection = { instanceId: providerInstanceId, model: "gpt-5.4" };
61 const driver = ProviderDriverKind.make("codex");
62
63 const createThread = Effect.fn(function* (
64 name: string,
65 overrides: Partial<OrchestrationV2AppThread> = {},
66 ) {
67 const projections = yield* ProjectionStore.ProjectionStoreV2;
68 const now = yield* DateTime.now;
69 const threadId = ThreadId.make(`thread:recovery:${name}`);
70 const thread: OrchestrationV2AppThread = {
71 createdBy: "user",
72 creationSource: "web",
73 id: threadId,
74 projectId: ProjectId.make("project:recovery"),
75 title: name,
76 providerInstanceId,
77 modelSelection,
78 runtimeMode: "full-access",
79 interactionMode: "default",
80 branch: null,
81 worktreePath: null,
82 activeProviderThreadId: null,
83 lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId },
84 forkedFrom: null,
85 createdAt: now,
86 updatedAt: now,
87 archivedAt: null,
88 settledOverride: null,
89 settledAt: null,
90 lastVisitedAt: null,
91 deletedAt: null,
92 ...overrides,
93 };
94 yield* projections.apply({
95 id: EventId.make(`event:${threadId}:created`),
96 type: "thread.created",
97 threadId,
98 occurredAt: now,
99 payload: thread,
100 });
101 return threadId;
102 });
103
104 const createRun = Effect.fn(function* (
105 threadId: ThreadId,
106 status: OrchestrationV2Run["status"],
107 overrides: Partial<OrchestrationV2Run> = {},
108 ) {
109 const projections = yield* ProjectionStore.ProjectionStoreV2;
110 const now = yield* DateTime.now;
111 const ordinal = overrides.ordinal ?? 1;
112 const runId = RunId.make(`run:${threadId}:${ordinal}`);
113 yield* projections.apply({
114 id: EventId.make(`event:${runId}:created`),
115 type: "run.created",
116 threadId,
117 runId,
118 occurredAt: now,
119 payload: {
120 id: runId,
121 threadId,
122 ordinal,
123 providerInstanceId,
124 modelSelection,
125 providerThreadId: null,
126 userMessageId: MessageId.make(`message:${runId}`),
127 rootNodeId: null,
128 activeAttemptId: null,
129 status,
130 requestedAt: now,
131 startedAt: now,
132 completedAt: status === "completed" ? now : null,
133 checkpointId: null,
134 contextHandoffId: null,
135 ...overrides,
136 },
137 });
138 return runId;
139 });
140
141 it.effect.each(["startup", "shutdown"] as const)(
142 "%s cancels queued buffered wakes and holds durable messages",
143 (trigger) =>
144 Effect.gen(function* () {
145 const projections = yield* ProjectionStore.ProjectionStoreV2;
146 const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
147 const now = yield* DateTime.now;
148 const bufferedThreads = [] as ThreadId[];
149 const durableThreads = [] as ThreadId[];
150 for (const kind of ["buffered", "user", "notification", "delegated"] as const) {
151 const threadId = yield* createThread(`wake:${trigger}:${kind}`);
152 const buffered = kind === "buffered";
153 (buffered ? bufferedThreads : durableThreads).push(threadId);
154 const messageId = MessageId.make(`message:${threadId}`);
155 const runId = yield* createRun(threadId, "queued", {
156 userMessageId: messageId,
157 startedAt: null,
158 queuePosition: 1,
159 });
160 yield* createRun(threadId, "completed", { ordinal: 2 });
161 yield* projections.apply({
162 id: EventId.make(`message:${threadId}`),
163 type: "message.updated",
164 threadId,
165 runId,
166 occurredAt: now,
167 payload: {
168 id: messageId,
169 threadId,
170 runId,
171 nodeId: null,
172 role: "user",
173 text: "Queued work",
174 attachments: [],
175 streaming: false,
176 createdAt: now,
177 updatedAt: now,
178 createdBy: kind === "user" ? "user" : "agent",
179 creationSource: buffered ? "provider" : "server",
180 ...(kind === "user" || kind === "delegated"
181 ? {}
182 : {
183 notification: {
184 source: { kind: "background_task" as const },
185 outcome: "updated" as const,
186 summary: "Background activity updated",
187 },
188 }),
189 ...(kind === "delegated"
190 ? { delegatedCompletion: { parentRunId: runId, generation: 1, taskIds: [] } }
191 : {}),
192 },
193 });
194 }
195 assert.deepEqual(
196 new Set(yield* projections.getRecoveryThreadIds("runtime")),
197 new Set([...bufferedThreads, ...durableThreads]),
198 );
199 const summary = yield* recovery.reconcile(trigger);
200 assert.equal(summary.terminalizedRuns, 1);
201 for (const threadId of bufferedThreads) {
202 const projection = yield* projections.getThreadProjection(threadId);
203 assert.equal(projection.runs[0]?.status, "cancelled");
204 assert.isNotNull(projection.runs[0]?.completedAt);
205 assert.isNull(projection.runs[0]?.queuePosition);
206 assert.equal(projection.runs[1]?.status, "completed");
207 }
208 for (const threadId of durableThreads) {
209 const projection = yield* projections.getThreadProjection(threadId);
210 assert.equal(projection.runs[0]?.status, "queued");
211 assert.isTrue(projection.runs[0]?.queueHeld);
212 }
213 assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), []);
214 }).pipe(Effect.provide(layerRecoveryTest)),
215 );
216
217 it.effect.each(["startup", "shutdown"] as const)(
218 "stops a detached session and its orphaned active provider thread on %s",
219 (trigger) =>
220 Effect.gen(function* () {
221 const projections = yield* ProjectionStore.ProjectionStoreV2;
222 const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
223 const now = yield* DateTime.now;
224 const threadId = yield* createThread("detached-session");
225 const providerSessionId = ProviderSessionId.make("session:recovery:detached");
226 yield* projections.apply({
227 id: EventId.make("event:recovery:detached-session:attached"),
228 type: "provider-session.attached",
229 threadId,
230 driver,
231 providerInstanceId,
232 occurredAt: now,
233 payload: {
234 id: providerSessionId,
235 driver,
236 providerInstanceId,
237 status: "ready",
238 cwd: "/workspace",
239 model: modelSelection.model,
240 capabilities: CodexProviderCapabilitiesV2,
241 createdAt: now,
242 updatedAt: now,
243 lastError: null,
244 },
245 });
246 yield* projections.apply({
247 id: EventId.make("event:recovery:detached-session:detached"),
248 type: "provider-session.detached",
249 threadId,
250 driver,
251 providerInstanceId,
252 occurredAt: now,
253 payload: { providerSessionId, detachedAt: now, reason: "Thread settled." },
254 });
255 const providerThreadId = ProviderThreadId.make("provider-thread:recovery:detached");
256 yield* projections.apply({
257 id: EventId.make("event:recovery:detached-provider-thread"),
258 type: "provider-thread.updated",
259 threadId,
260 occurredAt: now,
261 payload: {
262 id: providerThreadId,
263 appThreadId: threadId,
264 ownerNodeId: null,
265 driver,
266 providerInstanceId,
267 providerSessionId: null,
268 nativeThreadRef: null,
269 nativeConversationHeadRef: null,
270 status: "active",
271 firstRunOrdinal: null,
272 lastRunOrdinal: null,
273 handoffIds: [],
274 forkedFrom: null,
275 createdAt: now,
276 updatedAt: now,
277 },
278 });
279 assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), [threadId]);
280
281 const summary = yield* recovery.reconcile(trigger);
282
283 assert.equal(summary.stoppedSessions, 1);
284 const sql = yield* SqlClient.SqlClient;
285 const rows = yield* sql<{ readonly status: string }>`
286 SELECT status FROM orchestration_v2_projection_provider_sessions
287 WHERE provider_session_id = ${providerSessionId}
288 `;
289 assert.deepEqual(rows, [{ status: "stopped" }]);
290 const projection = yield* projections.getThreadProjection(threadId);
291 assert.equal(projection.providerThreads[0]?.status, "idle");
292 const eventStore = yield* EventStore.EventStoreV2;
293 const sequence = yield* eventStore.latestSequence();
294 assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), []);
295 yield* recovery.reconcile(trigger);
296 assert.equal(yield* eventStore.latestSequence(), sequence);
297 }).pipe(Effect.provide(layerRecoveryTest)),
298 );
299
300 it.effect.each(["attached", "detached", "replaced"] as const)(
301 "recovers attached sessions without restoring detached errors, session=%s",
302 (state) =>
303 Effect.gen(function* () {
304 const projections = yield* ProjectionStore.ProjectionStoreV2;
305 const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
306 const now = yield* DateTime.now;
307 const threadId = yield* createThread("failed-status-session");
308 const providerSessionId = ProviderSessionId.make("session:recovery:failed-status");
309 const session = {
310 id: providerSessionId,
311 driver,
312 providerInstanceId,
313 status: "error" as const,
314 cwd: "/workspace",
315 model: modelSelection.model,
316 capabilities: CodexProviderCapabilitiesV2,
317 extensionStatuses: [{ key: "watch", text: "Watching" }],
318 createdAt: now,
319 updatedAt: now,
320 lastError: "Rate limited",
321 };
322 yield* projections.apply({
323 id: EventId.make("event:recovery:failed-status:attached"),
324 type: "provider-session.attached",
325 threadId,
326 driver,
327 providerInstanceId,
328 occurredAt: now,
329 payload: session,
330 });
331 if (state !== "attached") {
332 yield* projections.apply({
333 id: EventId.make("event:recovery:failed-status:detached"),
334 type: "provider-session.detached",
335 threadId,
336 driver,
337 providerInstanceId,
338 occurredAt: now,
339 payload: { providerSessionId, detachedAt: now, reason: "Thread settled." },
340 });
341 }
342 const replacementSessionId = ProviderSessionId.make("session:recovery:replacement");
343 if (state === "replaced") {
344 yield* projections.apply({
345 id: EventId.make("event:recovery:replacement:attached"),
346 type: "provider-session.attached",
347 threadId,
348 driver,
349 providerInstanceId,
350 occurredAt: now,
351 payload: {
352 ...session,
353 id: replacementSessionId,
354 status: "ready",
355 extensionStatuses: [],
356 lastError: null,
357 },
358 });
359 yield* createRun(threadId, "completed");
360 }
361 assert.deepEqual(
362 yield* projections.getRecoveryThreadIds("runtime"),
363 state === "detached" ? [] : [threadId],
364 );
365 yield* recovery.reconcile("startup");
366 const eventStore = yield* EventStore.EventStoreV2;
367 const stored = yield* eventStore.read({ threadId }).pipe(Stream.runCollect);
368 const sessionUpdates = stored.flatMap(({ event }) =>
369 event.type === "provider-session.updated"
370 ? [
371 {
372 id: event.payload.id,
373 status: event.payload.status,
374 lastError: event.payload.lastError,
375 extensionStatuses: event.payload.extensionStatuses,
376 },
377 ]
378 : [],
379 );
380 assert.deepEqual(
381 sessionUpdates,
382 state === "detached"
383 ? []
384 : [
385 {
386 id: state === "attached" ? providerSessionId : replacementSessionId,
387 status: state === "attached" ? "error" : "stopped",
388 lastError: state === "attached" ? "Rate limited" : null,
389 extensionStatuses: [],
390 },
391 ],
392 );
393 assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), []);
394 }).pipe(Effect.provide(layerRecoveryTest)),
395 );
396
397 it.effect("selects unfinished recovery work without reading settled thread histories", () =>
398 Effect.gen(function* () {
399 const projections = yield* ProjectionStore.ProjectionStoreV2;
400 const outbox = yield* EffectOutbox.EffectOutboxV2;
401 const sql = yield* SqlClient.SqlClient;
402 const now = yield* DateTime.now;
403 for (let index = 0; index < 600; index += 1) {
404 yield* createRun(yield* createThread(`settled-${index}`), "completed");
405 }
406 const queued = yield* createThread("queued");
407 const archived = yield* createThread("archived", { archivedAt: now });
408 const deleted = yield* createThread("deleted", { deletedAt: now });
409 const blocked = yield* createThread("blocked");
410 yield* createRun(queued, "completed");
411 for (const threadId of [queued, archived, deleted, blocked]) {
412 yield* createRun(threadId, "queued", threadId === queued ? { ordinal: 2 } : {});
413 }
414 yield* createRun(blocked, "waiting", { ordinal: 2 });
415 const background = yield* createThread("background");
416 yield* createRun(background, "completed");
417 yield* projections.apply({
418 id: EventId.make("event:recovery:background"),
419 type: "turn-item.updated",
420 threadId: background,
421 occurredAt: now,
422 payload: {
423 id: TurnItemId.make("item:recovery:background"),
424 threadId: background,
425 runId: null,
426 nodeId: null,
427 providerThreadId: null,
428 providerTurnId: null,
429 nativeItemRef: null,
430 parentItemId: null,
431 ordinal: 1,
432 type: "dynamic_tool",
433 status: "waiting",
434 title: null,
435 startedAt: now,
436 completedAt: null,
437 updatedAt: now,
438 toolName: "background",
439 input: null,
440 output: null,
441 },
442 });
443 const outboxOnly = yield* createThread("outbox-only");
444 yield* outbox.enqueue([
445 {
446 id: "effect:recovery:outbox-only",
447 commandId: CommandId.make("command:recovery:outbox-only"),
448 threadId: outboxOnly,
449 request: { type: "provider-turn.start", runId: RunId.make("run:outbox-only") },
450 },
451 ]);
452 const requestOnly = yield* createThread("request-only");
453 yield* projections.apply({
454 id: EventId.make("event:recovery:request-only"),
455 type: "runtime-request.updated",
456 threadId: requestOnly,
457 occurredAt: now,
458 payload: {
459 id: RuntimeRequestId.make("request:recovery:pending"),
460 nodeId: NodeId.make("node:recovery:request"),
461 providerTurnId: null,
462 nativeRequestRef: null,
463 kind: "user_input",
464 status: "pending",
465 responseCapability: { type: "not_resumable", reason: "Process stopped" },
466 createdAt: now,
467 resolvedAt: null,
468 },
469 });
470 const delivery = yield* createThread("delivery", { archivedAt: now });
471 yield* createRun(delivery, "completed", {
472 delegatedCompletion: {
473 disposition: "open",
474 nextGeneration: 2,
475 delivery: { generation: 1, messageId: MessageId.make("message:delivery"), taskIds: [] },
476 },
477 });
478
479 yield* sql`
480 UPDATE orchestration_v2_projection_runs SET payload_json = '{broken'
481 WHERE thread_id = ${ThreadId.make("thread:recovery:settled-599")}
482 `;
483 yield* sql`
484 UPDATE orchestration_v2_projection_runs SET payload_json = '{broken'
485 WHERE thread_id = ${queued} AND ordinal = 1
486 `;
487 assert.deepEqual(yield* projections.getRecoveryThreadIds("queued-runs"), [queued]);
488 for (const [kind, expected, index] of [
489 [
490 "runtime",
491 [queued, archived, blocked, background, outboxOnly, requestOnly],
492 "nodes_live_idx",
493 ],
494 ["delegated-completions", [delivery], "runs_delegated_delivery_idx"],
495 ["subagent-results", [], "threads_subagent_result_idx"],
496 ] as const) {
497 const statements: Array<ReturnType<Statement.Statement<unknown>["compile"]>> = [];
498 const observe: Statement.Transformer = (statement) => {
499 statements.push(statement.compile());
500 return Effect.succeed(statement);
501 };
502 assert.deepEqual(
503 new Set(
504 yield* projections
505 .getRecoveryThreadIds(kind)
506 .pipe(Effect.provideService(Statement.CurrentTransformer, observe)),
507 ),
508 new Set<ThreadId>(expected),
509 );
510 const [query, params] = statements[0]!;
511 const plan = yield* sql.unsafe<{ detail: string }>(`EXPLAIN QUERY PLAN ${query}`, params);
512 assert.isTrue(
513 plan.some((row) => row.detail.includes(index)),
514 JSON.stringify(plan),
515 );
516 }
517 const recoveryState = yield* projections.getRuntimeRecoveryProjection(queued);
518 assert.deepEqual(
519 recoveryState.runs.map((run) => run.id),
520 [RunId.make(`run:${queued}:2`)],
521 );
522 assert.deepEqual(yield* projections.getUnreadableThreadIds(), [
523 queued,
524 ThreadId.make("thread:recovery:settled-599"),
525 ]);
526 }).pipe(Effect.provide(layerTest)),
527 );
528
529 it.effect("maintains delegated candidate indexes across legacy rows and payload transitions", () =>
530 Effect.gen(function* () {
531 const projections = yield* ProjectionStore.ProjectionStoreV2;
532 const sql = yield* SqlClient.SqlClient;
533 const parent = yield* createThread("index-parent");
534 const child = yield* createThread("index-child");
535 const run = yield* createRun(child, "completed");
536 const rows = yield* sql<{ payload_json: string }>`
537 SELECT payload_json FROM orchestration_v2_projection_threads WHERE thread_id = ${child}
538 `;
539 const original = JSON.parse(rows[0]!.payload_json);
540 for (const payload of [
541 "{broken",
542 JSON.stringify({
543 ...original,
544 lineage: { parentThreadId: parent, relationshipToParent: "subagent" },
545 forkedFrom: { type: "provider_thread" },
546 }),
547 JSON.stringify({
548 ...original,
549 lineage: { parentThreadId: parent, relationshipToParent: "subagent" },
550 forkedFrom: { type: "node", nodeId: "node:index" },
551 }),
552 JSON.stringify({
553 ...original,
554 lineage: { parentThreadId: child, relationshipToParent: "subagent" },
555 forkedFrom: { type: "node", nodeId: "node:index" },
556 }),
557 JSON.stringify(original),
558 ]) {
559 yield* sql`UPDATE orchestration_v2_projection_threads SET payload_json = ${payload}
560 WHERE thread_id = ${child}`;
561 const expected = payload.includes('"type":"node"') ? [child] : [];
562 assert.deepEqual(yield* projections.getRecoveryThreadIds("subagent-results"), expected);
563 }
564 for (const entry of [
565 ["{broken", false],
566 ["{}", false],
567 ['{"delegatedCompletion":{"delivery":null}}', false],
568 ['{"delegatedCompletion":{"delivery":[]}}', false],
569 ['{"delegatedCompletion":{"delivery":{}}}', true],
570 ["{}", false],
571 ] as const) {
572 const [payload, included] = entry;
573 yield* sql`UPDATE orchestration_v2_projection_runs SET payload_json = ${payload}
574 WHERE run_id = ${run}`;
575 assert.deepEqual(
576 yield* projections.getRecoveryThreadIds("delegated-completions"),
577 included ? [child] : [],
578 );
579 }
580 yield* sql`UPDATE orchestration_v2_projection_threads SET payload_json = '{broken'
581 WHERE thread_id = ${parent}`;
582 yield* sql`UPDATE orchestration_v2_projection_runs SET payload_json = '{broken'
583 WHERE run_id = ${run}`;
584 const existing = yield* createThread("index-existing", {
585 lineage: { parentThreadId: child, relationshipToParent: "subagent", rootThreadId: parent },
586 forkedFrom: { type: "node", nodeId: NodeId.make("node:index-existing") },
587 });
588 yield* createRun(existing, "completed", {
589 delegatedCompletion: {
590 disposition: "open",
591 nextGeneration: 2,
592 delivery: {
593 generation: 1,
594 messageId: MessageId.make("message:index-existing"),
595 taskIds: [],
596 },
597 },
598 });
599 yield* sql`DROP INDEX orchestration_v2_projection_threads_subagent_result_idx`;
600 yield* sql`DROP INDEX orchestration_v2_projection_runs_delegated_delivery_idx`;
601 yield* delegatedRecoveryIndexes;
602 assert.deepEqual(yield* projections.getRecoveryThreadIds("subagent-results"), [existing]);
603 assert.deepEqual(yield* projections.getRecoveryThreadIds("delegated-completions"), [existing]);
604 }).pipe(Effect.provide(layerTest)),
605 );
606
607 it.effect("recovers terminal subagent results until their cross-thread transfer exists", () =>
608 Effect.gen(function* () {
609 const projections = yield* ProjectionStore.ProjectionStoreV2;
610 const sql = yield* SqlClient.SqlClient;
611 const now = yield* DateTime.now;
612 const parent = yield* createThread("subagent-parent");
613 const children: Array<ThreadId> = [];
614 for (const name of ["terminal", "archived", "deleted", "running", "held", "queued"]) {
615 const child = yield* createThread(`subagent-${name}`, {
616 lineage: { parentThreadId: parent, relationshipToParent: "subagent", rootThreadId: parent },
617 forkedFrom: { type: "node", nodeId: NodeId.make(`node:${name}`) },
618 archivedAt: name === "archived" ? now : null,
619 deletedAt: name === "deleted" ? now : null,
620 });
621 yield* createRun(child, name === "running" ? "running" : "completed");
622 if (name === "held" || name === "queued") {
623 yield* createRun(child, "queued", {
624 ordinal: 2,
625 startedAt: null,
626 ...(name === "held" ? { queueHeld: true } : {}),
627 });
628 }
629 children.push(child);
630 }
631 const terminalChildren = new Set([children[0]!, children[1]!, children[4]!]);
632 assert.deepEqual(
633 new Set(yield* projections.getRecoveryThreadIds("subagent-results")),
634 terminalChildren,
635 );
636
637 const completed = children[0]!;
638 const transferId = ContextTransferId.make("transfer:recovery:subagent-result");
639 yield* projections.apply({
640 id: EventId.make("event:recovery:subagent-result"),
641 type: "context-transfer.created",
642 threadId: parent,
643 occurredAt: now,
644 payload: {
645 id: transferId,
646 type: "subagent_result",
647 sourceThreadId: completed,
648 targetThreadId: parent,
649 sourcePoint: { threadId: completed },
650 basePoint: null,
651 sourceProviderInstanceId: providerInstanceId,
652 targetProviderInstanceId: providerInstanceId,
653 targetRunId: null,
654 status: "pending",
655 resolution: null,
656 createdBy: "system",
657 error: null,
658 createdAt: now,
659 updatedAt: now,
660 consumedAt: null,
661 },
662 });
663 const archivedChild = children[1];
664 assert.isDefined(archivedChild);
665 assert.deepEqual(
666 new Set(yield* projections.getRecoveryThreadIds("subagent-results")),
667 new Set([archivedChild, children[4]!]),
668 );
669 assert.deepEqual(yield* projections.getUnreadableThreadIds(), []);
670 yield* sql`
671 UPDATE orchestration_v2_projection_context_transfers SET payload_json = '{}'
672 WHERE context_transfer_id = ${transferId}
673 `;
674 assert.deepEqual(
675 new Set(yield* projections.getUnreadableThreadIds()),
676 new Set([parent, completed]),
677 );
678 }).pipe(Effect.provide(layerTest)),
679 );
680
681 it.effect("recovers only recorded follow-up results with SQLite projections", () =>
682 Effect.gen(function* () {
683 const projections = yield* ProjectionStore.ProjectionStoreV2;
684 const now = yield* DateTime.now;
685 const parent = yield* createThread("followup-parent");
686 const child = yield* createThread("followup-child", {
687 lineage: { parentThreadId: parent, relationshipToParent: "subagent", rootThreadId: parent },
688 forkedFrom: { type: "node", nodeId: NodeId.make("node:original") },
689 });
690 const originalRunId = yield* createRun(child, "completed");
691 yield* projections.apply({
692 id: EventId.make("event:original-result"),
693 type: "context-transfer.created",
694 threadId: parent,
695 occurredAt: now,
696 payload: {
697 id: ContextTransferId.make("transfer:original-result"),
698 type: "subagent_result",
699 sourceThreadId: child,
700 targetThreadId: parent,
701 sourcePoint: { threadId: child, runId: originalRunId },
702 basePoint: null,
703 sourceProviderInstanceId: providerInstanceId,
704 targetProviderInstanceId: providerInstanceId,
705 targetRunId: null,
706 status: "pending",
707 resolution: null,
708 createdBy: "system",
709 error: null,
710 createdAt: now,
711 updatedAt: now,
712 consumedAt: null,
713 },
714 });
715 yield* createRun(child, "completed", { ordinal: 2 });
716 assert.deepEqual(yield* projections.getRecoveryThreadIds("subagent-results"), []);
717 const taskId = NodeId.make("node:followup");
718 yield* createRun(child, "completed", { ordinal: 3, delegatedTaskId: taskId });
719 yield* createRun(child, "running", { ordinal: 4 });
720 const task = {
721 id: taskId,
722 threadId: parent,
723 runId: null,
724 parentNodeId: NodeId.make("node:parent"),
725 origin: "app_owned" as const,
726 createdBy: "agent" as const,
727 driver,
728 providerInstanceId,
729 providerThreadId: null,
730 childThreadId: child,
731 nativeTaskRef: null,
732 prompt: "Check the follow-up.",
733 title: null,
734 model: null,
735 status: "running" as const,
736 result: null,
737 startedAt: now,
738 completedAt: null,
739 updatedAt: now,
740 };
741 yield* projections.apply({
742 id: EventId.make("event:followup-task"),
743 type: "subagent.updated",
744 threadId: parent,
745 occurredAt: now,
746 payload: task,
747 });
748 assert.deepEqual(yield* projections.getRecoveryThreadIds("subagent-results"), [child]);
749 yield* projections.apply({
750 id: EventId.make("event:followup-settled"),
751 type: "subagent.updated",
752 threadId: parent,
753 occurredAt: now,
754 payload: { ...task, status: "completed", result: "Follow-up result.", completedAt: now },
755 });
756 assert.deepEqual(yield* projections.getRecoveryThreadIds("subagent-results"), []);
757 }).pipe(Effect.provide(layerTest)),
758 );
759
760 it.effect("includes shared sessions and provider-owned background rosters in recovery", () =>
761 Effect.gen(function* () {
762 const projections = yield* ProjectionStore.ProjectionStoreV2;
763 const now = yield* DateTime.now;
764 const first = yield* createThread("shared-first");
765 const second = yield* createThread("shared-second", { archivedAt: now });
766 const idleShared = [first, second];
767 const sharedWork = yield* createThread("shared-work");
768 const providerSessionId = ProviderSessionId.make("session:recovery:shared");
769 for (const threadId of [...idleShared, sharedWork]) {
770 yield* projections.apply({
771 id: EventId.make(`event:${threadId}:session`),
772 type: "provider-session.attached",
773 threadId,
774 driver,
775 providerInstanceId,
776 occurredAt: now,
777 payload: {
778 id: providerSessionId,
779 driver,
780 providerInstanceId,
781 status: "ready",
782 cwd: "/workspace",
783 model: modelSelection.model,
784 capabilities: CodexProviderCapabilitiesV2,
785 createdAt: now,
786 updatedAt: now,
787 lastError: null,
788 },
789 });
790 }
791 yield* projections.apply({
792 id: EventId.make("event:recovery:shared-work"),
793 type: "provider-thread.updated",
794 threadId: sharedWork,
795 driver,
796 providerInstanceId,
797 occurredAt: now,
798 payload: {
799 id: ProviderThreadId.make("provider-thread:recovery:shared-work"),
800 appThreadId: sharedWork,
801 ownerNodeId: null,
802 driver,
803 providerInstanceId,
804 providerSessionId,
805 nativeThreadRef: { driver, nativeId: "native:shared-work", strength: "strong" },
806 nativeConversationHeadRef: null,
807 status: "idle",
808 firstRunOrdinal: 1,
809 lastRunOrdinal: 1,
810 handoffIds: [],
811 forkedFrom: null,
812 createdAt: now,
813 updatedAt: now,
814 pendingBackgroundTasks: [
815 { taskId: "awaited", description: "Awaited subagent", kind: "subagent" },
816 ],
817 },
818 });
819 const roster = yield* createThread("roster");
820 yield* projections.apply({
821 id: EventId.make("event:recovery:roster"),
822 type: "provider-thread.updated",
823 threadId: roster,
824 driver,
825 providerInstanceId,
826 occurredAt: now,
827 payload: {
828 id: ProviderThreadId.make("provider-thread:recovery:roster"),
829 appThreadId: roster,
830 ownerNodeId: null,
831 driver,
832 providerInstanceId,
833 providerSessionId: null,
834 nativeThreadRef: null,
835 nativeConversationHeadRef: null,
836 status: "idle",
837 firstRunOrdinal: null,
838 lastRunOrdinal: null,
839 handoffIds: [],
840 forkedFrom: null,
841 createdAt: now,
842 updatedAt: now,
843 pendingBackgroundTasks: [
844 { taskId: "background", description: "Still running", kind: "command" },
845 ],
846 },
847 });
848 const prepared = yield* createThread("prepared-continuation");
849 const preparedSessionId = ProviderSessionId.make("session:recovery:prepared");
850 const preparedProviderThreadId = ProviderThreadId.make("provider-thread:recovery:prepared");
851 yield* projections.apply({
852 id: EventId.make("event:recovery:prepared-session"),
853 type: "provider-session.attached",
854 threadId: prepared,
855 driver,
856 providerInstanceId,
857 occurredAt: now,
858 payload: {
859 id: preparedSessionId,
860 driver,
861 providerInstanceId,
862 status: "stopped",
863 cwd: "/workspace",
864 model: modelSelection.model,
865 capabilities: CodexProviderCapabilitiesV2,
866 createdAt: now,
867 updatedAt: now,
868 lastError: null,
869 },
870 });
871 yield* projections.apply({
872 id: EventId.make("event:recovery:prepared-thread"),
873 type: "provider-thread.updated",
874 threadId: prepared,
875 driver,
876 providerInstanceId,
877 occurredAt: now,
878 payload: {
879 id: preparedProviderThreadId,
880 appThreadId: prepared,
881 ownerNodeId: null,
882 driver,
883 providerInstanceId,
884 providerSessionId: preparedSessionId,
885 nativeThreadRef: { driver, nativeId: "native:prepared", strength: "strong" },
886 nativeConversationHeadRef: null,
887 status: "idle",
888 firstRunOrdinal: 1,
889 lastRunOrdinal: 1,
890 handoffIds: [],
891 forkedFrom: null,
892 createdAt: now,
893 updatedAt: now,
894 pendingBackgroundTasks: [],
895 },
896 });
897 const preparedRunId = yield* createRun(prepared, "starting", {
898 providerThreadId: preparedProviderThreadId,
899 restartContinuationOfRunId: RunId.make("run:recovery:source"),
900 });
901 assert.deepEqual(
902 new Set(yield* projections.getRecoveryThreadIds("restart-continuations")),
903 new Set([sharedWork, prepared]),
904 );
905 // One idle bound thread stops the shared session; the rest have nothing to recover.
906 const runtime = yield* projections.getRecoveryThreadIds("runtime");
907 assert.deepEqual(
908 new Set(runtime.filter((threadId) => !idleShared.includes(threadId))),
909 new Set([sharedWork, roster, prepared]),
910 );
911 assert.equal(runtime.filter((threadId) => idleShared.includes(threadId)).length, 1);
912 assert.deepEqual(
913 yield* projections.getRecoveryThreadIds("restart-continuations", { include: [] }),
914 [],
915 );
916 assert.deepEqual(
917 yield* projections.getRecoveryThreadIds("restart-continuations", {
918 exclude: ["project:recovery"],
919 }),
920 [],
921 );
922 assert.deepEqual(
923 new Set(
924 yield* projections.getRecoveryThreadIds("restart-continuations", {
925 include: ["project:recovery"],
926 }),
927 ),
928 new Set([sharedWork, prepared]),
929 );
930 const preparedState = yield* projections.getRuntimeRecoveryProjection(prepared);
931 assert.deepEqual(
932 preparedState.providerSessions.map((session) => session.id),
933 [preparedSessionId],
934 );
935 assert.equal(restartContinuationRun(preparedState, yield* DateTime.now)?.id, preparedRunId);
936 assert.deepEqual(yield* projections.getUnreadableThreadIds(), []);
937 const sql = yield* SqlClient.SqlClient;
938 yield* sql`
939 UPDATE orchestration_v2_projection_provider_sessions SET payload_json = '{}'
940 WHERE provider_session_id = ${providerSessionId}
941 `;
942 assert.deepEqual(
943 new Set(yield* projections.getUnreadableThreadIds()),
944 new Set([...idleShared, sharedWork]),
945 );
946 }).pipe(Effect.provide(layerTest)),
947 );
948
949 it.effect("bounds streaming recovery reads by candidate runs and preserves message ordering", () =>
950 Effect.gen(function* () {
951 const projections = yield* ProjectionStore.ProjectionStoreV2;
952 const sql = yield* SqlClient.SqlClient;
953 const now = yield* DateTime.now;
954 const threadId = yield* createThread("streaming");
955 const other = yield* createThread("streaming-other");
956 const empty = yield* createThread("streaming-empty");
957 const statements: Array<ReturnType<Statement.Statement<unknown>["compile"]>> = [];
958 const observe: Statement.Transformer = (statement) => {
959 const compiled = statement.compile();
960 if (compiled[0].includes("SELECT message.payload_json")) statements.push(compiled);
961 return Effect.succeed(statement);
962 };
963 const expected: MessageId[] = [];
964 const statuses = [
965 "queued",
966 "preparing",
967 "starting",
968 "running",
969 "waiting",
970 "completed",
971 "interrupted",
972 "rolled_back",
973 "failed",
974 "cancelled",
975 ] as const;
976 for (const [index, status] of statuses.entries()) {
977 const runId = yield* createRun(threadId, status, { ordinal: index + 1 });
978 for (const suffix of ["b", "a", "settled", "wrong-thread"]) {
979 const id = MessageId.make(`message:streaming:${index}:${suffix}`);
980 const streaming = suffix !== "settled";
981 const messageThread = suffix === "wrong-thread" ? other : threadId;
982 const createdAt = DateTime.add(now, { seconds: -index });
983 yield* projections.apply({
984 id: EventId.make(`event:${id}`),
985 type: "message.updated",
986 threadId: messageThread,
987 runId,
988 occurredAt: now,
989 payload: {
990 id,
991 threadId: messageThread,
992 runId,
993 nodeId: null,
994 role: "assistant",
995 text: id,
996 attachments: [],
997 streaming,
998 createdAt,
999 updatedAt: now,
1000 createdBy: "agent",
1001 creationSource: "provider",
1002 },
1003 });
1004 }
1005 if (index < 5)
1006 expected.unshift(
1007 MessageId.make(`message:streaming:${index}:a`),
1008 MessageId.make(`message:streaming:${index}:b`),
1009 );
1010 }
1011 const foreignRun = yield* createRun(other, "running");
1012 yield* sql`
1013 INSERT INTO orchestration_v2_projection_messages
1014 (message_id, thread_id, run_id, role, streaming, created_at, updated_at, payload_json)
1015 VALUES ('foreign-run', ${threadId}, ${foreignRun}, 'assistant', 1, '', '', '{}')
1016 `;
1017 yield* sql`
1018 INSERT INTO orchestration_v2_projection_messages
1019 (message_id, thread_id, run_id, role, streaming, created_at, updated_at, payload_json)
1020 VALUES ('old-history', ${empty}, NULL, 'assistant', 1, '', '', '{}')
1021 `;
1022 const projection = yield* projections
1023 .getRuntimeRecoveryProjection(threadId)
1024 .pipe(Effect.provideService(Statement.CurrentTransformer, observe));
1025 assert.deepEqual(
1026 projection.messages.map((message) => message.id),
1027 expected,
1028 );
1029 const emptyProjection = yield* projections
1030 .getRuntimeRecoveryProjection(empty)
1031 .pipe(Effect.provideService(Statement.CurrentTransformer, observe));
1032 assert.deepEqual(emptyProjection.messages, []);
1033 assert.equal(statements.length, 2);
1034 for (const [query, params] of statements) {
1035 const plan = yield* sql.unsafe<{ detail: string }>(`EXPLAIN QUERY PLAN ${query}`, params);
1036 assert.isTrue(
1037 plan.some((row) => row.detail.includes("SEARCH message_run ")),
1038 JSON.stringify(plan),
1039 );
1040 assert.isTrue(
1041 plan.some((row) => row.detail.includes("messages_run_idx")),
1042 JSON.stringify(plan),
1043 );
1044 assert.isFalse(
1045 plan.some((row) => /\bSCAN message\b/.test(row.detail)),
1046 JSON.stringify(plan),
1047 );
1048 assert.isFalse(
1049 plan.some((row) => row.detail.includes("messages_thread_created_idx")),
1050 JSON.stringify(plan),
1051 );
1052 }
1053 }).pipe(Effect.provide(layerTest)),
1054 );
1055
1056 it.effect("bounds runtime provider-thread reads by ownership and preserves recovery liveness", () =>
1057 Effect.gen(function* () {
1058 const projections = yield* ProjectionStore.ProjectionStoreV2;
1059 const sql = yield* SqlClient.SqlClient;
1060 const now = yield* DateTime.now;
1061 const threadId = yield* createThread("ownership");
1062 const other = yield* createThread("other-owner");
1063 const ownerNodeId = NodeId.make("node:recovery:owner");
1064 yield* projections.apply({
1065 id: EventId.make("event:recovery:owner"),
1066 type: "node.updated",
1067 threadId,
1068 nodeId: ownerNodeId,
1069 driver,
1070 occurredAt: now,
1071 payload: {
1072 id: ownerNodeId,
1073 threadId,
1074 runId: null,
1075 parentNodeId: null,
1076 rootNodeId: ownerNodeId,
1077 kind: "root_turn",
1078 status: "completed",
1079 countsForRun: false,
1080 providerThreadId: null,
1081 providerTurnId: null,
1082 nativeItemRef: null,
1083 runtimeRequestId: null,
1084 checkpointScopeId: null,
1085 startedAt: now,
1086 completedAt: now,
1087 },
1088 });
1089 const cases = [
1090 { name: "direct", ownership: "direct", status: "active", ordinal: 2, included: true },
1091 { name: "owner", ownership: "owner", status: "active", ordinal: null, included: true },
1092 { name: "owner-only", ownership: "owner", status: "active", ordinal: 1, included: true },
1093 { name: "subagent", ownership: "subagent", status: "active", ordinal: 1, included: true },
1094 { name: "duplicate", ownership: "all", status: "active", ordinal: 1, included: true },
1095 { name: "roster", ownership: "direct", status: "idle", ordinal: 3, included: true },
1096 { name: "run", ownership: "owner", status: "idle", ordinal: 4, included: true },
1097 { name: "item", ownership: "subagent", status: "idle", ordinal: 5, included: true },
1098 { name: "waiting-run", ownership: "direct", status: "idle", ordinal: 6, included: true },
1099 { name: "settled", ownership: "all", status: "idle", ordinal: 7, included: false },
1100 { name: "unrelated", ownership: "none", status: "active", ordinal: 8, included: false },
1101 { name: "unrelated-run", ownership: "none", status: "idle", ordinal: 9, included: false },
1102 ] as const;
1103 for (const entry of cases) {
1104 const id = ProviderThreadId.make(`provider-thread:ownership:${entry.name}`);
1105 yield* projections.apply({
1106 id: EventId.make(`event:${id}`),
1107 type: "provider-thread.updated",
1108 threadId,
1109 driver,
1110 providerInstanceId,
1111 occurredAt: now,
1112 payload: {
1113 id,
1114 appThreadId:
1115 entry.name === "owner-only"
1116 ? null
1117 : entry.ownership === "direct" || entry.ownership === "all"
1118 ? threadId
1119 : other,
1120 ownerNodeId:
1121 entry.ownership === "owner" || entry.ownership === "all" ? ownerNodeId : null,
1122 driver,
1123 providerInstanceId,
1124 providerSessionId: null,
1125 nativeThreadRef: null,
1126 nativeConversationHeadRef: null,
1127 status: entry.status,
1128 firstRunOrdinal: entry.ordinal,
1129 lastRunOrdinal: entry.ordinal,
1130 handoffIds: [],
1131 forkedFrom: null,
1132 createdAt: now,
1133 updatedAt: now,
1134 pendingBackgroundTasks:
1135 entry.name === "roster"
1136 ? [{ taskId: "shell", description: "background", kind: "command" }]
1137 : [],
1138 },
1139 });
1140 if (entry.ownership === "subagent" || entry.ownership === "all") {
1141 for (const suffix of ["a", "b"]) {
1142 yield* sql`
1143 INSERT INTO orchestration_v2_projection_subagents
1144 (subagent_id, thread_id, parent_node_id, provider, provider_thread_id,
1145 origin, status, updated_at, payload_json)
1146 VALUES (${`${id}:${suffix}`}, ${threadId}, ${ownerNodeId}, 'codex', ${id},
1147 'provider_native', 'completed', '', '{}')
1148 `;
1149 }
1150 }
1151 if (entry.name === "run" || entry.name === "unrelated-run") {
1152 yield* createRun(threadId, "waiting", { ordinal: entry.ordinal, providerThreadId: id });
1153 }
1154 }
1155 const waitingProviderThreadId = ProviderThreadId.make("provider-thread:ownership:waiting-run");
1156 const waitingRunId = yield* createRun(threadId, "completed", {
1157 ordinal: 6,
1158 providerThreadId: waitingProviderThreadId,
1159 });
1160 yield* sql`
1161 INSERT INTO orchestration_v2_projection_turn_items
1162 (turn_item_id, thread_id, run_id, node_id, provider_thread_id, ordinal, type,
1163 status, updated_at, payload_json)
1164 VALUES ('item:ownership', ${threadId}, ${waitingRunId}, ${ownerNodeId},
1165 ${ProviderThreadId.make("provider-thread:ownership:item")}, 1, 'assistant_message', 'waiting', '', '{}')
1166 `;
1167 const statements: Array<ReturnType<Statement.Statement<unknown>["compile"]>> = [];
1168 const nodeStatements: typeof statements = [];
1169 const observe: Statement.Transformer = (statement) => {
1170 const compiled = statement.compile();
1171 if (compiled[0].includes("SELECT provider_thread.payload_json")) statements.push(compiled);
1172 if (compiled[0].includes("SELECT node.payload_json")) nodeStatements.push(compiled);
1173 return Effect.succeed(statement);
1174 };
1175 const projection = yield* projections
1176 .getRuntimeRecoveryProjection(threadId)
1177 .pipe(Effect.provideService(Statement.CurrentTransformer, observe));
1178 assert.equal(nodeStatements.length, 1);
1179 const nodePlan = yield* sql.unsafe<{ detail: string }>(
1180 `EXPLAIN QUERY PLAN ${nodeStatements[0]![0]}`,
1181 nodeStatements[0]![1],
1182 );
1183 assert.isTrue(
1184 nodePlan.some((row) => /^SEARCH node USING INDEX .*nodes_live_idx/.test(row.detail)),
1185 JSON.stringify(nodePlan),
1186 );
1187 assert.deepEqual(
1188 projection.providerThreads.map((row) => row.id),
1189 cases
1190 .filter((entry) => entry.included)
1191 .toSorted((a, b) => (a.ordinal ?? 0) - (b.ordinal ?? 0) || a.name.localeCompare(b.name))
1192 .map((entry) => ProviderThreadId.make(`provider-thread:ownership:${entry.name}`)),
1193 );
1194 assert.equal(statements.length, 1);
1195 const [query, params] = statements[0]!;
1196 const plan = yield* sql.unsafe<{ detail: string }>(`EXPLAIN QUERY PLAN ${query}`, params);
1197 assert.isFalse(
1198 plan.some((row) => /^SCAN provider_thread(?:$| )/.test(row.detail)),
1199 JSON.stringify(plan),
1200 );
1201 assert.isTrue(
1202 plan.some((row) => row.detail.includes("provider_threads_owner_idx")),
1203 JSON.stringify(plan),
1204 );
1205 assert.isFalse(
1206 plan.some((row) => /SEARCH (owner|n) USING INDEX .*nodes_thread/.test(row.detail)),
1207 JSON.stringify(plan),
1208 );
1209 }).pipe(Effect.provide(layerTest)),
1210 );
1211
1212 it.effect("reads the run that owns a background roster, not a queued or resumed one", () =>
1213 Effect.gen(function* () {
1214 const projections = yield* ProjectionStore.ProjectionStoreV2;
1215 const now = yield* DateTime.now;
1216 const withRoster = Effect.fn(function* (threadId: ThreadId) {
1217 yield* projections.apply({
1218 id: EventId.make(`event:${threadId}:roster`),
1219 type: "provider-thread.updated",
1220 threadId,
1221 driver,
1222 providerInstanceId,
1223 occurredAt: now,
1224 payload: {
1225 id: ProviderThreadId.make(`provider-thread:${threadId}`),
1226 appThreadId: threadId,
1227 ownerNodeId: null,
1228 driver,
1229 providerInstanceId,
1230 providerSessionId: null,
1231 nativeThreadRef: null,
1232 nativeConversationHeadRef: null,
1233 status: "idle",
1234 firstRunOrdinal: null,
1235 lastRunOrdinal: null,
1236 handoffIds: [],
1237 forkedFrom: null,
1238 createdAt: now,
1239 updatedAt: now,
1240 pendingBackgroundTasks: [
1241 { taskId: "background", description: "Still running", kind: "command" },
1242 ],
1243 },
1244 });
1245 });
1246 const runIds = (threadId: ThreadId) =>
1247 projections
1248 .getRuntimeRecoveryProjection(threadId)
1249 .pipe(Effect.map((state) => state.runs.map((run) => run.id)));
1250 const settled = yield* createThread("roster-before-queue");
1251 const settledRun = yield* createRun(settled, "completed", {
1252 providerThreadId: ProviderThreadId.make(`provider-thread:${settled}`),
1253 });
1254 const queuedRun = yield* createRun(settled, "queued", { ordinal: 2, completedAt: null });
1255 yield* withRoster(settled);
1256 assert.deepEqual(yield* runIds(settled), [settledRun, queuedRun]);
1257 const resumed = yield* createThread("roster-after-resume");
1258 const resumedRun = yield* createRun(resumed, "completed", {
1259 providerThreadId: ProviderThreadId.make(`provider-thread:${resumed}`),
1260 completedAt: DateTime.makeUnsafe("2026-10-03T10:05:00.000Z"),
1261 });
1262 const continuationRun = yield* createRun(resumed, "completed", {
1263 providerThreadId: ProviderThreadId.make(`provider-thread:${resumed}`),
1264 ordinal: 2,
1265 completedAt: DateTime.makeUnsafe("2026-10-03T10:00:00.000Z"),
1266 });
1267 yield* withRoster(resumed);
1268 assert.deepEqual(yield* runIds(resumed), [resumedRun, continuationRun]);
1269 const quiet = yield* createThread("no-roster-before-queue");
1270 yield* createRun(quiet, "completed");
1271 const quietQueued = yield* createRun(quiet, "queued", { ordinal: 2, completedAt: null });
1272 assert.deepEqual(yield* runIds(quiet), [quietQueued]);
1273 }).pipe(Effect.provide(layerTest)),
1274 );
1275
1276 it.effect.each([
1277 ["a wake cut before it started", { status: "starting", startedAt: null }],
1278 [
1279 "a prompt the provider never accepted",
1280 { status: "running", activeAttemptId: RunAttemptId.make("attempt:unaccepted") },
1281 ],
1282 ] as const)("wakes the run whose subagent lost its shell, past %s", ([, later]) =>
1283 Effect.gen(function* () {
1284 const projections = yield* ProjectionStore.ProjectionStoreV2;
1285 const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
1286 const sql = yield* SqlClient.SqlClient;
1287 const now = yield* DateTime.now;
1288 const threadId = yield* createThread("subagent-shell");
1289 const providerThreadId = ProviderThreadId.make(`provider-thread:${threadId}`);
1290 const providerSessionId = ProviderSessionId.make(`session:${threadId}`);
1291 yield* projections.apply({
1292 id: EventId.make(`event:${threadId}:session`),
1293 type: "provider-session.attached",
1294 threadId,
1295 driver,
1296 providerInstanceId,
1297 occurredAt: now,
1298 payload: {
1299 id: providerSessionId,
1300 driver,
1301 providerInstanceId,
1302 status: "ready",
1303 cwd: "/workspace",
1304 model: modelSelection.model,
1305 capabilities: CodexProviderCapabilitiesV2,
1306 createdAt: now,
1307 updatedAt: now,
1308 lastError: null,
1309 },
1310 });
1311 const settledRun = yield* createRun(threadId, "completed", { providerThreadId });
1312 yield* createRun(threadId, later.status, {
1313 ordinal: 2,
1314 providerThreadId,
1315 completedAt: null,
1316 ...later,
1317 });
1318 yield* projections.apply({
1319 id: EventId.make(`event:${threadId}:roster`),
1320 type: "provider-thread.updated",
1321 threadId,
1322 driver,
1323 providerInstanceId,
1324 occurredAt: now,
1325 payload: {
1326 id: providerThreadId,
1327 appThreadId: threadId,
1328 ownerNodeId: null,
1329 driver,
1330 providerInstanceId,
1331 providerSessionId,
1332 nativeThreadRef: { driver, nativeId: "native-thread", strength: "strong" },
1333 nativeConversationHeadRef: null,
1334 status: "idle",
1335 firstRunOrdinal: 1,
1336 lastRunOrdinal: 2,
1337 handoffIds: [],
1338 forkedFrom: null,
1339 createdAt: now,
1340 updatedAt: now,
1341 pendingBackgroundTasks: [
1342 { taskId: "shell", description: "sleep 150", kind: "command", parentTaskId: "agent" },
1343 ],
1344 },
1345 });
1346
1347 yield* recovery.reconcile("startup");
1348
1349 const effects = yield* sql<{ readonly effect_id: string }>`
1350 SELECT effect_id FROM orchestration_v2_effect_outbox WHERE thread_id = ${threadId}
1351 `;
1352 assert.deepEqual(effects, [{ effect_id: `effect:restart-continuation:${settledRun}` }]);
1353 const projection = yield* projections.getThreadProjection(threadId);
1354 assert.deepEqual(projection.runs[0]?.restartCancelledBackgroundWork, [
1355 { kind: "subagent", label: "sleep 150 (subagent agent)", id: "agent" },
1356 ]);
1357 }).pipe(Effect.provide(layerRecoveryTest)),
1358 );
1359
1360 it.effect("wakes a settled run whose native subagent ran on its own provider thread", () =>
1361 Effect.gen(function* () {
1362 const projections = yield* ProjectionStore.ProjectionStoreV2;
1363 const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
1364 const sql = yield* SqlClient.SqlClient;
1365 const now = yield* DateTime.now;
1366 const threadId = yield* createThread("native-subagent");
1367 const parentProviderThreadId = ProviderThreadId.make(`provider-thread:${threadId}`);
1368 const childProviderThreadId = ProviderThreadId.make(`provider-thread:${threadId}:child`);
1369 const providerSessionId = ProviderSessionId.make(`session:${threadId}`);
1370 const subagentId = NodeId.make(`node:${threadId}:subagent`);
1371 // An older thread on the same Codex session is recovered first and stops it.
1372 const sibling = yield* createThread("a-shared-session");
1373 yield* createRun(sibling, "completed");
1374 for (const boundThreadId of [sibling, threadId]) {
1375 yield* projections.apply({
1376 id: EventId.make(`event:${boundThreadId}:session`),
1377 type: "provider-session.attached",
1378 threadId: boundThreadId,
1379 driver,
1380 providerInstanceId,
1381 occurredAt: now,
1382 payload: {
1383 id: providerSessionId,
1384 driver,
1385 providerInstanceId,
1386 status: "ready",
1387 cwd: "/workspace",
1388 model: modelSelection.model,
1389 capabilities: CodexProviderCapabilitiesV2,
1390 createdAt: now,
1391 updatedAt: now,
1392 lastError: null,
1393 },
1394 });
1395 }
1396 const settledRun = yield* createRun(threadId, "completed", {
1397 providerThreadId: parentProviderThreadId,
1398 });
1399 for (const [id, nativeId] of [
1400 [parentProviderThreadId, "native-thread"],
1401 [childProviderThreadId, "agent-thread"],
1402 ] as const) {
1403 yield* projections.apply({
1404 id: EventId.make(`event:${id}`),
1405 type: "provider-thread.updated",
1406 threadId,
1407 driver,
1408 providerInstanceId,
1409 occurredAt: now,
1410 payload: {
1411 id,
1412 appThreadId:
1413 id === parentProviderThreadId ? threadId : ThreadId.make(`${threadId}:child`),
1414 ownerNodeId: null,
1415 driver,
1416 providerInstanceId,
1417 providerSessionId,
1418 nativeThreadRef: { driver, nativeId, strength: "strong" },
1419 nativeConversationHeadRef: null,
1420 status: "idle",
1421 firstRunOrdinal: id === parentProviderThreadId ? 1 : null,
1422 lastRunOrdinal: id === parentProviderThreadId ? 1 : null,
1423 handoffIds: [],
1424 forkedFrom: null,
1425 createdAt: now,
1426 updatedAt: now,
1427 },
1428 });
1429 }
1430 const subagent = {
1431 threadId,
1432 runId: settledRun,
1433 origin: "provider_native",
1434 driver,
1435 providerInstanceId,
1436 childThreadId: null,
1437 prompt: "audit src",
1438 status: "running",
1439 result: null,
1440 startedAt: now,
1441 completedAt: null,
1442 updatedAt: now,
1443 } as const;
1444 yield* projections.apply({
1445 id: EventId.make(`event:${threadId}:subagent`),
1446 type: "subagent.updated",
1447 threadId,
1448 occurredAt: now,
1449 payload: {
1450 ...subagent,
1451 id: subagentId,
1452 parentNodeId: NodeId.make(`node:${threadId}:root`),
1453 createdBy: "agent",
1454 providerThreadId: childProviderThreadId,
1455 nativeTaskRef: { driver, nativeId: "call_1:agent-thread", strength: "strong" },
1456 title: null,
1457 model: null,
1458 },
1459 });
1460 yield* projections.apply({
1461 id: EventId.make(`event:${threadId}:subagent-item`),
1462 type: "turn-item.updated",
1463 threadId,
1464 occurredAt: now,
1465 payload: {
1466 ...subagent,
1467 id: TurnItemId.make(`item:${threadId}:subagent`),
1468 nodeId: subagentId,
1469 providerThreadId: childProviderThreadId,
1470 providerTurnId: null,
1471 nativeItemRef: { driver, nativeId: "call_1:agent-thread", strength: "strong" },
1472 parentItemId: null,
1473 ordinal: 1,
1474 type: "subagent",
1475 title: "audit",
1476 subagentId,
1477 },
1478 });
1479
1480 yield* recovery.reconcile("startup");
1481
1482 const effects = yield* sql<{ readonly effect_id: string }>`
1483 SELECT effect_id FROM orchestration_v2_effect_outbox WHERE thread_id = ${threadId}
1484 `;
1485 assert.deepEqual(effects, [{ effect_id: `effect:restart-continuation:${settledRun}` }]);
1486 const projection = yield* projections.getThreadProjection(threadId);
1487 assert.deepEqual(projection.runs[0]?.restartCancelledBackgroundWork, [
1488 {
1489 kind: "subagent",
1490 label: "audit (id agent-thread)",
1491 id: TurnItemId.make(`item:${threadId}:subagent`),
1492 },
1493 ]);
1494 }).pipe(Effect.provide(layerRecoveryTest)),
1495 );
1496
1497 it.effect("leaves a finished delegated child's background roster to the child", () =>
1498 Effect.gen(function* () {
1499 const projections = yield* ProjectionStore.ProjectionStoreV2;
1500 const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
1501 const sql = yield* SqlClient.SqlClient;
1502 const now = yield* DateTime.now;
1503 // The parent sorts first, so it is reconciled before the child.
1504 const parent = yield* createThread("delegation");
1505 yield* createRun(parent, "completed");
1506 const child = yield* createThread("delegation:child", {
1507 lineage: { parentThreadId: parent, relationshipToParent: "subagent", rootThreadId: parent },
1508 });
1509 const childProviderThreadId = ProviderThreadId.make(`provider-thread:${child}`);
1510 const childSessionId = ProviderSessionId.make(`session:${child}`);
1511 yield* projections.apply({
1512 id: EventId.make(`event:${child}:session`),
1513 type: "provider-session.attached",
1514 threadId: child,
1515 driver,
1516 providerInstanceId,
1517 occurredAt: now,
1518 payload: {
1519 id: childSessionId,
1520 driver,
1521 providerInstanceId,
1522 status: "ready",
1523 cwd: "/workspace",
1524 model: modelSelection.model,
1525 capabilities: CodexProviderCapabilitiesV2,
1526 createdAt: now,
1527 updatedAt: now,
1528 lastError: null,
1529 },
1530 });
1531 const childRun = yield* createRun(child, "completed", {
1532 providerThreadId: childProviderThreadId,
1533 });
1534 yield* projections.apply({
1535 id: EventId.make(`event:${child}:roster`),
1536 type: "provider-thread.updated",
1537 threadId: child,
1538 driver,
1539 providerInstanceId,
1540 occurredAt: now,
1541 payload: {
1542 id: childProviderThreadId,
1543 appThreadId: child,
1544 ownerNodeId: null,
1545 driver,
1546 providerInstanceId,
1547 providerSessionId: childSessionId,
1548 nativeThreadRef: { driver, nativeId: "child-thread", strength: "strong" },
1549 nativeConversationHeadRef: null,
1550 status: "idle",
1551 firstRunOrdinal: 1,
1552 lastRunOrdinal: 1,
1553 handoffIds: [],
1554 forkedFrom: null,
1555 createdAt: now,
1556 updatedAt: now,
1557 pendingBackgroundTasks: [
1558 { taskId: "wait", description: "sleep 600", kind: "command", awaited: true },
1559 ],
1560 },
1561 });
1562 // Finalizing a delegated task records the child's provider thread on the parent's row.
1563 const taskId = NodeId.make(`node:${parent}:task`);
1564 yield* projections.apply({
1565 id: EventId.make(`event:${parent}:task`),
1566 type: "subagent.updated",
1567 threadId: parent,
1568 occurredAt: now,
1569 payload: {
1570 id: taskId,
1571 threadId: parent,
1572 runId: RunId.make(`run:${parent}:1`),
1573 parentNodeId: NodeId.make(`node:${parent}:root`),
1574 origin: "app_owned",
1575 createdBy: "agent",
1576 driver,
1577 providerInstanceId,
1578 providerThreadId: childProviderThreadId,
1579 childThreadId: child,
1580 nativeTaskRef: null,
1581 prompt: "wait for the build",
1582 title: null,
1583 model: null,
1584 status: "completed",
1585 result: "Waiting on the build",
1586 startedAt: now,
1587 completedAt: now,
1588 updatedAt: now,
1589 },
1590 });
1591
1592 yield* recovery.reconcile("startup");
1593
1594 const effects = yield* sql<{ readonly effect_id: string }>`
1595 SELECT effect_id FROM orchestration_v2_effect_outbox WHERE thread_id = ${child}
1596 `;
1597 assert.deepEqual(effects, [{ effect_id: `effect:restart-continuation:${childRun}` }]);
1598 const projection = yield* projections.getThreadProjection(child);
1599 assert.deepEqual(projection.runs[0]?.restartCancelledBackgroundWork, [
1600 { kind: "shell", label: "sleep 600 (id wait)", id: "wait", awaited: true },
1601 ]);
1602 }).pipe(Effect.provide(layerRecoveryTest)),
1603 );
1604
1605 it.effect("marks fork descendants unreadable when their source is missing or corrupt", () =>
1606 Effect.gen(function* () {
1607 const projections = yield* ProjectionStore.ProjectionStoreV2;
1608 const sql = yield* SqlClient.SqlClient;
1609 const source = yield* createThread("source");
1610 const sourceRun = yield* createRun(source, "completed");
1611 const fork = yield* createThread("fork", {
1612 forkedFrom: { type: "run", threadId: source, runId: sourceRun },
1613 });
1614 const descendant = yield* createThread("descendant", {
1615 forkedFrom: { type: "run", threadId: fork, runId: RunId.make("run:fork") },
1616 });
1617 assert.deepEqual(yield* projections.getUnreadableThreadIds(), []);
1618 yield* sql`
1619 UPDATE orchestration_v2_projection_runs SET payload_json = '{}'
1620 WHERE run_id = ${sourceRun}
1621 `;
1622 assert.deepEqual(
1623 new Set(yield* projections.getUnreadableThreadIds()),
1624 new Set([source, fork, descendant]),
1625 );
1626 yield* sql`DELETE FROM orchestration_v2_projection_threads WHERE thread_id = ${source}`;
1627 assert.deepEqual(
1628 new Set(yield* projections.getUnreadableThreadIds()),
1629 new Set([fork, descendant]),
1630 );
1631 }).pipe(Effect.provide(layerTest)),
1632 );
1633
1634 it.effect("startup ends the provider turn of a native child root it cancels", () =>
1635 Effect.gen(function* () {
1636 const projections = yield* ProjectionStore.ProjectionStoreV2;
1637 const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
1638 const now = yield* DateTime.now;
1639 const parentThreadId = yield* createThread("native-child-root-turn-parent");
1640 const threadId = yield* createThread("native-child-root-turn");
1641 const rootNodeId = NodeId.make(`node:${threadId}:thread-root`);
1642 const providerThreadId = ProviderThreadId.make(`provider-thread:${threadId}`);
1643 const providerTurnId = ProviderTurnId.make(`provider-turn:${threadId}`);
1644 yield* projections.apply({
1645 id: EventId.make(`event:${threadId}:root`),
1646 type: "node.updated",
1647 threadId,
1648 nodeId: rootNodeId,
1649 driver,
1650 occurredAt: now,
1651 payload: {
1652 id: rootNodeId,
1653 threadId,
1654 runId: null,
1655 parentNodeId: null,
1656 rootNodeId,
1657 kind: "root_turn",
1658 status: "running",
1659 countsForRun: false,
1660 providerThreadId,
1661 providerTurnId,
1662 nativeItemRef: null,
1663 runtimeRequestId: null,
1664 checkpointScopeId: null,
1665 startedAt: now,
1666 completedAt: null,
1667 },
1668 });
1669 // A native subagent's turn belongs to no run attempt.
1670 yield* projections.apply({
1671 id: EventId.make(`event:${threadId}:provider-turn`),
1672 type: "provider-turn.updated",
1673 threadId,
1674 nodeId: rootNodeId,
1675 driver,
1676 occurredAt: now,
1677 payload: {
1678 id: providerTurnId,
1679 providerThreadId,
1680 nodeId: rootNodeId,
1681 runAttemptId: null,
1682 nativeTurnRef: null,
1683 ordinal: 1,
1684 status: "running",
1685 startedAt: now,
1686 completedAt: null,
1687 },
1688 });
1689
1690 const subagentId = NodeId.make(`node:${parentThreadId}:subagent`);
1691 yield* projections.apply({
1692 id: EventId.make(`event:${parentThreadId}:subagent`),
1693 type: "subagent.updated",
1694 threadId: parentThreadId,
1695 occurredAt: now,
1696 payload: {
1697 id: subagentId,
1698 threadId: parentThreadId,
1699 runId: null,
1700 parentNodeId: NodeId.make(`node:${parentThreadId}:root`),
1701 origin: "provider_native",
1702 createdBy: "agent",
1703 driver,
1704 providerInstanceId,
1705 providerThreadId,
1706 childThreadId: threadId,
1707 nativeTaskRef: null,
1708 title: null,
1709 model: null,
1710 prompt: "sleep 300",
1711 status: "running",
1712 result: null,
1713 startedAt: now,
1714 completedAt: null,
1715 updatedAt: now,
1716 },
1717 });
1718
1719 yield* recovery.reconcile("startup");
1720
1721 const projection = yield* projections.getThreadProjection(threadId);
1722 assert.deepEqual(
1723 {
1724 root: projection.nodes.find((node) => node.id === rootNodeId)?.status,
1725 providerTurn: projection.providerTurns.find((turn) => turn.id === providerTurnId)?.status,
1726 },
1727 { root: "cancelled", providerTurn: "cancelled" },
1728 "a root startup recovery cancelled left its provider turn running",
1729 );
1730 }).pipe(Effect.provide(layerRecoveryTest)),
1731 );