@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 1,685 lines 59.2 KB
Raw
1 import { assert, it, vi } from "@effect/vitest";
2 import {
3 MessageId,
4 NodeId,
5 ProviderDriverKind,
6 ProviderInstanceId,
7 ProviderSessionId,
8 ProviderThreadId,
9 ProviderTurnId,
10 RunAttemptId,
11 RunId,
12 RuntimeRequestId,
13 ThreadId,
14 TurnItemId,
15 type OrchestrationV2ThreadProjection,
16 } from "@t3tools/contracts";
17 import * as DateTime from "effect/DateTime";
18 import * as Effect from "effect/Effect";
19 import * as Layer from "effect/Layer";
20
21 import * as EffectWorker from "./EffectWorker.ts";
22 import * as EffectOutbox from "./EffectOutbox.ts";
23 import * as EventSink from "./EventSink.ts";
24 import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
25 import * as ProjectionStore from "./ProjectionStore.ts";
26 import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts";
27 import * as ServerSettings from "../serverSettings.ts";
28
29 it.effect("expires orphaned runtime requests before command readiness", () => {
30 const threadId = ThreadId.make("thread_recovery_requests");
31 let committedInput: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] | null =
32 null;
33 const committed = vi.fn(
34 (input: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0]) => {
35 committedInput = input;
36 return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never);
37 },
38 );
39 const projection = {
40 thread: { id: threadId },
41 runtimeRequests: [
42 {
43 id: RuntimeRequestId.make("request_orphaned"),
44 nodeId: NodeId.make("node_orphaned"),
45 status: "pending",
46 responseCapability: { type: "not_resumable", reason: "old process" },
47 },
48 ],
49 providerSessions: [],
50 providerThreads: [],
51 runs: [],
52 nodes: [],
53 } as unknown as OrchestrationV2ThreadProjection;
54 const layer = ProviderRuntimeRecovery.layer.pipe(
55 Layer.provide(ServerSettings.layerTest()),
56 Layer.provide(
57 Layer.mergeAll(
58 Layer.mock(ProjectionStore.ProjectionStoreV2)({
59 getRecoveryThreadIds: () => Effect.succeed([threadId]),
60 getRuntimeRecoveryProjection: () => Effect.succeed(projection),
61 }),
62 Layer.mock(EventSink.EventSinkV2)({ commitCommand: committed }),
63 IdAllocator.layer,
64 Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({
65 runRecoveryOnce: Effect.succeed(false),
66 }),
67 Layer.mock(EffectOutbox.EffectOutboxV2)({
68 reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }),
69 }),
70 ),
71 ),
72 );
73 return Effect.gen(function* () {
74 yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).recover;
75 const command = committedInput;
76 assert.isNotNull(command);
77 if (command === null) return;
78 assert.equal(command?.events[0]?.type, "runtime-request.updated");
79 if (command?.events[0]?.type === "runtime-request.updated") {
80 assert.equal(command.events[0].payload.status, "expired");
81 assert.equal(command.events[0].payload.responseCapability.type, "not_resumable");
82 }
83 }).pipe(Effect.provide(layer));
84 });
85
86 it.effect("preserves async questions across startup and shutdown", () => {
87 const threadId = ThreadId.make("async-recovery-thread");
88 const nodeId = NodeId.make("async-recovery-node");
89 const projection = {
90 thread: { id: threadId },
91 runtimeRequests: [
92 {
93 id: RuntimeRequestId.make("async-recovery-request"),
94 nodeId,
95 status: "pending",
96 responseCapability: { type: "message" },
97 },
98 ],
99 providerSessions: [],
100 providerThreads: [],
101 runs: [],
102 nodes: [],
103 turnItems: [],
104 } as unknown as OrchestrationV2ThreadProjection;
105 const commitCommand = vi.fn(() => Effect.die("an async question needs no process-loss write"));
106 const layer = ProviderRuntimeRecovery.layer.pipe(
107 Layer.provide(ServerSettings.layerTest()),
108 Layer.provide(
109 Layer.mergeAll(
110 Layer.mock(ProjectionStore.ProjectionStoreV2)({
111 getRecoveryThreadIds: () => Effect.succeed([threadId]),
112 getRuntimeRecoveryProjection: () => Effect.succeed(projection),
113 }),
114 Layer.mock(EventSink.EventSinkV2)({ commitCommand }),
115 IdAllocator.layer,
116 Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({
117 runRecoveryOnce: Effect.succeed(false),
118 }),
119 Layer.mock(EffectOutbox.EffectOutboxV2)({
120 reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }),
121 cancelUnsettled: () => Effect.succeed([]),
122 signalCancellations: () => Effect.void,
123 }),
124 ),
125 ),
126 );
127 return Effect.gen(function* () {
128 const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
129 assert.equal((yield* recovery.reconcile("startup")).closedRequests, 0);
130 assert.equal((yield* recovery.reconcile("startup")).closedRequests, 0);
131 assert.isFalse(commitCommand.mock.calls.length > 0);
132 }).pipe(Effect.provide(layer));
133 });
134
135 it.effect(
136 "preserves a replayable waiting run while cancelling its process-bound background work",
137 () => {
138 const threadId = ThreadId.make("thread_waiting_checkpoint");
139 const runId = RunId.make("run_waiting_checkpoint");
140 const providerThreadId = ProviderThreadId.make("provider_thread_waiting_checkpoint");
141 const providerInstanceId = ProviderInstanceId.make("claude");
142 const backgroundItemId = TurnItemId.make("turn_item_waiting_checkpoint");
143 let committedInput: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] | null =
144 null;
145 const projection = {
146 thread: { id: threadId },
147 runtimeRequests: [],
148 providerSessions: [],
149 providerThreads: [
150 {
151 id: providerThreadId,
152 driver: ProviderDriverKind.make("claude"),
153 providerInstanceId,
154 status: "idle",
155 pendingBackgroundTasks: [
156 { taskId: String(backgroundItemId), description: "Finish background task" },
157 ],
158 },
159 ],
160 runs: [{ id: runId, status: "waiting", providerInstanceId, providerThreadId }],
161 attempts: [],
162 nodes: [],
163 subagents: [],
164 messages: [],
165 turnItems: [
166 {
167 id: backgroundItemId,
168 runId,
169 nodeId: null,
170 providerThreadId,
171 type: "command_execution",
172 status: "running",
173 },
174 ],
175 } as unknown as OrchestrationV2ThreadProjection;
176 const layer = ProviderRuntimeRecovery.layer.pipe(
177 Layer.provide(ServerSettings.layerTest()),
178 Layer.provide(
179 Layer.mergeAll(
180 Layer.mock(ProjectionStore.ProjectionStoreV2)({
181 getRecoveryThreadIds: () => Effect.succeed([threadId]),
182 getRuntimeRecoveryProjection: () => Effect.succeed(projection),
183 }),
184 Layer.mock(EventSink.EventSinkV2)({
185 commitCommand: (input) => {
186 committedInput = input;
187 return Effect.succeed({
188 committed: true,
189 cancelledEffectCount: 0,
190 } as never);
191 },
192 }),
193 IdAllocator.layer,
194 Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({
195 runRecoveryOnce: Effect.succeed(false),
196 }),
197 Layer.mock(EffectOutbox.EffectOutboxV2)({
198 listByCommandId: () =>
199 Effect.succeed([
200 {
201 request: { type: "checkpoint.capture", runId },
202 status: "running",
203 },
204 ] as never),
205 reconcileAfterProcessLoss: Effect.succeed({ requeued: 1, cancelled: 0 }),
206 }),
207 ),
208 ),
209 );
210
211 return Effect.gen(function* () {
212 const summary =
213 yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup");
214 assert.equal(summary.terminalizedRuns, 0);
215 assert.equal(summary.requeuedEffects, 1);
216
217 const command = committedInput;
218 assert.isNotNull(command);
219 if (command === null) return;
220
221 assert.isFalse(command.events.some((event) => event.type === "run.updated"));
222 const recorded = command.events.flatMap((event) =>
223 event.type === "run.background-work-cancelled" ? [event.payload] : [],
224 );
225
226 assert.deepEqual(
227 recorded.map((entry) => ({
228 runId: entry.runId,
229 kinds: entry.restartCancelledBackgroundWork.map((work) => work.kind),
230 })),
231 [{ runId, kinds: ["shell"] }],
232 );
233 assert.isTrue(
234 command.events.some(
235 (event) =>
236 event.type === "turn-item.updated" &&
237 event.payload.id === backgroundItemId &&
238 event.payload.status === "cancelled",
239 ),
240 );
241 assert.isTrue(
242 command.events.some(
243 (event) =>
244 event.type === "provider-thread.updated" &&
245 event.payload.id === providerThreadId &&
246 event.payload.pendingBackgroundTasks?.length === 0,
247 ),
248 );
249 }).pipe(Effect.provide(layer));
250 },
251 );
252
253 it.effect(
254 "records cancelled work on the provider thread that lost it after a provider switch",
255 () => {
256 const threadId = ThreadId.make("thread_switch_note");
257 const claudeRunId = RunId.make("run_switch_note_claude");
258 const codexRunId = RunId.make("run_switch_note_codex");
259 const claudeThreadId = ProviderThreadId.make("provider_thread_switch_note_claude");
260 const codexThreadId = ProviderThreadId.make("provider_thread_switch_note_codex");
261 const claude = ProviderInstanceId.make("claude");
262 const codex = ProviderInstanceId.make("codex");
263 let committedInput: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] | null =
264 null;
265
266 const projection = {
267 thread: { id: threadId },
268 runtimeRequests: [],
269 providerSessions: [],
270 providerThreads: [
271 {
272 id: claudeThreadId,
273 ownerNodeId: null,
274 driver: ProviderDriverKind.make("claudeAgent"),
275 providerInstanceId: claude,
276 status: "idle",
277 pendingBackgroundTasks: [{ taskId: "task-claude", description: "Watch the build" }],
278 },
279 {
280 id: codexThreadId,
281 ownerNodeId: null,
282 driver: ProviderDriverKind.make("codex"),
283 providerInstanceId: codex,
284 status: "idle",
285 },
286 ],
287 providerTurns: [],
288 runs: [
289 {
290 id: claudeRunId,
291 ordinal: 1,
292 status: "completed",
293 providerInstanceId: claude,
294 providerThreadId: claudeThreadId,
295 },
296 {
297 id: codexRunId,
298 ordinal: 2,
299 status: "completed",
300 providerInstanceId: codex,
301 providerThreadId: codexThreadId,
302 },
303 ],
304 attempts: [],
305 nodes: [],
306 subagents: [],
307 messages: [],
308 turnItems: [
309 {
310 id: TurnItemId.make("turn_item_switch_note_subagent"),
311 runId: claudeRunId,
312 nodeId: null,
313 providerThreadId: claudeThreadId,
314 nativeItemRef: null,
315 type: "subagent",
316 title: "Background subagent",
317 status: "running",
318 },
319 ],
320 } as unknown as OrchestrationV2ThreadProjection;
321 const layer = ProviderRuntimeRecovery.layer.pipe(
322 Layer.provide(ServerSettings.layerTest()),
323 Layer.provide(
324 Layer.mergeAll(
325 Layer.mock(ProjectionStore.ProjectionStoreV2)({
326 getRecoveryThreadIds: () => Effect.succeed([threadId]),
327 getRuntimeRecoveryProjection: () => Effect.succeed(projection),
328 }),
329 Layer.mock(EventSink.EventSinkV2)({
330 commitCommand: (input) => {
331 committedInput = input;
332 return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never);
333 },
334 }),
335 IdAllocator.layer,
336 Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({
337 runRecoveryOnce: Effect.succeed(false),
338 }),
339 Layer.mock(EffectOutbox.EffectOutboxV2)({
340 reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }),
341 }),
342 ),
343 ),
344 );
345
346 return Effect.gen(function* () {
347 yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup");
348 const recorded = (committedInput?.events ?? []).flatMap((event) =>
349 event.type === "run.background-work-cancelled" ? [event.payload] : [],
350 );
351
352 assert.deepEqual(
353 recorded.map((entry) => ({
354 runId: entry.runId,
355 kinds: entry.restartCancelledBackgroundWork.map((work) => work.kind),
356 })),
357 [{ runId: claudeRunId, kinds: ["subagent", "task"] }],
358 );
359 }).pipe(Effect.provide(layer));
360 },
361 );
362
363 it.effect("cancels a stale waiting run when no checkpoint capture can finish it", () => {
364 const threadId = ThreadId.make("thread_stale_waiting");
365 const runId = RunId.make("run_stale_waiting");
366 let committedInput: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] | null =
367 null;
368 const projection = {
369 thread: { id: threadId },
370 runtimeRequests: [],
371 providerSessions: [],
372 providerThreads: [],
373 providerTurns: [],
374 runs: [
375 {
376 id: runId,
377 status: "waiting",
378 providerInstanceId: ProviderInstanceId.make("codex"),
379 },
380 ],
381 attempts: [],
382 nodes: [],
383 subagents: [],
384 messages: [],
385 turnItems: [],
386 } as unknown as OrchestrationV2ThreadProjection;
387 const layer = ProviderRuntimeRecovery.layer.pipe(
388 Layer.provide(ServerSettings.layerTest()),
389 Layer.provide(
390 Layer.mergeAll(
391 Layer.mock(ProjectionStore.ProjectionStoreV2)({
392 getRecoveryThreadIds: () => Effect.succeed([threadId]),
393 getRuntimeRecoveryProjection: () => Effect.succeed(projection),
394 }),
395 Layer.mock(EventSink.EventSinkV2)({
396 commitCommand: (input) => {
397 committedInput = input;
398 return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never);
399 },
400 }),
401 IdAllocator.layer,
402 Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({
403 runRecoveryOnce: Effect.succeed(false),
404 }),
405 Layer.mock(EffectOutbox.EffectOutboxV2)({
406 listByCommandId: () => Effect.succeed([]),
407 reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }),
408 }),
409 ),
410 ),
411 );
412
413 return Effect.gen(function* () {
414 const summary =
415 yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup");
416 assert.equal(summary.terminalizedRuns, 1);
417 const runEvent = committedInput?.events.find((event) => event.type === "run.updated");
418 assert.equal(runEvent?.type === "run.updated" ? runEvent.payload.status : null, "cancelled");
419 }).pipe(Effect.provide(layer));
420 });
421
422 it.effect("recovery closes a secret card but leaves a link card and an unread steer open", () => {
423 const threadId = ThreadId.make("thread_secret_recovery");
424 const runId = RunId.make("run_secret_recovery");
425 let committedInput: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] | null =
426 null;
427 const projection = {
428 thread: { id: threadId },
429 runtimeRequests: [],
430 providerSessions: [],
431 providerThreads: [],
432 providerTurns: [
433 { runAttemptId: RunAttemptId.make("attempt_secret_recovery"), status: "running" },
434 ],
435 runs: [
436 {
437 id: runId,
438 status: "running",
439 activeAttemptId: RunAttemptId.make("attempt_secret_recovery"),
440 providerInstanceId: ProviderInstanceId.make("codex"),
441 },
442 ],
443 attempts: [],
444 nodes: [],
445 subagents: [],
446 messages: [],
447 turnItems: [
448 {
449 id: "turn-item:secret-request:recovery",
450 threadId,
451 runId,
452 nodeId: null,
453 type: "secret_request",
454 status: "waiting",
455 secretStatus: "pending",
456 label: "GitHub token",
457 reason: "Used as GH_TOKEN.",
458 },
459 {
460 id: "turn-item:link-request:recovery",
461 threadId,
462 runId,
463 nodeId: null,
464 type: "link_request",
465 status: "waiting",
466 linkStatus: "pending",
467 url: "https://box.example.ts.net",
468 reason: "Run the suite there.",
469 },
470 {
471 id: "turn-item:steer:recovery",
472 threadId,
473 runId,
474 nodeId: null,
475 type: "user_message",
476 status: "pending",
477 inputIntent: "steer",
478 messageId: "message:steer:recovery",
479 },
480 ],
481 } as unknown as OrchestrationV2ThreadProjection;
482 const layer = ProviderRuntimeRecovery.layer.pipe(
483 Layer.provide(ServerSettings.layerTest()),
484 Layer.provide(
485 Layer.mergeAll(
486 Layer.mock(ProjectionStore.ProjectionStoreV2)({
487 getRecoveryThreadIds: () => Effect.succeed([threadId]),
488 getRuntimeRecoveryProjection: () => Effect.succeed(projection),
489 }),
490 Layer.mock(EventSink.EventSinkV2)({
491 commitCommand: (input) => {
492 committedInput = input;
493 return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never);
494 },
495 }),
496 IdAllocator.layer,
497 Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({
498 runRecoveryOnce: Effect.succeed(false),
499 }),
500 Layer.mock(EffectOutbox.EffectOutboxV2)({
501 listByCommandId: () => Effect.succeed([]),
502 reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }),
503 }),
504 ),
505 ),
506 );
507
508 return Effect.gen(function* () {
509 yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup");
510 const cards = (committedInput?.events ?? []).flatMap((event) =>
511 event.type === "turn-item.updated" ? [event.payload] : [],
512 );
513 const secret = cards.find((card) => card.type === "secret_request");
514 assert.equal(secret?.status, "cancelled");
515 assert.equal(secret?.type === "secret_request" ? secret.secretStatus : null, "cancelled");
516 // Nothing in memory waits on a link card; the user can still answer it.
517 // Claude never reported reading the steer, so the orchestrator queues it.
518 assert.deepEqual(
519 cards.filter((card) => card.type === "link_request" || card.type === "user_message"),
520 [],
521 );
522 }).pipe(Effect.provide(layer));
523 });
524
525 it.effect.each([
526 "queued",
527 "preparing",
528 "starting",
529 "starting-other-request",
530 "starting-own-request",
531 "starting-scheduled",
532 "starting-steered",
533 "running",
534 "running-delegated",
535 "running-delegated-capped",
536 "running-delegated-snoozed",
537 "running-delegated-stopped",
538 "running-resume",
539 ] as const)("recovers $0 work after restart", (scenario) => {
540 const status = scenario.startsWith("starting")
541 ? "starting"
542 : scenario.startsWith("running")
543 ? "running"
544 : scenario;
545 const resent = scenario === "starting-scheduled" || scenario === "running-delegated";
546 const terminalized =
547 status === "preparing" ||
548 scenario === "starting-own-request" ||
549 scenario === "starting-steered" ||
550 scenario === "running-delegated-capped" ||
551 scenario === "running-delegated-snoozed" ||
552 scenario === "running-delegated-stopped" ||
553 scenario === "running-resume";
554 const startedAt =
555 status === "running" || scenario === "starting-steered"
556 ? DateTime.makeUnsafe("2026-11-01T19:34:40.403Z")
557 : null;
558 const threadId = ThreadId.make("thread_queued_restart");
559 const runId = RunId.make("run_queued_restart");
560 const attemptId = RunAttemptId.make("attempt_queued_restart");
561 const steeredAttemptId = RunAttemptId.make("attempt_queued_restart_steered");
562 const rootNodeId = NodeId.make("node_queued_restart");
563 let committedInput: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] | null =
564 null;
565 const written: Array<EffectOutbox.PendingOrchestrationEffectV2> = [];
566 const projection = {
567 thread: {
568 id: threadId,
569 archivedAt: null,
570 deletedAt: null,
571 settledOverride: null,
572 snoozedUntil:
573 scenario === "running-delegated-snoozed"
574 ? DateTime.makeUnsafe("2099-07-25T09:00:00.000Z")
575 : null,
576 lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId },
577 forkedFrom: null,
578 ...(scenario === "running-resume"
579 ? {
580 limitRecovery: {
581 runId: RunId.make("run_queued_restart_limited"),
582 resetAt: "2026-11-01T19:34:00.000Z",
583 autoResume: true,
584 },
585 }
586 : {}),
587 },
588 runtimeRequests: scenario.endsWith("request")
589 ? [
590 {
591 id: RuntimeRequestId.make("pending-recovery-request"),
592 nodeId:
593 scenario === "starting-own-request" ? rootNodeId : NodeId.make("other-run-request"),
594 kind: "user_input",
595 status: "pending",
596 responseCapability: { type: "message" },
597 },
598 ]
599 : [],
600 providerSessions: [],
601 providerThreads: [],
602 providerTurns:
603 scenario === "starting-steered"
604 ? [
605 {
606 id: ProviderTurnId.make("provider_turn_queued_restart_steered"),
607 runAttemptId: steeredAttemptId,
608 status: "interrupted",
609 },
610 ]
611 : [],
612 runs: [
613 {
614 id: runId,
615 status,
616 startedAt,
617 rootNodeId,
618 activeAttemptId: attemptId,
619 userMessageId: MessageId.make("message_queued_restart"),
620 queuePosition: 1,
621 providerInstanceId: ProviderInstanceId.make("codex"),
622 ...(scenario === "running-resume"
623 ? { usageLimitContinuationOfRunId: RunId.make("run_queued_restart_limited") }
624 : {}),
625 ...(status === "preparing"
626 ? {
627 workspacePreparation: {
628 type: "worktree",
629 baseRef: "parent-local-commit",
630 startFromOrigin: false,
631 },
632 }
633 : {}),
634 },
635 ],
636 attempts: [
637 ...(scenario === "starting-steered"
638 ? [{ id: steeredAttemptId, runId, rootNodeId, status: "interrupted" }]
639 : []),
640 {
641 id: attemptId,
642 runId,
643 rootNodeId,
644 status: status === "running" ? "running" : "pending",
645 startedAt: status === "running" ? startedAt : null,
646 },
647 ],
648 nodes: [
649 ...(scenario === "starting-other-request"
650 ? [
651 {
652 id: NodeId.make("other-run-request"),
653 runId: RunId.make("earlier-run"),
654 status: "waiting",
655 },
656 ]
657 : []),
658 {
659 id: rootNodeId,
660 runId,
661 status: status === "running" ? "running" : "pending",
662 startedAt: status === "running" ? startedAt : null,
663 },
664 ],
665 subagents: [],
666 messages:
667 status === "starting" || status === "running"
668 ? [
669 {
670 id: MessageId.make("message_queued_restart"),
671 ...(scenario.startsWith("running-delegated")
672 ? { createdBy: "agent", creationSource: "mcp" }
673 : scenario === "running-resume"
674 ? { createdBy: "agent", creationSource: "server" }
675 : { createdBy: "user" }),
676 ...(scenario === "starting-scheduled" ? { scheduledTaskId: "scheduled-task" } : {}),
677 },
678 ]
679 : [],
680 turnItems:
681 scenario === "running-delegated-stopped"
682 ? [
683 {
684 id: TurnItemId.make("turn_item_queued_restart_stop"),
685 type: "run_interrupt_request",
686 runId,
687 nodeId: rootNodeId,
688 status: "completed",
689 },
690 ]
691 : [],
692 } as unknown as OrchestrationV2ThreadProjection;
693 const layer = ProviderRuntimeRecovery.layer.pipe(
694 Layer.provide(ServerSettings.layerTest()),
695 Layer.provide(
696 Layer.mergeAll(
697 Layer.mock(ProjectionStore.ProjectionStoreV2)({
698 getRecoveryThreadIds: () => Effect.succeed([threadId]),
699 // Like the store, recovery state carries only provider turns that have not ended.
700 getRuntimeRecoveryProjection: () =>
701 Effect.succeed({
702 ...projection,
703 providerTurns: projection.providerTurns.filter((turn) => turn.status === "running"),
704 }),
705 getMessagesById: (requestedThreadId, messageIds) =>
706 Effect.succeed(
707 new Map(
708 projection.messages
709 .filter(
710 (message) => requestedThreadId === threadId && messageIds.includes(message.id),
711 )
712 .map((message) => [message.id, message]),
713 ),
714 ),
715 getThreadRecords: () => Effect.succeed(projection),
716 }),
717 Layer.mock(EventSink.EventSinkV2)({
718 commitCommand: (input) => {
719 committedInput = input;
720 return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never);
721 },
722 writeWithEffects: (input) => {
723 written.push(...input.effects);
724 return Effect.succeed([]);
725 },
726 }),
727 IdAllocator.layer,
728 Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({
729 runRecoveryOnce: Effect.succeed(false),
730 }),
731 Layer.mock(EffectOutbox.EffectOutboxV2)({
732 // Three earlier restarts each cut a resend the worker had claimed.
733 listByCommandId: () =>
734 Effect.succeed(
735 scenario === "running-delegated-capped"
736 ? ([1, 1, 1].map((attemptCount) => ({ attemptCount })) as never)
737 : [],
738 ),
739 reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }),
740 }),
741 ),
742 ),
743 );
744
745 return Effect.gen(function* () {
746 const summary =
747 yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup");
748 assert.equal(summary.terminalizedRuns, terminalized ? 1 : 0);
749 const command = committedInput;
750 assert.isNotNull(command);
751 if (command === null) return;
752 const runEvent = command.events.find((event) => event.type === "run.updated");
753 const attemptEvent = command.events.find((event) => event.type === "run-attempt.updated");
754 const nodeEvent = command.events.find((event) => event.type === "node.updated");
755 assert.deepInclude(runEvent?.type === "run.updated" ? runEvent.payload : null, {
756 status: terminalized ? "cancelled" : resent ? "starting" : "queued",
757 startedAt: scenario === "starting-steered" ? startedAt : null,
758 });
759 assert.isEmpty(command.effects);
760 assert.deepEqual(
761 written.map((effect) => effect.request),
762 resent ? [{ type: "provider-turn.start", runId }] : [],
763 );
764 if (terminalized) {
765 assert.equal(
766 attemptEvent?.type === "run-attempt.updated" ? attemptEvent.payload.status : null,
767 "cancelled",
768 );
769 assert.equal(
770 nodeEvent?.type === "node.updated" ? nodeEvent.payload.status : null,
771 scenario === "starting-own-request" ? null : "cancelled",
772 );
773 } else {
774 if (!resent) {
775 assert.equal(runEvent?.type === "run.updated" ? runEvent.payload.queuePosition : null, 1);
776 assert.equal(runEvent?.type === "run.updated" ? runEvent.payload.queueHeld : false, true);
777 }
778 if (status === "running") {
779 assert.deepInclude(
780 attemptEvent?.type === "run-attempt.updated" ? attemptEvent.payload : null,
781 { status: "pending", startedAt: null },
782 );
783 assert.deepInclude(nodeEvent?.type === "node.updated" ? nodeEvent.payload : null, {
784 status: "pending",
785 startedAt: null,
786 });
787 } else {
788 assert.isUndefined(attemptEvent);
789 assert.isUndefined(nodeEvent);
790 }
791 }
792 // The unsent auto-resume gets a fresh request id so the limit worker sends it again.
793 if (scenario === "running-resume") {
794 const rearmed = command.events.find((event) => event.type === "thread.metadata-updated");
795 assert.equal(
796 rearmed?.type === "thread.metadata-updated"
797 ? rearmed.payload.limitRecovery?.requestId
798 : null,
799 command.commandId,
800 );
801 }
802 }).pipe(Effect.provide(layer));
803 });
804
805 it.effect(
806 "cancels the complete in-flight subtree and stops its persisted session without reopening it",
807 () => {
808 const threadId = ThreadId.make("thread_recovery_cancel");
809 const runId = RunId.make("run_recovery_cancel");
810 const attemptId = RunAttemptId.make("attempt_recovery_cancel");
811 const rootNodeId = NodeId.make("node_recovery_cancel");
812 const providerThreadId = ProviderThreadId.make("provider_thread_recovery_cancel");
813 const providerTurnId = ProviderTurnId.make("provider_turn_recovery_cancel");
814 const providerSessionId = ProviderSessionId.make("provider_session_recovery_cancel");
815 let committedInput: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] | null =
816 null;
817 const projection = {
818 thread: { id: threadId },
819 runtimeRequests: [],
820 providerSessions: [
821 {
822 id: providerSessionId,
823 driver: ProviderDriverKind.make("codex"),
824 providerInstanceId: ProviderInstanceId.make("codex"),
825 status: "ready",
826 extensionStatuses: [{ key: "watch", text: "Watching" }],
827 },
828 ],
829 providerThreads: [
830 {
831 id: providerThreadId,
832 driver: ProviderDriverKind.make("codex"),
833 providerInstanceId: ProviderInstanceId.make("codex"),
834 status: "active",
835 },
836 ],
837 providerTurns: [
838 {
839 id: providerTurnId,
840 runAttemptId: attemptId,
841 nodeId: rootNodeId,
842 status: "running",
843 },
844 ],
845 runs: [
846 {
847 id: runId,
848 status: "starting",
849 activeAttemptId: attemptId,
850 providerThreadId,
851 providerInstanceId: ProviderInstanceId.make("codex"),
852 },
853 ],
854 attempts: [
855 {
856 id: attemptId,
857 runId,
858 rootNodeId,
859 status: "running",
860 },
861 ],
862 nodes: [{ id: rootNodeId, runId, status: "running" }],
863 subagents: [],
864 messages: [{ id: MessageId.make("message_recovery_cancel"), runId, streaming: true }],
865 turnItems: [
866 {
867 id: TurnItemId.make("turn_item_recovery_cancel"),
868 runId,
869 nodeId: rootNodeId,
870 status: "running",
871 },
872 ...(["assistant_message", "reasoning", "proposed_plan"] as const).map((type) => ({
873 id: TurnItemId.make(`turn_item_recovery_cancel_${type}`),
874 runId,
875 nodeId: rootNodeId,
876 type,
877 status: "running",
878 streaming: true,
879 })),
880 ],
881 } as unknown as OrchestrationV2ThreadProjection;
882 const layer = ProviderRuntimeRecovery.layer.pipe(
883 Layer.provide(ServerSettings.layerTest()),
884 Layer.provide(
885 Layer.mergeAll(
886 Layer.mock(ProjectionStore.ProjectionStoreV2)({
887 getRecoveryThreadIds: () => Effect.succeed([threadId]),
888 getRuntimeRecoveryProjection: () => Effect.succeed(projection),
889 }),
890 Layer.mock(EventSink.EventSinkV2)({
891 commitCommand: (input) => {
892 committedInput = input;
893 return Effect.succeed({ committed: true, cancelledEffectCount: 2 } as never);
894 },
895 }),
896 IdAllocator.layer,
897 Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({
898 runRecoveryOnce: Effect.succeed(false),
899 }),
900 Layer.mock(EffectOutbox.EffectOutboxV2)({
901 reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }),
902 }),
903 ),
904 ),
905 );
906
907 return Effect.gen(function* () {
908 const summary = yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService)
909 .recover;
910 assert.equal(summary.terminalizedRuns, 1);
911 assert.equal(summary.stoppedSessions, 1);
912 assert.equal(summary.retiredEffects, 2);
913 assert.deepEqual(committedInput?.cancelUnsettledEffects?.effectTypes, [
914 "provider-session.apply-model-selection",
915 "provider-session.apply-thread-title",
916 "provider-turn.start",
917 "provider-turn.interrupt",
918 "provider-turn.steer",
919 "provider-turn.restart",
920 "runtime-request.respond",
921 ]);
922 const events = committedInput?.events ?? [];
923 assert.deepEqual(
924 events.map((event) => [
925 event.type,
926 "status" in event.payload ? event.payload.status : null,
927 ]),
928 [
929 ["run.updated", "cancelled"],
930 ["run-attempt.updated", "cancelled"],
931 ["node.updated", "cancelled"],
932 ["provider-turn.updated", "cancelled"],
933 ["message.updated", null],
934 ["turn-item.updated", "cancelled"],
935 ["turn-item.updated", "cancelled"],
936 ["turn-item.updated", "cancelled"],
937 ["turn-item.updated", "cancelled"],
938 ["turn-item.updated", "completed"],
939 ["provider-thread.updated", "idle"],
940 ["provider-session.updated", "stopped"],
941 ],
942 );
943 const messageEvent = events.find((event) => event.type === "message.updated");
944 assert.isFalse(messageEvent?.type === "message.updated" && messageEvent.payload.streaming);
945 const stoppedSession = events.find((event) => event.type === "provider-session.updated");
946 assert.deepEqual(
947 stoppedSession?.type === "provider-session.updated"
948 ? stoppedSession.payload.extensionStatuses
949 : undefined,
950 [],
951 );
952 assert.deepEqual(
953 events.flatMap((event) =>
954 event.type === "turn-item.updated" && "streaming" in event.payload
955 ? [[event.payload.type, event.payload.streaming]]
956 : [],
957 ),
958 [
959 ["assistant_message", false],
960 ["reasoning", false],
961 ["proposed_plan", false],
962 ],
963 );
964 }).pipe(Effect.provide(layer));
965 },
966 );
967
968 it.effect("cancels workflow members and child roots without run ids after restart", () => {
969 const parentThreadId = ThreadId.make("thread_recovery_workflow_members");
970 const childThreadId = ThreadId.make("thread_recovery_workflow_member_child");
971 const memberId = NodeId.make("node_recovery_workflow_member");
972 const childRootId = NodeId.make("node_recovery_workflow_member_root");
973 const providerInstanceId = ProviderInstanceId.make("claudeAgent");
974 const driver = ProviderDriverKind.make("claudeAgent");
975 const member = {
976 id: memberId,
977 threadId: parentThreadId,
978 runId: null,
979 origin: "provider_native",
980 status: "running",
981 driver,
982 providerInstanceId,
983 };
984 const parentProjection = {
985 thread: { id: parentThreadId, providerInstanceId },
986 runs: [],
987 attempts: [],
988 nodes: [
989 { id: memberId, threadId: parentThreadId, runId: null, kind: "subagent", status: "running" },
990 ],
991 subagents: [member],
992 runtimeRequests: [],
993 providerSessions: [],
994 providerThreads: [],
995 providerTurns: [],
996 messages: [],
997 turnItems: [],
998 } as unknown as OrchestrationV2ThreadProjection;
999 const childProjection = {
1000 ...parentProjection,
1001 thread: { id: childThreadId, providerInstanceId },
1002 nodes: [
1003 {
1004 id: childRootId,
1005 threadId: childThreadId,
1006 runId: null,
1007 kind: "root_turn",
1008 status: "running",
1009 },
1010 ],
1011 subagents: [],
1012 } as unknown as OrchestrationV2ThreadProjection;
1013 const committed: Array<Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0]> = [];
1014 const layer = ProviderRuntimeRecovery.layer.pipe(
1015 Layer.provide(ServerSettings.layerTest()),
1016 Layer.provide(
1017 Layer.mergeAll(
1018 Layer.mock(ProjectionStore.ProjectionStoreV2)({
1019 getRecoveryThreadIds: () => Effect.succeed([parentThreadId, childThreadId]),
1020 getRuntimeRecoveryProjection: (threadId) =>
1021 Effect.succeed(threadId === parentThreadId ? parentProjection : childProjection),
1022 }),
1023 Layer.mock(EventSink.EventSinkV2)({
1024 commitCommand: (input) => {
1025 committed.push(input);
1026 return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never);
1027 },
1028 }),
1029 IdAllocator.layer,
1030 Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({
1031 runRecoveryOnce: Effect.succeed(false),
1032 }),
1033 Layer.mock(EffectOutbox.EffectOutboxV2)({
1034 reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }),
1035 }),
1036 ),
1037 ),
1038 );
1039
1040 return Effect.gen(function* () {
1041 yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup");
1042 assert.deepEqual(
1043 committed.flatMap((command) =>
1044 command.events.map((event) => [
1045 command.threadId,
1046 event.type,
1047 "status" in event.payload ? event.payload.status : null,
1048 ]),
1049 ),
1050 [
1051 [parentThreadId, "subagent.updated", "cancelled"],
1052 [parentThreadId, "node.updated", "cancelled"],
1053 [childThreadId, "node.updated", "cancelled"],
1054 ],
1055 );
1056 }).pipe(Effect.provide(layer));
1057 });
1058
1059 it.effect(
1060 "clears persisted pendingBackgroundTasks and terminalizes stale background items on settled runs",
1061 () => {
1062 const threadId = ThreadId.make("thread_recovery_background");
1063 const settledRunId = RunId.make("run_recovery_background_settled");
1064 const activeRunId = RunId.make("run_recovery_background_active");
1065 const activeAttemptId = RunAttemptId.make("attempt_recovery_background_active");
1066 const activeRootNodeId = NodeId.make("node_recovery_background_active");
1067 const idleProviderThreadId = ProviderThreadId.make("provider_thread_recovery_background_idle");
1068 const activeProviderThreadId = ProviderThreadId.make(
1069 "provider_thread_recovery_background_active",
1070 );
1071 const secondaryProviderThreadId = ProviderThreadId.make(
1072 "provider_thread_recovery_background_secondary",
1073 );
1074 const providerSessionId = ProviderSessionId.make("provider_session_recovery_background");
1075 const settledStaleItemId = TurnItemId.make("turn_item_recovery_background_stale");
1076 const activeRunItemId = TurnItemId.make("turn_item_recovery_background_active");
1077 const nullRunCommandItemId = TurnItemId.make("turn_item_recovery_background_null_run");
1078 const nullRunSubagentItemId = TurnItemId.make("turn_item_recovery_background_null_subagent");
1079 const claudeInstanceId = ProviderInstanceId.make("claude");
1080 const secondaryInstanceId = ProviderInstanceId.make("claude-secondary");
1081 const subagentInstanceId = ProviderInstanceId.make("claude-subagent");
1082 let committedInput: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] | null =
1083 null;
1084 const projection = {
1085 thread: { id: threadId },
1086 runtimeRequests: [],
1087 providerSessions: [
1088 {
1089 id: providerSessionId,
1090 driver: ProviderDriverKind.make("claude"),
1091 providerInstanceId: claudeInstanceId,
1092 status: "ready",
1093 },
1094 ],
1095 providerThreads: [
1096 {
1097 id: idleProviderThreadId,
1098 driver: ProviderDriverKind.make("claude"),
1099
1100 providerInstanceId: claudeInstanceId,
1101 status: "idle",
1102 pendingBackgroundTasks: [{ taskId: "bg-settled", description: "sleep 30" }],
1103 },
1104 {
1105 id: activeProviderThreadId,
1106 driver: ProviderDriverKind.make("claude"),
1107 providerInstanceId: claudeInstanceId,
1108 status: "active",
1109 pendingBackgroundTasks: [{ taskId: "bg-active", description: "npm test" }],
1110 },
1111 {
1112 id: secondaryProviderThreadId,
1113 driver: ProviderDriverKind.make("claude"),
1114 providerInstanceId: secondaryInstanceId,
1115 status: "idle",
1116 pendingBackgroundTasks: [],
1117 },
1118 ],
1119 providerTurns: [{ runAttemptId: activeAttemptId, status: "running" }],
1120 runs: [
1121 {
1122 id: settledRunId,
1123 status: "completed",
1124 providerInstanceId: claudeInstanceId,
1125 },
1126 {
1127 id: activeRunId,
1128 status: "running",
1129 activeAttemptId,
1130 providerInstanceId: claudeInstanceId,
1131 },
1132 ],
1133 attempts: [
1134 {
1135 id: activeAttemptId,
1136 runId: activeRunId,
1137 rootNodeId: activeRootNodeId,
1138 status: "running",
1139 },
1140 ],
1141 nodes: [{ id: activeRootNodeId, runId: activeRunId, status: "running" }],
1142 subagents: [],
1143 messages: [],
1144 turnItems: [
1145 {
1146 id: settledStaleItemId,
1147 runId: settledRunId,
1148 nodeId: null,
1149 providerThreadId: idleProviderThreadId,
1150 type: "command_execution",
1151 status: "running",
1152 },
1153 {
1154 id: activeRunItemId,
1155 runId: activeRunId,
1156 nodeId: activeRootNodeId,
1157 providerThreadId: activeProviderThreadId,
1158 type: "dynamic_tool",
1159 status: "running",
1160 },
1161 {
1162 id: nullRunCommandItemId,
1163 runId: null,
1164 nodeId: null,
1165 providerThreadId: secondaryProviderThreadId,
1166 type: "command_execution",
1167 status: "running",
1168 },
1169 {
1170 id: nullRunSubagentItemId,
1171 runId: null,
1172 nodeId: null,
1173 providerThreadId: secondaryProviderThreadId,
1174 type: "subagent",
1175 status: "running",
1176 providerInstanceId: subagentInstanceId,
1177 },
1178 ],
1179 } as unknown as OrchestrationV2ThreadProjection;
1180 const layer = ProviderRuntimeRecovery.layer.pipe(
1181 Layer.provide(ServerSettings.layerTest()),
1182 Layer.provide(
1183 Layer.mergeAll(
1184 Layer.mock(ProjectionStore.ProjectionStoreV2)({
1185 getRecoveryThreadIds: () => Effect.succeed([threadId]),
1186 getRuntimeRecoveryProjection: () => Effect.succeed(projection),
1187 }),
1188 Layer.mock(EventSink.EventSinkV2)({
1189 commitCommand: (input) => {
1190 committedInput = input;
1191 return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never);
1192 },
1193 }),
1194 IdAllocator.layer,
1195 Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({
1196 runRecoveryOnce: Effect.succeed(false),
1197 }),
1198 Layer.mock(EffectOutbox.EffectOutboxV2)({
1199 listByCommandId: () => Effect.succeed([]),
1200 reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }),
1201 }),
1202 ),
1203 ),
1204 );
1205
1206 return Effect.gen(function* () {
1207 const summary =
1208 yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup");
1209 assert.equal(summary.terminalizedRuns, 1);
1210 const events = committedInput?.events ?? [];
1211
1212 const turnItemCancels = events.filter(
1213 (event) => event.type === "turn-item.updated" && event.payload.status === "cancelled",
1214 );
1215
1216 assert.equal(turnItemCancels.length, 4);
1217 assert.deepEqual(
1218 turnItemCancels
1219 .map((event) => event.type === "turn-item.updated" && event.payload.id)
1220 .sort(),
1221 [activeRunItemId, nullRunCommandItemId, nullRunSubagentItemId, settledStaleItemId].sort(),
1222 );
1223
1224 const cancelById = (id: TurnItemId) =>
1225 turnItemCancels.find(
1226 (event) => event.type === "turn-item.updated" && event.payload.id === id,
1227 );
1228 assert.equal(cancelById(nullRunCommandItemId)?.providerInstanceId, secondaryInstanceId);
1229
1230 assert.notEqual(subagentInstanceId, secondaryInstanceId);
1231 assert.equal(cancelById(nullRunSubagentItemId)?.providerInstanceId, subagentInstanceId);
1232
1233 assert.equal(cancelById(settledStaleItemId)?.providerInstanceId, claudeInstanceId);
1234
1235 const providerThreadEvents = events.filter(
1236 (event) => event.type === "provider-thread.updated",
1237 );
1238
1239 assert.equal(providerThreadEvents.length, 2);
1240 for (const event of providerThreadEvents) {
1241 if (event.type !== "provider-thread.updated") continue;
1242 assert.deepEqual(event.payload.pendingBackgroundTasks ?? [], []);
1243 }
1244 const idleThreadEvent = providerThreadEvents.find(
1245 (event) =>
1246 event.type === "provider-thread.updated" && event.payload.id === idleProviderThreadId,
1247 );
1248 assert.equal(
1249 idleThreadEvent?.type === "provider-thread.updated" ? idleThreadEvent.payload.status : null,
1250 "idle",
1251 );
1252 const activeThreadEvent = providerThreadEvents.find(
1253 (event) =>
1254 event.type === "provider-thread.updated" && event.payload.id === activeProviderThreadId,
1255 );
1256 assert.equal(
1257 activeThreadEvent?.type === "provider-thread.updated"
1258 ? activeThreadEvent.payload.status
1259 : null,
1260 "idle",
1261 );
1262 }).pipe(Effect.provide(layer));
1263 },
1264 );
1265
1266 it.effect(
1267 "terminalizes a leftover nonpersistent dynamic_tool on a settled run after process loss",
1268 () => {
1269 const threadId = ThreadId.make("thread_recovery_orphan_wait");
1270 const settledRunId = RunId.make("run_recovery_orphan_wait_settled");
1271 const providerThreadId = ProviderThreadId.make("provider_thread_recovery_orphan_wait");
1272 const orphanWaitItemId = TurnItemId.make(
1273 "turn-item:provider:codex:native-item:exec-4669f3bb-78c9-4af1-b44e-daa340d2c538",
1274 );
1275 const persistentMonitorItemId = TurnItemId.make(
1276 "turn-item:provider:codex:native-item:exec-persistent-monitor",
1277 );
1278 const orphanWaitNodeId = NodeId.make("node_recovery_orphan_wait");
1279 const persistentMonitorNodeId = NodeId.make("node_recovery_persistent_monitor");
1280 const codexInstanceId = ProviderInstanceId.make("codex");
1281 let committedInput: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] | null =
1282 null;
1283 const projection = {
1284 thread: { id: threadId, providerInstanceId: codexInstanceId },
1285 runtimeRequests: [],
1286 providerSessions: [],
1287 providerThreads: [
1288 {
1289 id: providerThreadId,
1290 driver: ProviderDriverKind.make("codex"),
1291 providerInstanceId: codexInstanceId,
1292 status: "idle",
1293 pendingBackgroundTasks: [],
1294 },
1295 ],
1296 providerTurns: [],
1297 runs: [{ id: settledRunId, status: "completed", providerInstanceId: codexInstanceId }],
1298 attempts: [],
1299 nodes: [
1300 { id: orphanWaitNodeId, runId: settledRunId, status: "running", kind: "tool_call" },
1301 {
1302 id: persistentMonitorNodeId,
1303 runId: settledRunId,
1304 status: "running",
1305 kind: "tool_call",
1306 },
1307 ],
1308 subagents: [],
1309 messages: [],
1310 turnItems: [
1311 {
1312 id: orphanWaitItemId,
1313 runId: settledRunId,
1314 nodeId: orphanWaitNodeId,
1315 providerThreadId,
1316 type: "dynamic_tool",
1317 status: "running",
1318 toolName: "t3-code.t3_thread_wait",
1319 input: {
1320 threadId:
1321 "thread:delegated-task:command%3Amcp%3Aaafffab1-e811-458a-ae83-558e542c61ff%3Adelegate-task%3Areview-mobile-reconnect-opus-20260815",
1322 timeoutMs: 30000,
1323 },
1324 },
1325 {
1326 id: persistentMonitorItemId,
1327 runId: settledRunId,
1328 nodeId: persistentMonitorNodeId,
1329 providerThreadId,
1330 type: "dynamic_tool",
1331 status: "running",
1332 toolName: "grok.monitor",
1333 input: { persistent: true, command: "tail -f" },
1334 },
1335 ],
1336 } as unknown as OrchestrationV2ThreadProjection;
1337 const layer = ProviderRuntimeRecovery.layer.pipe(
1338 Layer.provide(ServerSettings.layerTest()),
1339 Layer.provide(
1340 Layer.mergeAll(
1341 Layer.mock(ProjectionStore.ProjectionStoreV2)({
1342 getRecoveryThreadIds: () => Effect.succeed([threadId]),
1343 getRuntimeRecoveryProjection: () => Effect.succeed(projection),
1344 }),
1345 Layer.mock(EventSink.EventSinkV2)({
1346 commitCommand: (input) => {
1347 committedInput = input;
1348 return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never);
1349 },
1350 }),
1351 IdAllocator.layer,
1352 Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({
1353 runRecoveryOnce: Effect.succeed(false),
1354 }),
1355 Layer.mock(EffectOutbox.EffectOutboxV2)({
1356 listByCommandId: () => Effect.succeed([]),
1357 reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }),
1358 }),
1359 ),
1360 ),
1361 );
1362
1363 return Effect.gen(function* () {
1364 const summary =
1365 yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup");
1366 assert.equal(summary.terminalizedRuns, 0);
1367 const turnItemCancels = (committedInput?.events ?? []).filter(
1368 (event) => event.type === "turn-item.updated" && event.payload.status === "cancelled",
1369 );
1370
1371 assert.equal(turnItemCancels.length, 2);
1372 assert.deepEqual(
1373 turnItemCancels
1374 .map((event) => event.type === "turn-item.updated" && event.payload.id)
1375 .sort(),
1376 [orphanWaitItemId, persistentMonitorItemId].sort(),
1377 );
1378 const nodeCancels = (committedInput?.events ?? []).filter(
1379 (event) => event.type === "node.updated" && event.payload.status === "cancelled",
1380 );
1381 assert.equal(nodeCancels.length, 2);
1382 assert.deepEqual(
1383 nodeCancels.map((event) => event.type === "node.updated" && event.payload.id).sort(),
1384 [orphanWaitNodeId, persistentMonitorNodeId].sort(),
1385 );
1386 }).pipe(Effect.provide(layer));
1387 },
1388 );
1389
1390 it.effect(
1391 "terminalizes the linked subagent and node for a stale subagent item on a settled run",
1392 () => {
1393 const threadId = ThreadId.make("thread_recovery_subagent");
1394 const settledRunId = RunId.make("run_recovery_subagent_settled");
1395 const providerThreadId = ProviderThreadId.make("provider_thread_recovery_subagent");
1396 const staleSubagentNodeId = NodeId.make("node_recovery_subagent_stale");
1397 const doneSubagentNodeId = NodeId.make("node_recovery_subagent_done");
1398 const staleItemId = TurnItemId.make("turn_item_recovery_subagent_stale");
1399 const doneItemId = TurnItemId.make("turn_item_recovery_subagent_done");
1400 const claudeInstanceId = ProviderInstanceId.make("claude");
1401 let committedInput: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] | null =
1402 null;
1403 const projection = {
1404 thread: { id: threadId },
1405 runtimeRequests: [],
1406 providerSessions: [],
1407 providerThreads: [
1408 {
1409 id: providerThreadId,
1410 driver: ProviderDriverKind.make("claude"),
1411 providerInstanceId: claudeInstanceId,
1412 status: "idle",
1413 pendingBackgroundTasks: [],
1414 },
1415 ],
1416 providerTurns: [],
1417
1418 runs: [{ id: settledRunId, status: "completed", providerInstanceId: claudeInstanceId }],
1419 attempts: [],
1420 nodes: [
1421 { id: staleSubagentNodeId, runId: settledRunId, status: "running" },
1422 { id: doneSubagentNodeId, runId: settledRunId, status: "completed" },
1423 ],
1424 subagents: [
1425 {
1426 id: staleSubagentNodeId,
1427 runId: settledRunId,
1428 origin: "provider_native",
1429 driver: ProviderDriverKind.make("claude"),
1430 providerInstanceId: claudeInstanceId,
1431 status: "running",
1432 },
1433 {
1434 id: doneSubagentNodeId,
1435 runId: settledRunId,
1436 origin: "provider_native",
1437 driver: ProviderDriverKind.make("claude"),
1438 providerInstanceId: claudeInstanceId,
1439 status: "completed",
1440 result: "done",
1441 },
1442 ],
1443 messages: [],
1444 turnItems: [
1445 {
1446 id: staleItemId,
1447 runId: settledRunId,
1448 nodeId: staleSubagentNodeId,
1449 providerThreadId,
1450 type: "subagent",
1451 status: "running",
1452 subagentId: staleSubagentNodeId,
1453 providerInstanceId: claudeInstanceId,
1454 },
1455 {
1456 id: doneItemId,
1457 runId: settledRunId,
1458 nodeId: doneSubagentNodeId,
1459 providerThreadId,
1460 type: "subagent",
1461 status: "completed",
1462 subagentId: doneSubagentNodeId,
1463 providerInstanceId: claudeInstanceId,
1464 },
1465 ],
1466 } as unknown as OrchestrationV2ThreadProjection;
1467 const layer = ProviderRuntimeRecovery.layer.pipe(
1468 Layer.provide(ServerSettings.layerTest()),
1469 Layer.provide(
1470 Layer.mergeAll(
1471 Layer.mock(ProjectionStore.ProjectionStoreV2)({
1472 getRecoveryThreadIds: () => Effect.succeed([threadId]),
1473 getRuntimeRecoveryProjection: () => Effect.succeed(projection),
1474 }),
1475 Layer.mock(EventSink.EventSinkV2)({
1476 commitCommand: (input) => {
1477 committedInput = input;
1478 return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never);
1479 },
1480 }),
1481 IdAllocator.layer,
1482 Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({
1483 runRecoveryOnce: Effect.succeed(false),
1484 }),
1485 Layer.mock(EffectOutbox.EffectOutboxV2)({
1486 listByCommandId: () => Effect.succeed([]),
1487 reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }),
1488 }),
1489 ),
1490 ),
1491 );
1492
1493 return Effect.gen(function* () {
1494 yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup");
1495 const events = committedInput?.events ?? [];
1496
1497 const turnItemCancels = events.filter(
1498 (event) => event.type === "turn-item.updated" && event.payload.status === "cancelled",
1499 );
1500 assert.equal(turnItemCancels.length, 1);
1501
1502 const subagentCancels = events.filter((event) => event.type === "subagent.updated");
1503 assert.equal(subagentCancels.length, 1);
1504 const subagentCancel = subagentCancels[0];
1505 assert.equal(
1506 subagentCancel?.type === "subagent.updated" ? subagentCancel.payload.id : null,
1507 staleSubagentNodeId,
1508 );
1509 assert.equal(
1510 subagentCancel?.type === "subagent.updated" ? subagentCancel.payload.status : null,
1511 "cancelled",
1512 );
1513
1514 const nodeCancels = events.filter((event) => event.type === "node.updated");
1515 assert.equal(nodeCancels.length, 1);
1516 const nodeCancel = nodeCancels[0];
1517 assert.equal(
1518 nodeCancel?.type === "node.updated" ? nodeCancel.payload.id : null,
1519 staleSubagentNodeId,
1520 );
1521
1522 assert.isFalse(
1523 events.some(
1524 (event) => event.type === "subagent.updated" && event.payload.id === doneSubagentNodeId,
1525 ),
1526 );
1527 assert.isFalse(
1528 events.some(
1529 (event) => event.type === "node.updated" && event.payload.id === doneSubagentNodeId,
1530 ),
1531 );
1532 }).pipe(Effect.provide(layer));
1533 },
1534 );
1535
1536 it.effect("leaves delegated tasks to their own child threads after process loss", () => {
1537 const threadId = ThreadId.make("thread_recovery_delegation");
1538 const settledRunId = RunId.make("run_recovery_delegation_settled");
1539 const runningRunId = RunId.make("run_recovery_delegation_running");
1540 const providerThreadId = ProviderThreadId.make("provider_thread_recovery_delegation");
1541 const claudeInstanceId = ProviderInstanceId.make("claude");
1542 const driver = ProviderDriverKind.make("claude");
1543 const settledTaskId = NodeId.make("node_recovery_delegation_settled");
1544 const runningTaskId = NodeId.make("node_recovery_delegation_running");
1545 const nativeSubagentId = NodeId.make("node_recovery_native_subagent");
1546 const delegatedTask = (id: NodeId, runId: RunId) => ({
1547 id,
1548 runId,
1549 origin: "app_owned",
1550 childThreadId: ThreadId.make(`thread:delegated-task:${id}`),
1551 driver,
1552 providerInstanceId: claudeInstanceId,
1553 status: "running",
1554 });
1555 const subagentItem = (subagentId: NodeId, runId: RunId, origin: string) => ({
1556 id: TurnItemId.make(`turn_item:${subagentId}`),
1557 runId,
1558 nodeId: subagentId,
1559 providerThreadId,
1560 type: "subagent",
1561 status: "running",
1562 subagentId,
1563 origin,
1564 childThreadId:
1565 origin === "app_owned" ? ThreadId.make(`thread:delegated-task:${subagentId}`) : null,
1566 providerInstanceId: claudeInstanceId,
1567 title: `subagent ${subagentId}`,
1568 });
1569 let committedInput: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] | null =
1570 null;
1571 const projection = {
1572 thread: { id: threadId, providerInstanceId: claudeInstanceId },
1573 runtimeRequests: [],
1574 providerSessions: [],
1575 providerThreads: [
1576 {
1577 id: providerThreadId,
1578 driver,
1579 providerInstanceId: claudeInstanceId,
1580 ownerNodeId: null,
1581 status: "idle",
1582 pendingBackgroundTasks: [],
1583 },
1584 ],
1585 providerTurns: [
1586 { runAttemptId: RunAttemptId.make("attempt_recovery_delegation"), status: "running" },
1587 ],
1588 runs: [
1589 {
1590 id: settledRunId,
1591 ordinal: 1,
1592 status: "completed",
1593 providerThreadId,
1594 providerInstanceId: claudeInstanceId,
1595 },
1596 {
1597 id: runningRunId,
1598 ordinal: 2,
1599 status: "running",
1600 activeAttemptId: RunAttemptId.make("attempt_recovery_delegation"),
1601 providerThreadId,
1602 providerInstanceId: claudeInstanceId,
1603 },
1604 ],
1605 attempts: [],
1606 nodes: [
1607 { id: settledTaskId, runId: settledRunId, status: "running", kind: "subagent" },
1608 { id: runningTaskId, runId: runningRunId, status: "running", kind: "subagent" },
1609 { id: nativeSubagentId, runId: settledRunId, status: "running", kind: "subagent" },
1610 ],
1611 subagents: [
1612 delegatedTask(settledTaskId, settledRunId),
1613 delegatedTask(runningTaskId, runningRunId),
1614 {
1615 ...delegatedTask(nativeSubagentId, settledRunId),
1616 origin: "provider_native",
1617 childThreadId: null,
1618 },
1619 ],
1620 messages: [],
1621 turnItems: [
1622 subagentItem(settledTaskId, settledRunId, "app_owned"),
1623 subagentItem(runningTaskId, runningRunId, "app_owned"),
1624 subagentItem(nativeSubagentId, settledRunId, "provider_native"),
1625 ],
1626 } as unknown as OrchestrationV2ThreadProjection;
1627 const layer = ProviderRuntimeRecovery.layer.pipe(
1628 Layer.provide(ServerSettings.layerTest()),
1629 Layer.provide(
1630 Layer.mergeAll(
1631 Layer.mock(ProjectionStore.ProjectionStoreV2)({
1632 getRecoveryThreadIds: () => Effect.succeed([threadId]),
1633 getRuntimeRecoveryProjection: () => Effect.succeed(projection),
1634 }),
1635 Layer.mock(EventSink.EventSinkV2)({
1636 commitCommand: (input) => {
1637 committedInput = input;
1638 return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never);
1639 },
1640 }),
1641 IdAllocator.layer,
1642 Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({
1643 runRecoveryOnce: Effect.succeed(false),
1644 }),
1645 Layer.mock(EffectOutbox.EffectOutboxV2)({
1646 listByCommandId: () => Effect.succeed([]),
1647 reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }),
1648 }),
1649 ),
1650 ),
1651 );
1652
1653 return Effect.gen(function* () {
1654 yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup");
1655 const events = committedInput?.events ?? [];
1656 const touched = events.flatMap((event) =>
1657 event.type === "turn-item.updated"
1658 ? event.payload.type === "subagent"
1659 ? [event.payload.subagentId]
1660 : []
1661 : event.type === "subagent.updated" || event.type === "node.updated"
1662 ? [event.payload.id]
1663 : [],
1664 );
1665
1666 assert.isTrue(
1667 events.some(
1668 (event) =>
1669 event.type === "run.updated" &&
1670 event.payload.id === runningRunId &&
1671 event.payload.status === "cancelled",
1672 ),
1673 );
1674 assert.notInclude(touched, settledTaskId);
1675 assert.notInclude(touched, runningTaskId);
1676 assert.include(touched, nativeSubagentId);
1677
1678 const noted = events.flatMap((event) =>
1679 event.type === "run.background-work-cancelled"
1680 ? event.payload.restartCancelledBackgroundWork.map((work) => work.label)
1681 : [],
1682 );
1683 assert.deepEqual(noted, [`subagent ${nativeSubagentId}`]);
1684 }).pipe(Effect.provide(layer));
1685 });