The best way to control Claude Code, Codex and more from any computer or phone

main
ts 1,481 lines 53.1 KB
Raw
1 import { assert, it } from "@effect/vitest";
2 import {
3 MessageId,
4 NodeId,
5 ProjectId,
6 ProviderDriverKind,
7 ProviderInstanceId,
8 ProviderSessionId,
9 ProviderThreadId,
10 ProviderTurnId,
11 RunAttemptId,
12 RunId,
13 RuntimeRequestId,
14 ThreadId,
15 type OrchestrationV2Command,
16 type OrchestrationV2ThreadProjection,
17 } from "@t3tools/contracts";
18 import * as DateTime from "effect/DateTime";
19 import * as Effect from "effect/Effect";
20 import * as Deferred from "effect/Deferred";
21 import * as Fiber from "effect/Fiber";
22 import * as Layer from "effect/Layer";
23 import * as Option from "effect/Option";
24 import * as ServerSettings from "../serverSettings.ts";
25 import { restartContinuationRun, continueRestartedRun } from "./RestartContinuation.ts";
26 import { continuationStartRetryDelayMs } from "./RestartBackgroundNote.ts";
27 import * as ThreadManagementService from "./ThreadManagementService.ts";
28 import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts";
29 import * as ProjectionStore from "./ProjectionStore.ts";
30 import * as EventSink from "./EventSink.ts";
31 import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
32 import * as EffectWorker from "./EffectWorker.ts";
33 import * as EffectOutbox from "./EffectOutbox.ts";
34
35 const threadId = ThreadId.make("thread:restart");
36 const runId = RunId.make("run:restart");
37 const instanceId = ProviderInstanceId.make("codex");
38 const driver = ProviderDriverKind.make("codex");
39 const providerThreadId = ProviderThreadId.make("provider-thread:restart");
40 const sessionId = ProviderSessionId.make("session:restart");
41 const attemptId = RunAttemptId.make("attempt:restart");
42 const scratchProjectId = ProjectId.make("project:scratch");
43 const now = DateTime.makeUnsafe("2026-10-03T12:00:00.000Z");
44
45 function makeProjection() {
46 return {
47 thread: {
48 id: threadId,
49 projectId: ProjectId.make("restart-project"),
50 providerInstanceId: instanceId,
51 archivedAt: null,
52 deletedAt: null,
53 lineage: { parentThreadId: null, relationshipToParent: null },
54 forkedFrom: null,
55 },
56 runs: [
57 {
58 id: runId,
59 ordinal: 1,
60 providerInstanceId: instanceId,
61 modelSelection: { instanceId, model: "gpt-6" },
62 providerThreadId,
63 activeAttemptId: attemptId,
64 status: "running",
65 requestedAt: now,
66 completedAt: null,
67 },
68 ],
69 providerThreads: [
70 {
71 id: providerThreadId,
72 appThreadId: threadId,
73 ownerNodeId: null,
74 driver,
75 providerInstanceId: instanceId,
76 providerSessionId: sessionId,
77 nativeThreadRef: { driver, nativeId: "native-thread", strength: "strong" },
78 status: "active",
79 },
80 ],
81 providerSessions: [
82 { id: sessionId, driver, providerInstanceId: instanceId, status: "running" },
83 ],
84 providerTurns: [
85 {
86 id: ProviderTurnId.make("turn:restart"),
87 providerThreadId,
88 runAttemptId: attemptId,
89 status: "running",
90 },
91 ],
92 runtimeRequests: [],
93 attempts: [],
94 nodes: [],
95 subagents: [],
96 messages: [],
97 turnItems: [],
98 } as unknown as OrchestrationV2ThreadProjection;
99 }
100
101 it("requires matching saved native state for an unfinished root run", () => {
102 const projection = makeProjection();
103 assert.equal(restartContinuationRun(projection, now)?.id, runId);
104 for (const invalid of [
105 { ...projection, thread: { ...projection.thread, archivedAt: {} } },
106 { ...projection, thread: { ...projection.thread, deletedAt: {} } },
107 {
108 ...projection,
109 thread: { ...projection.thread, providerInstanceId: ProviderInstanceId.make("other") },
110 },
111 {
112 ...projection,
113 providerThreads: [{ ...projection.providerThreads[0]!, nativeThreadRef: null }],
114 },
115 {
116 ...projection,
117 providerSessions: [
118 {
119 ...projection.providerSessions[0]!,
120 providerInstanceId: ProviderInstanceId.make("other"),
121 },
122 ],
123 },
124 { ...projection, providerTurns: [] },
125 ...[
126 "queued",
127 "preparing",
128 "starting",
129 "waiting",
130 "completed",
131 "cancelled",
132 "failed",
133 "interrupted",
134 ].map((status) => ({ ...projection, runs: [{ ...projection.runs[0]!, status }] })),
135 ])
136 assert.isUndefined(restartContinuationRun(invalid as OrchestrationV2ThreadProjection, now));
137 });
138
139 it("continues a live turn whose session the adapter never marked running", () => {
140 const projection = makeProjection();
141 const withSessionStatus = (status: string) =>
142 ({
143 ...projection,
144 providerSessions: [{ ...projection.providerSessions[0]!, status }],
145 }) as OrchestrationV2ThreadProjection;
146 for (const status of ["starting", "ready", "running", "waiting"])
147 assert.equal(restartContinuationRun(withSessionStatus(status), now)?.id, runId, status);
148 for (const status of ["stopped", "error"])
149 assert.isUndefined(restartContinuationRun(withSessionStatus(status), now), status);
150 });
151
152 it.effect("recovers an admitted continuation after another crash before provider start", () =>
153 Effect.gen(function* () {
154 const projection = makeProjection();
155 const starting = {
156 ...projection,
157 runs: [
158 {
159 ...projection.runs[0]!,
160 status: "starting" as const,
161 restartContinuationOfRunId: RunId.make("run:previous-crash"),
162 },
163 ],
164 providerThreads: [{ ...projection.providerThreads[0]!, status: "idle" as const }],
165 providerSessions: [{ ...projection.providerSessions[0]!, status: "stopped" as const }],
166 providerTurns: [],
167 };
168 assert.equal(restartContinuationRun(starting, now)?.id, runId);
169
170 // Marked running, but the provider never accepted its prompt.
171 const running = {
172 ...projection,
173 runs: [
174 { ...projection.runs[0]!, restartContinuationOfRunId: RunId.make("run:previous-crash") },
175 ],
176 providerTurns: [],
177 } as unknown as OrchestrationV2ThreadProjection;
178 const commits: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0][] = [];
179 const recovery = yield* ProviderRuntimeRecovery.make.pipe(
180 Effect.provide(
181 Layer.mergeAll(
182 ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
183 Layer.mock(ProjectionStore.ProjectionStoreV2)({
184 getRecoveryThreadIds: () => Effect.succeed([threadId]),
185 getRuntimeRecoveryProjection: () => Effect.succeed(running),
186 getThreadRecords: () => Effect.succeed(running),
187 }),
188 Layer.mock(EventSink.EventSinkV2)({
189 commitCommand: (input) =>
190 Effect.sync(() => {
191 commits.push(input);
192 return { committed: true, cancelledEffectCount: 0 } as never;
193 }),
194 }),
195 IdAllocator.layer,
196 Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({}),
197 Layer.mock(EffectOutbox.EffectOutboxV2)({
198 get: () => Effect.succeedNone,
199 reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }),
200 }),
201 ),
202 ),
203 );
204 yield* recovery.reconcile("startup");
205 assert.deepEqual(
206 commits.flatMap((commit) => commit.effects.map((effect) => effect.request)),
207 [{ type: "provider-runtime.continue", sourceRunId: runId, processLossCount: 1 }],
208 );
209 }),
210 );
211
212 it("does not continue settled root runs with restart-cancelled background work", () => {
213 const projection = makeProjection();
214 const settled = {
215 ...projection,
216 runs: [
217 {
218 ...projection.runs[0]!,
219 status: "completed" as const,
220 restartCancelledBackgroundWork: [{ kind: "shell" as const, label: "sleep 25" }],
221 },
222 ],
223 providerThreads: [{ ...projection.providerThreads[0]!, status: "idle" as const }],
224 providerSessions: [{ ...projection.providerSessions[0]!, status: "stopped" as const }],
225 providerTurns: [],
226 };
227 for (const projectId of [scratchProjectId, projection.thread.projectId])
228 for (const status of ["completed", "waiting"] as const)
229 assert.isUndefined(
230 restartContinuationRun(
231 {
232 ...settled,
233 thread: { ...settled.thread, projectId },
234 runs: [{ ...settled.runs[0]!, status }],
235 },
236 now,
237 ),
238 );
239 });
240
241 it.effect.each(["completed", "waiting", "cancelled"] as const)(
242 "wakes a %s run whose awaited monitor the restart cancelled, with the note as its prompt",
243 (status) =>
244 Effect.gen(function* () {
245 const base = makeProjection();
246 const work = [
247 { kind: "monitor" as const, label: "Watch CI run 42", awaited: true as const },
248 { kind: "shell" as const, label: "vp run dev" },
249 ];
250 const projection = {
251 ...base,
252 runs: [{ ...base.runs[0]!, status, restartCancelledBackgroundWork: work }],
253 providerTurns: [{ ...base.providerTurns[0]!, status: "completed" }],
254 } as unknown as OrchestrationV2ThreadProjection;
255 const commands: Parameters<
256 ThreadManagementService.ThreadManagementService["Service"]["dispatch"]
257 >[0][] = [];
258 yield* continueRestartedRun({ threadId, sourceRunId: runId }).pipe(
259 Effect.provide(
260 Layer.merge(
261 Layer.mock(ThreadManagementService.ThreadManagementService)({
262 getThreadRecords: () => Effect.succeed(projection),
263 recoverDelegatedTask: () => Effect.void,
264 dispatch: (command) => {
265 commands.push(command);
266 return Effect.succeed({} as never);
267 },
268 }),
269 ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
270 ),
271 ),
272 );
273 assert.lengthOf(commands, 1);
274 const command = commands[0]!;
275 assert.equal(command.type, "message.dispatch");
276 if (command.type !== "message.dispatch") return;
277 assert.equal(command.restartContinuationOfRunId, runId);
278 assert.include(command.text, "- monitor: Watch CI run 42");
279 assert.include(command.text, "- shell: vp run dev");
280 }),
281 );
282
283 it.effect.each([
284 [false, undefined],
285 [true, undefined],
286 [false, true],
287 [true, false],
288 ] as const)(
289 "atomically records restart intent with cancellation when opt-in is %s and project override is %s",
290 ([enabled, projectOverride]) =>
291 Effect.gen(function* () {
292 let committed: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] | undefined;
293 const recovery = yield* ProviderRuntimeRecovery.make.pipe(
294 Effect.provide(
295 Layer.mergeAll(
296 ServerSettings.layerTest({
297 continueThreadsAfterServerUpdate: enabled,
298 projectSettingsOverrides:
299 projectOverride === undefined
300 ? {}
301 : {
302 [ProjectId.make("restart-project")]: {
303 continueThreadsAfterServerUpdate: projectOverride,
304 },
305 },
306 }),
307 Layer.mock(ProjectionStore.ProjectionStoreV2)({
308 getRecoveryThreadIds: () => Effect.succeed([threadId]),
309 getRuntimeRecoveryProjection: () => Effect.succeed(makeProjection()),
310 }),
311 Layer.mock(EventSink.EventSinkV2)({
312 commitCommand: (input) => {
313 committed = input;
314 return Effect.succeed({ committed: true, cancelledEffectCount: 1 } as never);
315 },
316 }),
317 IdAllocator.layer,
318 Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({
319 runRecoveryOnce: Effect.succeed(false),
320 }),
321 Layer.mock(EffectOutbox.EffectOutboxV2)({
322 reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }),
323 }),
324 ),
325 ),
326 );
327 yield* recovery.reconcile("startup");
328 assert.isDefined(committed);
329 assert.isTrue(
330 committed!.events.some(
331 (event) => event.type === "run.updated" && event.payload.status === "cancelled",
332 ),
333 );
334 assert.lengthOf(committed!.effects, (projectOverride ?? enabled) ? 1 : 0);
335 if (projectOverride ?? enabled)
336 assert.deepEqual(committed!.effects[0]?.request, {
337 type: "provider-runtime.continue",
338 sourceRunId: runId,
339 processLossCount: 1,
340 });
341 }),
342 );
343
344 it.effect("lets a cut blocking delegate_task wake its parent like an async task", () =>
345 Effect.gen(function* () {
346 const base = makeProjection();
347 const task = (id: string, completionWake: "settled_only" | "always", status: string) => ({
348 id: NodeId.make(id),
349 threadId,
350 runId,
351 origin: "app_owned",
352 childThreadId: ThreadId.make(`thread:${id}`),
353 driver,
354 providerInstanceId: instanceId,
355 completionWake,
356 status,
357 });
358 const projection = {
359 ...base,
360 subagents: [
361 task("task:waited", "settled_only", "running"),
362 task("task:async", "always", "running"),
363 task("task:finished", "settled_only", "completed"),
364 ],
365 } as unknown as OrchestrationV2ThreadProjection;
366 let committed: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] | undefined;
367 const recovery = yield* ProviderRuntimeRecovery.make.pipe(
368 Effect.provide(
369 Layer.mergeAll(
370 ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
371 Layer.mock(ProjectionStore.ProjectionStoreV2)({
372 getRecoveryThreadIds: () => Effect.succeed([threadId]),
373 getRuntimeRecoveryProjection: () => Effect.succeed(projection),
374 }),
375 Layer.mock(EventSink.EventSinkV2)({
376 commitCommand: (input) => {
377 committed = input;
378 return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never);
379 },
380 }),
381 IdAllocator.layer,
382 Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({}),
383 Layer.mock(EffectOutbox.EffectOutboxV2)({
384 reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }),
385 }),
386 ),
387 ),
388 );
389 yield* recovery.reconcile("startup");
390 const upgraded = (committed?.events ?? []).flatMap((event) =>
391 event.type === "subagent.updated" ? [event.payload] : [],
392 );
393 assert.deepEqual(
394 upgraded.map((row) => ({ id: row.id, completionWake: row.completionWake })),
395 [{ id: NodeId.make("task:waited"), completionWake: "always" }],
396 );
397 }),
398 );
399
400 it.effect.each(["running", "waiting"])(
401 "wakes a %s run to restart the native subagent it waited on",
402 (status) =>
403 Effect.gen(function* () {
404 const base = makeProjection();
405 const item = (id: string, extra: Record<string, unknown>) => ({
406 id,
407 threadId,
408 runId,
409 nodeId: null,
410 status: "running",
411 ...extra,
412 });
413 const projection = {
414 ...base,
415 runs: [{ ...base.runs[0]!, status }],
416 providerTurns: [
417 { ...base.providerTurns[0]!, status: status === "waiting" ? "completed" : "running" },
418 ],
419 turnItems: [
420 item("item:native", {
421 type: "subagent",
422 origin: "provider_native",
423 providerThreadId: ProviderThreadId.make("provider-thread:child"),
424 nativeItemRef: { driver, nativeId: "agent-1" },
425 title: "Sleeper",
426 childThreadId: ThreadId.make("thread:child"),
427 }),
428 item("item:shell", { type: "command_execution", providerThreadId, input: "ls" }),
429 ],
430 } as unknown as OrchestrationV2ThreadProjection;
431 let committed: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] | undefined;
432 const recovery = yield* ProviderRuntimeRecovery.make.pipe(
433 Effect.provide(
434 Layer.mergeAll(
435 ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
436 Layer.mock(ProjectionStore.ProjectionStoreV2)({
437 getRecoveryThreadIds: () => Effect.succeed([threadId]),
438 getRuntimeRecoveryProjection: () => Effect.succeed(projection),
439 }),
440 Layer.mock(EventSink.EventSinkV2)({
441 commitCommand: (input) => {
442 committed = input;
443 return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never);
444 },
445 }),
446 IdAllocator.layer,
447 Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({}),
448 Layer.mock(EffectOutbox.EffectOutboxV2)({
449 listByCommandId: () => Effect.succeed([]),
450 reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }),
451 }),
452 ),
453 ),
454 );
455 yield* recovery.reconcile("startup");
456 const work = (committed?.events ?? []).flatMap((event) =>
457 event.type === "run.background-work-cancelled" ? [event.payload] : [],
458 );
459 assert.deepEqual(work, [
460 {
461 runId,
462 restartCancelledBackgroundWork: [
463 { kind: "subagent", label: "Sleeper (id agent-1)", id: "item:native" },
464 ...(status === "waiting"
465 ? [{ kind: "shell" as const, label: "ls", id: "item:shell" }]
466 : []),
467 ],
468 },
469 ]);
470 assert.deepEqual(
471 committed?.effects.map((effect) => effect.request),
472 [{ type: "provider-runtime.continue", sourceRunId: runId, processLossCount: 1 }],
473 );
474 }),
475 );
476
477 it.effect("does not duplicate delivery and yields to newer user work or opt-out", () =>
478 Effect.gen(function* () {
479 let projection = makeProjection();
480 projection = { ...projection, runs: [{ ...projection.runs[0]!, status: "cancelled" }] };
481 const commands: Parameters<
482 ThreadManagementService.ThreadManagementService["Service"]["dispatch"]
483 >[0][] = [];
484 const layerThreads = Layer.mock(ThreadManagementService.ThreadManagementService)({
485 getThreadRecords: () => Effect.succeed(projection),
486 recoverDelegatedTask: () => Effect.void,
487 dispatch: (command) => {
488 commands.push(command);
489 if (command.type === "message.dispatch")
490 projection = { ...projection, messages: [{ id: command.messageId } as never] };
491 return Effect.succeed({} as never);
492 },
493 });
494 const layerEnabled = Layer.merge(
495 layerThreads,
496 ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
497 );
498 yield* continueRestartedRun({ threadId, sourceRunId: runId }).pipe(
499 Effect.provide(layerEnabled),
500 );
501 yield* continueRestartedRun({ threadId, sourceRunId: runId }).pipe(
502 Effect.provide(layerEnabled),
503 );
504 assert.lengthOf(commands, 1);
505 assert.match(String(commands[0]!.commandId), /run:restart$/);
506 if (commands[0]!.type === "message.dispatch")
507 assert.equal(commands[0]!.restartContinuationOfRunId, runId);
508 projection = {
509 ...projection,
510 messages: [],
511 runs: [
512 ...projection.runs,
513 {
514 ...projection.runs[0]!,
515 id: RunId.make("run:user-newer"),
516 ordinal: 2,
517 status: "completed",
518 },
519 ],
520 };
521 yield* continueRestartedRun({ threadId, sourceRunId: runId }).pipe(
522 Effect.provide(layerEnabled),
523 );
524 assert.lengthOf(commands, 1);
525 projection = { ...projection, runs: [projection.runs[0]!] };
526 yield* continueRestartedRun({ threadId, sourceRunId: runId }).pipe(
527 Effect.provide(
528 Layer.merge(
529 layerThreads,
530 ServerSettings.layerTest({ continueThreadsAfterServerUpdate: false }),
531 ),
532 ),
533 );
534 assert.lengthOf(commands, 1);
535 }),
536 );
537
538 it.effect("prepares no continuation for background work another provider thread launched", () =>
539 Effect.gen(function* () {
540 const base = makeProjection();
541 const claudeThreadId = ProviderThreadId.make("provider-thread:claude");
542 const projection = {
543 ...base,
544 runs: [{ ...base.runs[0]!, status: "completed" }],
545 providerTurns: [{ ...base.providerTurns[0]!, status: "completed" }],
546 turnItems: [
547 {
548 id: "turn-item:claude-subagent",
549 runId: RunId.make("run:claude"),
550 providerThreadId: claudeThreadId,
551 type: "subagent",
552 status: "running",
553 },
554 ],
555 } as unknown as OrchestrationV2ThreadProjection;
556 const writes: Parameters<EventSink.EventSinkV2["Service"]["writeWithEffects"]>[0][] = [];
557 const recovery = yield* ProviderRuntimeRecovery.make.pipe(
558 Effect.provide(
559 Layer.mergeAll(
560 ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
561 Layer.mock(ProjectionStore.ProjectionStoreV2)({
562 getRecoveryThreadIds: () => Effect.succeed([threadId]),
563 getRuntimeRecoveryProjection: () => Effect.succeed(projection),
564 }),
565 Layer.mock(EventSink.EventSinkV2)({
566 writeWithEffects: (input) =>
567 Effect.sync(() => {
568 writes.push(input);
569 return [];
570 }),
571 }),
572 IdAllocator.layer,
573 Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({}),
574 Layer.mock(EffectOutbox.EffectOutboxV2)({}),
575 ),
576 ),
577 );
578 yield* recovery.prepareForShutdown;
579 assert.lengthOf(writes, 0);
580 const ownWork = {
581 ...projection,
582 turnItems: [{ ...projection.turnItems[0]!, providerThreadId }],
583 } as OrchestrationV2ThreadProjection;
584 const ownRecovery = yield* ProviderRuntimeRecovery.make.pipe(
585 Effect.provide(
586 Layer.mergeAll(
587 ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
588 Layer.mock(ProjectionStore.ProjectionStoreV2)({
589 getRecoveryThreadIds: () => Effect.succeed([threadId]),
590 getRuntimeRecoveryProjection: () => Effect.succeed(ownWork),
591 }),
592 Layer.mock(EventSink.EventSinkV2)({
593 writeWithEffects: (input) =>
594 Effect.sync(() => {
595 writes.push(input);
596 return [];
597 }),
598 }),
599 IdAllocator.layer,
600 Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({}),
601 Layer.mock(EffectOutbox.EffectOutboxV2)({}),
602 ),
603 ),
604 );
605 yield* ownRecovery.prepareForShutdown;
606 assert.lengthOf(writes, 1);
607 assert.deepEqual(writes[0]!.effects[0]?.request, {
608 type: "provider-runtime.continue",
609 sourceRunId: runId,
610 processLossCount: 0,
611 });
612 }),
613 );
614
615 const backgroundSubagent = {
616 turnItem: {
617 id: "turn-item:background-subagent",
618 runId,
619 nodeId: null,
620 providerThreadId,
621 providerInstanceId: instanceId,
622 type: "subagent",
623 subagentId: "subagent:background",
624 title: "Background reviewer",
625 status: "running",
626 },
627 work: {
628 kind: "subagent" as const,
629 label: "Background reviewer",
630 id: "turn-item:background-subagent",
631 },
632 };
633 const devServer = {
634 turnItem: {
635 id: "turn-item:dev-server",
636 runId,
637 nodeId: null,
638 providerThreadId,
639 providerInstanceId: instanceId,
640 type: "command_execution",
641 input: "vp run dev",
642 status: "running",
643 },
644 work: { kind: "shell" as const, label: "vp run dev", id: "turn-item:dev-server" },
645 };
646
647 const unacceptedFollowUp = {
648 id: RunId.make("run:follow-up"),
649 ordinal: 2,
650 activeAttemptId: RunAttemptId.make("attempt:follow-up"),
651 userMessageId: MessageId.make("message:follow-up"),
652 status: "running",
653 startedAt: now,
654 };
655
656 it.effect.each([
657 ["completed", scratchProjectId, backgroundSubagent, true, ""],
658 ["waiting", ProjectId.make("restart-project"), backgroundSubagent, true, ""],
659 ["completed", ProjectId.make("restart-project"), devServer, false, ""],
660 ["waiting", scratchProjectId, devServer, false, ""],
661 [
662 "completed",
663 ProjectId.make("restart-project"),
664 backgroundSubagent,
665 true,
666 ", past a later prompt the provider never accepted",
667 ],
668 ] as const)(
669 "cleans up background work of a $0 run in project $1 and wakes it only for awaited work$4",
670 ([status, projectId, background, wakes, followUp]) =>
671 Effect.gen(function* () {
672 const base = makeProjection();
673 const projection = {
674 ...base,
675 thread: { ...base.thread, projectId },
676 runs: [
677 { ...base.runs[0]!, status },
678 ...(followUp === "" ? [] : [{ ...base.runs[0]!, ...unacceptedFollowUp }]),
679 ],
680 providerTurns: [{ ...base.providerTurns[0]!, status: "completed" }],
681 turnItems: [background.turnItem],
682 subagents:
683 background === backgroundSubagent
684 ? [
685 {
686 id: "subagent:background",
687 runId,
688 origin: "provider_native",
689 driver,
690 providerInstanceId: instanceId,
691 status: "running",
692 },
693 ]
694 : [],
695 } as unknown as OrchestrationV2ThreadProjection;
696 for (const trigger of ["startup", "shutdown"] as const) {
697 const commits: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0][] = [];
698 const writes: Parameters<EventSink.EventSinkV2["Service"]["writeWithEffects"]>[0][] = [];
699 const recovery = yield* ProviderRuntimeRecovery.make.pipe(
700 Effect.provide(
701 Layer.mergeAll(
702 ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
703 Layer.mock(ProjectionStore.ProjectionStoreV2)({
704 getRecoveryThreadIds: () => Effect.succeed([threadId]),
705 getRuntimeRecoveryProjection: () => Effect.succeed(projection),
706 getThreadRecords: () => Effect.succeed(projection),
707 getMessagesById: () =>
708 Effect.succeed(
709 new Map([
710 [
711 unacceptedFollowUp.userMessageId,
712 { id: unacceptedFollowUp.userMessageId, createdBy: "user" } as never,
713 ],
714 ]),
715 ),
716 }),
717 Layer.mock(EventSink.EventSinkV2)({
718 writeWithEffects: (input) =>
719 Effect.sync(() => {
720 writes.push(input);
721 return [];
722 }),
723 commitCommand: (input) =>
724 Effect.sync(() => {
725 commits.push(input);
726 return { committed: true, cancelledEffectCount: 0 } as never;
727 }),
728 }),
729 IdAllocator.layer,
730 Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({
731 runRecoveryOnce: Effect.succeed(false),
732 }),
733 Layer.mock(EffectOutbox.EffectOutboxV2)({
734 listByCommandId: () => Effect.succeed([]),
735 reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }),
736 }),
737 ),
738 ),
739 );
740 if (trigger === "shutdown") yield* recovery.prepareForShutdown;
741 yield* recovery.reconcile(trigger);
742 const recorded = [...writes.flatMap((write) => write.effects), ...commits[0]!.effects];
743 assert.deepEqual(
744 recorded.map((effect) => effect.request),
745 wakes
746 ? [
747 {
748 type: "provider-runtime.continue",
749 sourceRunId: runId,
750 processLossCount: trigger === "shutdown" ? 0 : 1,
751 },
752 ]
753 : [],
754 );
755 assert.lengthOf(commits, 1);
756 const events = commits[0]!.events;
757 assert.isTrue(
758 events.some(
759 (event) => event.type === "turn-item.updated" && event.payload.status === "cancelled",
760 ),
761 );
762 const note = events.find((event) => event.type === "run.background-work-cancelled");
763 assert.equal(note?.runId, runId);
764 assert.deepEqual(note?.payload.restartCancelledBackgroundWork, [background.work]);
765 if (status === "completed")
766 assert.isFalse(
767 events.some((event) => event.type === "run.updated" && event.runId === runId),
768 );
769 }
770 }),
771 );
772
773 it("does not wake a settled run for work left by a session that already stopped", () => {
774 const base = makeProjection();
775 const projection = {
776 ...base,
777 runs: [{ ...base.runs[0]!, status: "completed" }],
778 providerThreads: [{ ...base.providerThreads[0]!, status: "idle" }],
779 turnItems: [backgroundSubagent.turnItem],
780 } as unknown as OrchestrationV2ThreadProjection;
781 assert.equal(restartContinuationRun(projection, now)?.id, runId);
782 for (const status of ["stopped", "error"])
783 assert.isUndefined(
784 restartContinuationRun(
785 {
786 ...projection,
787 providerSessions: [{ ...projection.providerSessions[0]!, status }],
788 } as OrchestrationV2ThreadProjection,
789 now,
790 ),
791 status,
792 );
793 });
794
795 it.effect("wakes a run whose subagent's own shell the restart cut, past a wake it cut first", () =>
796 Effect.gen(function* () {
797 const base = makeProjection();
798 const cutWake = {
799 ...base.runs[0]!,
800 id: RunId.make("run:wake"),
801 ordinal: 2,
802 status: "starting",
803 startedAt: null,
804 };
805 const withShell = (parentTaskId?: string) =>
806 ({
807 ...base,
808 runs: [{ ...base.runs[0]!, status: "waiting", startedAt: now }, cutWake],
809 providerThreads: [
810 {
811 ...base.providerThreads[0]!,
812 status: "idle",
813 pendingBackgroundTasks: [
814 {
815 kind: "command",
816 taskId: "shell:1",
817 description: "sleep 150",
818 ...(parentTaskId === undefined ? {} : { parentTaskId }),
819 },
820 ],
821 },
822 ],
823 providerTurns: [{ ...base.providerTurns[0]!, status: "completed" }],
824 }) as unknown as OrchestrationV2ThreadProjection;
825 assert.equal(restartContinuationRun(withShell("agent:1"), now)?.id, runId);
826 assert.isUndefined(restartContinuationRun(withShell(), now));
827
828 const texts = yield* continuationTexts({
829 ...base,
830 runs: [
831 {
832 ...base.runs[0]!,
833 status: "completed",
834 startedAt: now,
835 completedAt: now,
836 restartCancelledBackgroundWork: [
837 { kind: "subagent", label: "sleep 150 (subagent agent:1)", id: "agent:1" },
838 ],
839 },
840 { ...cutWake, status: "cancelled", completedAt: now },
841 ],
842 providerTurns: [{ ...base.providerTurns[0]!, status: "completed" }],
843 } as unknown as OrchestrationV2ThreadProjection);
844 assert.lengthOf(texts, 1);
845 assert.include(texts[0], "- subagent: sleep 150 (subagent agent:1)");
846 const stopped = yield* continuationTexts({
847 ...base,
848 runs: [
849 {
850 ...base.runs[0]!,
851 status: "completed",
852 startedAt: now,
853 completedAt: now,
854 restartCancelledBackgroundWork: [
855 { kind: "subagent", label: "sleep 150 (subagent agent:1)", id: "agent:1" },
856 ],
857 },
858 { ...cutWake, status: "interrupted", completedAt: now },
859 ],
860 providerTurns: [{ ...base.providerTurns[0]!, status: "completed" }],
861 } as unknown as OrchestrationV2ThreadProjection);
862 assert.lengthOf(stopped, 0);
863 }),
864 );
865
866 it.effect("does not cancel or resume a run that completes while shutdown intent commits", () =>
867 Effect.gen(function* () {
868 let projection = makeProjection();
869 const commits: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0][] = [];
870 const recovery = yield* ProviderRuntimeRecovery.make.pipe(
871 Effect.provide(
872 Layer.mergeAll(
873 ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
874 Layer.mock(ProjectionStore.ProjectionStoreV2)({
875 getRecoveryThreadIds: () => Effect.succeed([threadId]),
876 getRuntimeRecoveryProjection: () => Effect.sync(() => projection),
877 }),
878 Layer.mock(EventSink.EventSinkV2)({
879 writeWithEffects: (input) =>
880 Effect.sync(() => {
881 assert.lengthOf(input.events, 0);
882 assert.equal(input.effects[0]?.request.type, "provider-runtime.continue");
883 projection = {
884 ...projection,
885 runs: [{ ...projection.runs[0]!, status: "completed" }],
886 };
887 return [];
888 }),
889 commitCommand: (input) =>
890 Effect.sync(() => {
891 commits.push(input);
892 return { committed: true, cancelledEffectCount: 0 } as never;
893 }),
894 }),
895 IdAllocator.layer,
896 Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({
897 runRecoveryOnce: Effect.succeed(false),
898 }),
899 Layer.mock(EffectOutbox.EffectOutboxV2)({
900 reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }),
901 }),
902 ),
903 ),
904 );
905 yield* recovery.prepareForShutdown;
906 yield* recovery.reconcile("shutdown");
907 assert.isFalse(
908 commits.some((commit) => commit.events.some((event) => event.type === "run.updated")),
909 );
910 let dispatched = false;
911 yield* continueRestartedRun({ threadId, sourceRunId: runId }).pipe(
912 Effect.provide(
913 Layer.mergeAll(
914 ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
915 Layer.mock(ThreadManagementService.ThreadManagementService)({
916 getThreadRecords: () => Effect.succeed(projection),
917 recoverDelegatedTask: () => Effect.void,
918 dispatch: () =>
919 Effect.sync(() => {
920 dispatched = true;
921 return {} as never;
922 }),
923 }),
924 ),
925 ),
926 );
927 assert.isFalse(dispatched);
928 }),
929 );
930
931 const continuationDispatches = (
932 projection: OrchestrationV2ThreadProjection,
933 sourceRunId: RunId = runId,
934 ) =>
935 Effect.gen(function* () {
936 const dispatches: Array<Extract<OrchestrationV2Command, { type: "message.dispatch" }>> = [];
937 yield* continueRestartedRun({ threadId, sourceRunId }).pipe(
938 Effect.provide(
939 Layer.merge(
940 Layer.mock(ThreadManagementService.ThreadManagementService)({
941 getThreadRecords: () => Effect.succeed(projection),
942 recoverDelegatedTask: () => Effect.void,
943 dispatch: (command) => {
944 if (command.type === "message.dispatch") dispatches.push(command);
945 return Effect.succeed({} as never);
946 },
947 }),
948 ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
949 ),
950 ),
951 );
952 return dispatches;
953 });
954
955 it.effect.each([
956 [3, "running"],
957 [3, "completed"],
958 [3, "waiting"],
959 ] as const)("persists recovery after count %s and %s work", ([previousCount, status]) =>
960 Effect.gen(function* () {
961 const base = makeProjection();
962 const source = RunId.make("run:previous-loss");
963 const projection = {
964 ...base,
965 runs: [
966 {
967 ...base.runs[0]!,
968 status,
969 restartContinuationOfRunId: source,
970 requestedAt: DateTime.subtract(now, { minutes: 22 }),
971 },
972 ],
973 providerTurns: [
974 { ...base.providerTurns[0]!, status: status === "running" ? "running" : "completed" },
975 ],
976 turnItems: status === "running" ? [] : [backgroundSubagent.turnItem],
977 } as unknown as OrchestrationV2ThreadProjection;
978 const commits: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0][] = [];
979 const recovery = yield* ProviderRuntimeRecovery.make.pipe(
980 Effect.provide(
981 Layer.mergeAll(
982 ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
983 Layer.mock(ProjectionStore.ProjectionStoreV2)({
984 getRecoveryThreadIds: () => Effect.succeed([threadId]),
985 getRuntimeRecoveryProjection: () => Effect.succeed(projection),
986 }),
987 Layer.mock(EventSink.EventSinkV2)({
988 commitCommand: (input) => {
989 commits.push(input);
990 return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never);
991 },
992 }),
993 IdAllocator.layer,
994 Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({}),
995 Layer.mock(EffectOutbox.EffectOutboxV2)({
996 get: (id) => {
997 return Effect.succeed(
998 id === `effect:restart-continuation:${source}`
999 ? Option.some({
1000 request: {
1001 type: "provider-runtime.continue",
1002 sourceRunId: source,
1003 processLossCount: previousCount,
1004 },
1005 } as EffectOutbox.OrchestrationEffectV2)
1006 : Option.none(),
1007 );
1008 },
1009 listByCommandId: () => Effect.succeed([]),
1010 reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }),
1011 }),
1012 ),
1013 ),
1014 );
1015 yield* recovery.reconcile("startup");
1016 assert.deepEqual(commits[0]?.effects[0]?.request, {
1017 type: "provider-runtime.continue",
1018 sourceRunId: runId,
1019 processLossCount: status === "running" ? previousCount + 1 : 1,
1020 });
1021 }),
1022 );
1023
1024 const continuationTexts = (projection: OrchestrationV2ThreadProjection) =>
1025 continuationDispatches(projection).pipe(
1026 Effect.map((dispatches) => dispatches.map((command) => command.text)),
1027 );
1028
1029 const cutMidTurn = (extra: Record<string, unknown> = {}) => {
1030 const base = makeProjection();
1031 return {
1032 ...base,
1033 runs: [
1034 {
1035 ...base.runs[0]!,
1036 status: "cancelled",
1037 completedAt: now,
1038 userMessageId: MessageId.make("message:user"),
1039 ...extra,
1040 },
1041 ],
1042 providerTurns: [{ ...base.providerTurns[0]!, status: "cancelled" }],
1043 } as unknown as OrchestrationV2ThreadProjection;
1044 };
1045
1046 const queuedFollowUp = {
1047 id: RunId.make("run:queued"),
1048 ordinal: 2,
1049 providerInstanceId: instanceId,
1050 providerThreadId,
1051 status: "queued",
1052 queueHeld: true,
1053 };
1054
1055 it.effect("continues a cut run past queued follow-ups, which stay held", () =>
1056 Effect.gen(function* () {
1057 const live = makeProjection();
1058 assert.equal(
1059 restartContinuationRun(
1060 {
1061 ...live,
1062 runs: [...live.runs, queuedFollowUp],
1063 } as unknown as OrchestrationV2ThreadProjection,
1064 now,
1065 )?.id,
1066 runId,
1067 );
1068 const cut = cutMidTurn();
1069 const texts = yield* continuationTexts({
1070 ...cut,
1071 runs: [...cut.runs, queuedFollowUp],
1072 } as unknown as OrchestrationV2ThreadProjection);
1073 assert.deepEqual(texts, [
1074 "Continue where you left off. Your last turn was cut short, so a command or other action it started may have finished or may still be running. Before you repeat one that changes state, confirm it did not take effect and is not still running; if you cannot tell, say so instead of repeating it.",
1075 ]);
1076 }),
1077 );
1078
1079 it.effect("continues a resumed queued run that ran after an earlier continuation", () =>
1080 Effect.gen(function* () {
1081 const finishedContinuation = {
1082 id: RunId.make("run:earlier-continuation"),
1083 ordinal: 3,
1084 providerInstanceId: instanceId,
1085 providerThreadId,
1086 status: "completed",
1087 completedAt: DateTime.makeUnsafe("2026-10-03T10:00:00.000Z"),
1088 };
1089 const live = makeProjection();
1090 assert.equal(
1091 restartContinuationRun(
1092 {
1093 ...live,
1094 runs: [...live.runs, finishedContinuation],
1095 } as unknown as OrchestrationV2ThreadProjection,
1096 now,
1097 )?.id,
1098 runId,
1099 );
1100 const cut = cutMidTurn({ completedAt: DateTime.makeUnsafe("2026-10-03T10:05:00.000Z") });
1101 const texts = yield* continuationTexts({
1102 ...cut,
1103 runs: [...cut.runs, finishedContinuation],
1104 } as unknown as OrchestrationV2ThreadProjection);
1105 assert.deepEqual(texts, [
1106 "Continue where you left off. Your last turn was cut short, so a command or other action it started may have finished or may still be running. Before you repeat one that changes state, confirm it did not take effect and is not still running; if you cannot tell, say so instead of repeating it.",
1107 ]);
1108 }),
1109 );
1110
1111 it("backs off twice from a continuation the provider never took, then lets it fail", () => {
1112 const base = makeProjection();
1113 const failedStart = (id: string, of: RunId) => ({
1114 ...base.runs[0]!,
1115 id: RunId.make(id),
1116 activeAttemptId: RunAttemptId.make(`attempt:${id}`),
1117 status: "failed" as const,
1118 restartContinuationOfRunId: of,
1119 });
1120 const first = failedStart("run:retry-1", runId);
1121 const second = failedStart("run:retry-2", first.id);
1122 const third = failedStart("run:retry-3", second.id);
1123 const projection = (failureClass = "provider_error", providerTurns = base.providerTurns) =>
1124 ({
1125 runs: [{ ...base.runs[0]!, status: "cancelled" }, first, second, third],
1126 providerTurns,
1127 turnItems: [first, second, third].map((run) => ({
1128 id: `error:${run.id}`,
1129 type: "error",
1130 status: "failed",
1131 runId: run.id,
1132 nodeId: run.rootNodeId,
1133 ordinal: 1,
1134 updatedAt: now,
1135 failure: { class: failureClass, message: "Failed.", code: null, retryable: null },
1136 })),
1137 }) as unknown as Parameters<typeof continuationStartRetryDelayMs>[1];
1138 assert.deepEqual(
1139 [first, second, third].map((run) => continuationStartRetryDelayMs(run, projection())),
1140 [60_000, 180_000, undefined],
1141 );
1142 for (const failureClass of ["usage_limit", "permission_error", "validation_error"])
1143 assert.isUndefined(continuationStartRetryDelayMs(first, projection(failureClass)));
1144 // Once the provider took the prompt, starting again would repeat the turn.
1145 assert.isUndefined(
1146 continuationStartRetryDelayMs(
1147 first,
1148 projection("provider_error", [
1149 { ...base.providerTurns[0]!, runAttemptId: first.activeAttemptId, status: "failed" },
1150 ] as never),
1151 ),
1152 );
1153 });
1154
1155 it.effect("continues again after a continuation fails before its prompt", () =>
1156 Effect.gen(function* () {
1157 const cut = cutMidTurn();
1158 const failedStart = {
1159 ...cut.runs[0]!,
1160 id: RunId.make("run:failed-start"),
1161 ordinal: 2,
1162 activeAttemptId: RunAttemptId.make("attempt:failed-start"),
1163 status: "failed",
1164 restartContinuationOfRunId: runId,
1165 };
1166 const dispatches = yield* continuationDispatches(
1167 { ...cut, runs: [...cut.runs, failedStart] } as unknown as OrchestrationV2ThreadProjection,
1168 failedStart.id,
1169 );
1170 assert.deepEqual(
1171 dispatches.map((command) => command.restartContinuationOfRunId),
1172 [failedStart.id],
1173 );
1174 }),
1175 );
1176
1177 it.effect.each([
1178 ["running", null, 0],
1179 ["completed", "Done.", 1],
1180 ] as const)(
1181 "a parent's Stop on the task's run blocks the child only while the task is %s",
1182 ([status, result, expected]) =>
1183 Effect.gen(function* () {
1184 const parentThreadId = ThreadId.make("thread:parent");
1185 const parentRunId = RunId.make("run:parent");
1186 const taskId = NodeId.make("node:task");
1187 const cut = cutMidTurn();
1188 const child = {
1189 ...cut,
1190 thread: {
1191 ...cut.thread,
1192 lineage: { parentThreadId, relationshipToParent: "subagent" },
1193 forkedFrom: { type: "node", nodeId: taskId },
1194 },
1195 } as unknown as OrchestrationV2ThreadProjection;
1196 const parent = {
1197 thread: { id: parentThreadId },
1198 subagents: [{ id: taskId, runId: parentRunId, origin: "app_owned", status, result }],
1199 turnItems: [{ id: "turn-item:stop", runId: parentRunId, type: "run_interrupt_request" }],
1200 };
1201 let dispatched = 0;
1202 yield* continueRestartedRun({ threadId, sourceRunId: runId }).pipe(
1203 Effect.provide(
1204 Layer.merge(
1205 Layer.mock(ThreadManagementService.ThreadManagementService)({
1206 getThreadRecords: (id) =>
1207 Effect.succeed((id === parentThreadId ? parent : child) as never),
1208 recoverDelegatedTask: () => Effect.void,
1209 dispatch: () =>
1210 Effect.sync(() => {
1211 dispatched += 1;
1212 return {} as never;
1213 }),
1214 }),
1215 ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
1216 ),
1217 ),
1218 );
1219 assert.equal(dispatched, expected);
1220 }),
1221 );
1222
1223 it.effect("does not continue a run the user asked to stop before the restart", () =>
1224 Effect.gen(function* () {
1225 const texts = yield* continuationTexts({
1226 ...cutMidTurn(),
1227 turnItems: [{ id: "turn-item:interrupt", runId, type: "run_interrupt_request" }],
1228 } as unknown as OrchestrationV2ThreadProjection);
1229 assert.deepEqual(texts, []);
1230 }),
1231 );
1232
1233 it.effect("does not continue a cut /compact or /logout turn", () =>
1234 Effect.gen(function* () {
1235 for (const text of ["/compact", " /LOGOUT "]) {
1236 const texts = yield* continuationTexts({
1237 ...cutMidTurn(),
1238 messages: [{ id: MessageId.make("message:user"), text, attachments: [] }],
1239 } as unknown as OrchestrationV2ThreadProjection);
1240 assert.deepEqual(texts, [], text);
1241 }
1242 const texts = yield* continuationTexts({
1243 ...cutMidTurn(),
1244 messages: [{ id: MessageId.make("message:user"), text: "/compact later", attachments: [] }],
1245 } as unknown as OrchestrationV2ThreadProjection);
1246 assert.lengthOf(texts, 1);
1247 }),
1248 );
1249
1250 it.effect(
1251 "carries the note forward when its continuation was cut before reaching the provider",
1252 () =>
1253 Effect.gen(function* () {
1254 const base = makeProjection();
1255 const original = {
1256 ...base.runs[0]!,
1257 id: RunId.make("run:original"),
1258 status: "completed",
1259 activeAttemptId: RunAttemptId.make("attempt:original"),
1260 restartCancelledBackgroundWork: [{ kind: "shell", label: "sleep 25 && echo DONE" }],
1261 };
1262 const projection = {
1263 ...base,
1264 runs: [
1265 original,
1266 {
1267 ...base.runs[0]!,
1268 ordinal: 2,
1269 status: "cancelled",
1270 restartContinuationOfRunId: original.id,
1271 },
1272 ],
1273 providerTurns: [
1274 {
1275 ...base.providerTurns[0]!,
1276 runAttemptId: original.activeAttemptId,
1277 status: "completed",
1278 },
1279 ],
1280 } as unknown as OrchestrationV2ThreadProjection;
1281 const texts = yield* continuationTexts(projection);
1282 assert.lengthOf(texts, 1);
1283 assert.include(texts[0]!, "sleep 25 && echo DONE");
1284 assert.notInclude(texts[0]!, "Continue where");
1285 }),
1286 );
1287
1288 it.effect("prepares later threads' continuations when one thread fails", () =>
1289 Effect.gen(function* () {
1290 const brokenThreadId = ThreadId.make("thread:broken");
1291 const writes: Parameters<EventSink.EventSinkV2["Service"]["writeWithEffects"]>[0][] = [];
1292 const recovery = yield* ProviderRuntimeRecovery.make.pipe(
1293 Effect.provide(
1294 Layer.mergeAll(
1295 ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
1296 Layer.mock(ProjectionStore.ProjectionStoreV2)({
1297 getRecoveryThreadIds: () => Effect.succeed([brokenThreadId, threadId]),
1298 getRuntimeRecoveryProjection: (id) =>
1299 id === brokenThreadId
1300 ? Effect.fail(
1301 new ProjectionStore.ProjectionStoreThreadNotFoundError({ threadId: id }),
1302 )
1303 : Effect.succeed(makeProjection()),
1304 }),
1305 Layer.mock(EventSink.EventSinkV2)({
1306 writeWithEffects: (input) =>
1307 Effect.sync(() => {
1308 writes.push(input);
1309 return [];
1310 }),
1311 }),
1312 IdAllocator.layer,
1313 Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({}),
1314 Layer.mock(EffectOutbox.EffectOutboxV2)({}),
1315 ),
1316 ),
1317 );
1318 yield* recovery.prepareForShutdown;
1319 assert.deepEqual(
1320 writes.map((write) => write.effects[0]?.request),
1321 [{ type: "provider-runtime.continue", sourceRunId: runId, processLossCount: 0 }],
1322 );
1323 }),
1324 );
1325
1326 it.effect.each([
1327 ["user_input", "pending", "live", false],
1328 ["user_input", "expired", "not_resumable", false],
1329 ["user_input", "cancelled", "not_resumable", false],
1330 ["command", "pending", "live", false],
1331 ["command", "expired", "not_resumable", false],
1332 ["user_input", "resolved", "live", true],
1333 ["command", "cancelled", "live", true],
1334 ["user_input", "pending", "message", true],
1335 ["dynamic_tool_call", "pending", "live", true],
1336 ["auth_refresh", "pending", "live", true],
1337 ] as const)(
1338 "a %s request that is %s with %s capability allows continuation: %s",
1339 ([kind, status, capability, shouldContinue]) =>
1340 Effect.gen(function* () {
1341 const base = makeProjection();
1342 const runtimeRequests = [
1343 {
1344 id: RuntimeRequestId.make("request:restart"),
1345 nodeId: NodeId.make("node:restart-question"),
1346 providerTurnId: base.providerTurns[0]!.id,
1347 kind,
1348 status,
1349 responseCapability:
1350 capability === "live"
1351 ? { type: capability, providerSessionId: sessionId }
1352 : capability === "not_resumable"
1353 ? { type: capability, reason: "The server restarted." }
1354 : { type: capability },
1355 },
1356 ];
1357 const live = { ...base, runtimeRequests } as unknown as OrchestrationV2ThreadProjection;
1358 assert.equal(restartContinuationRun(live, now) !== undefined, shouldContinue);
1359 const cut = {
1360 ...cutMidTurn(),
1361 runtimeRequests,
1362 } as unknown as OrchestrationV2ThreadProjection;
1363 assert.lengthOf(yield* continuationTexts(cut), shouldContinue ? 1 : 0);
1364 }),
1365 );
1366
1367 it.effect("does not wake a snoozed or settled thread, but continues once a snooze elapsed", () =>
1368 Effect.gen(function* () {
1369 const live = makeProjection();
1370 for (const held of [
1371 { snoozedUntil: DateTime.add(now, { days: 1 }) },
1372 { settledOverride: "settled" as const },
1373 ]) {
1374 const thread = { ...live.thread, ...held };
1375 assert.isUndefined(restartContinuationRun({ ...live, thread }, now));
1376 assert.deepEqual(yield* continuationTexts({ ...cutMidTurn(), thread }), []);
1377 }
1378 const thread = { ...live.thread, snoozedUntil: DateTime.subtract(now, { minutes: 1 }) };
1379 assert.equal(restartContinuationRun({ ...live, thread }, now)?.id, runId);
1380 }),
1381 );
1382
1383 it.effect("does not continue a run waiting on a secret the restart closed", () =>
1384 Effect.gen(function* () {
1385 const completedAt = DateTime.makeUnsafe("2026-10-03T10:05:00.000Z");
1386 const secret = (secretStatus: string, at: DateTime.Utc | null) =>
1387 ({ type: "secret_request", runId, secretStatus, completedAt: at }) as never;
1388 assert.isUndefined(
1389 restartContinuationRun({ ...makeProjection(), turnItems: [secret("pending", null)] }, now),
1390 );
1391 const cut = cutMidTurn({ completedAt });
1392 assert.deepEqual(
1393 yield* continuationTexts({ ...cut, turnItems: [secret("cancelled", completedAt)] }),
1394 [],
1395 );
1396 assert.lengthOf(
1397 yield* continuationTexts({
1398 ...cut,
1399 turnItems: [secret("cancelled", DateTime.subtract(completedAt, { seconds: 1 }))],
1400 }),
1401 1,
1402 );
1403 }),
1404 );
1405
1406 it.effect.each(["completed", "failed", "interrupted"] as const)(
1407 "does not continue when the provider turn %s before the app run settled",
1408 (status) =>
1409 Effect.gen(function* () {
1410 const cut = cutMidTurn();
1411 assert.deepEqual(
1412 yield* continuationTexts({
1413 ...cut,
1414 providerTurns: [{ ...cut.providerTurns[0]!, status }],
1415 }),
1416 [],
1417 );
1418 }),
1419 );
1420
1421 it.effect("prepares independent continuations together and waits for durable writes", () =>
1422 Effect.gen(function* () {
1423 const bothReading = yield* Deferred.make<void>();
1424 const bothWriting = yield* Deferred.make<void>();
1425 const committed = yield* Deferred.make<void>();
1426 let reads = 0;
1427 let writes = 0;
1428 const recovery = yield* ProviderRuntimeRecovery.make.pipe(
1429 Effect.provide(
1430 Layer.mergeAll(
1431 ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
1432 Layer.mock(ProjectionStore.ProjectionStoreV2)({
1433 getRecoveryThreadIds: () => Effect.succeed([threadId, ThreadId.make("thread:second")]),
1434 getRuntimeRecoveryProjection: () =>
1435 Effect.gen(function* () {
1436 if (++reads === 2) yield* Deferred.succeed(bothReading, undefined);
1437 yield* Deferred.await(bothReading);
1438 return makeProjection();
1439 }),
1440 }),
1441 Layer.mock(EventSink.EventSinkV2)({
1442 writeWithEffects: () =>
1443 Effect.gen(function* () {
1444 if (++writes === 2) yield* Deferred.succeed(bothWriting, undefined);
1445 yield* Deferred.await(committed);
1446 return [];
1447 }),
1448 }),
1449 IdAllocator.layer,
1450 Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({}),
1451 Layer.mock(EffectOutbox.EffectOutboxV2)({}),
1452 ),
1453 ),
1454 );
1455 const fiber = yield* recovery.prepareForShutdown.pipe(Effect.forkChild);
1456 yield* Deferred.await(bothWriting);
1457 assert.equal(writes, 2);
1458 assert.isUndefined(fiber.pollUnsafe());
1459 yield* Deferred.succeed(committed, undefined);
1460 yield* Fiber.join(fiber);
1461 }),
1462 );
1463
1464 it.effect("skips restart discovery when no project enables continuation", () =>
1465 Effect.gen(function* () {
1466 const recovery = yield* ProviderRuntimeRecovery.make.pipe(
1467 Effect.provide(
1468 Layer.mergeAll(
1469 ServerSettings.layerTest({ continueThreadsAfterServerUpdate: false }),
1470 Layer.mock(ProjectionStore.ProjectionStoreV2)({}),
1471 Layer.mock(EventSink.EventSinkV2)({}),
1472 IdAllocator.layer,
1473 Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({}),
1474 Layer.mock(EffectOutbox.EffectOutboxV2)({}),
1475 ),
1476 ),
1477 );
1478 yield* recovery.prepareForShutdown;
1479 yield* recovery.prepareForShutdown;
1480 }),
1481 );